move src/examples to examples

examples/starter-apps/chatbot.py

@@ -0,0 +1,145 @@
+import asyncio
+import aiohttp
+import logging
+import os
+from PIL import Image
+from typing import AsyncGenerator
+
+from dailyai.pipeline.aggregators import (
+    LLMResponseAggregator,
+    UserResponseAggregator,
+)
+from dailyai.pipeline.frames import (
+    ImageFrame,
+    SpriteFrame,
+    Frame,
+    LLMResponseEndFrame,
+    LLMMessagesQueueFrame,
+    AudioFrame,
+    PipelineStartedFrame,
+)
+from dailyai.services.ai_services import AIService
+from dailyai.pipeline.pipeline import Pipeline
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+sprites = []
+
+script_dir = os.path.dirname(__file__)
+
+for i in range(1, 26):
+    # Build the full path to the image file
+    full_path = os.path.join(script_dir, f"assets/robot0{i}.png")
+    # Get the filename without the extension to use as the dictionary key
+    # Open the image and convert it to bytes
+    with Image.open(full_path) as img:
+        sprites.append(img.tobytes())
+
+flipped = sprites[::-1]
+sprites.extend(flipped)
+# When the bot isn't talking, show a static image of the cat listening
+quiet_frame = ImageFrame("", sprites[0])
+talking_frame = SpriteFrame(images=sprites)
+
+
+class TalkingAnimation(AIService):
+    """
+    This class starts a talking animation when it receives an first AudioFrame,
+    and then returns to a "quiet" sprite when it sees a LLMResponseEndFrame.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._is_talking = False
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, AudioFrame):
+            if not self._is_talking:
+                yield talking_frame
+                yield frame
+                self._is_talking = True
+            else:
+                yield frame
+        elif isinstance(frame, LLMResponseEndFrame):
+            yield quiet_frame
+            yield frame
+            self._is_talking = False
+        else:
+            yield frame
+
+
+class AnimationInitializer(AIService):
+    def __init__(self):
+        super().__init__()
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, PipelineStartedFrame):
+            yield quiet_frame
+            yield frame
+        else:
+            yield frame
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransportService(
+            room_url,
+            token,
+            "Chatbot",
+            duration_minutes=5,
+            start_transcription=True,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            camera_enabled=True,
+            camera_width=1024,
+            camera_height=576,
+            vad_enabled=True,
+        )
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id="pNInz6obpgDQGcFmaJgB",
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        ta = TalkingAnimation()
+        ai = AnimationInitializer()
+        pipeline = Pipeline([ai, llm, tts, ta])
+        messages = [
+            {
+                "role": "system",
+                "content": "You are Chatbot, a friendly, helpful robot. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way, but keep your responses brief. Start by introducing yourself.",
+            },
+        ]
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            print(f"!!! in here, pipeline.source is {pipeline.source}")
+            await pipeline.queue_frames([LLMMessagesQueueFrame(messages)])
+
+        async def run_conversation():
+
+            await transport.run_interruptible_pipeline(
+                pipeline,
+                post_processor=LLMResponseAggregator(messages),
+                pre_processor=UserResponseAggregator(messages),
+            )
+
+        transport.transcription_settings["extra"]["endpointing"] = True
+        transport.transcription_settings["extra"]["punctuate"] = True
+        await asyncio.gather(transport.run(), run_conversation())
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/starter-apps/patient-intake.py

@@ -0,0 +1,353 @@
+import copy
+import aiohttp
+import asyncio
+import json
+import random
+import logging
+import os
+import re
+import wave
+from typing import AsyncGenerator, List
+from PIL import Image
+from dailyai.pipeline.opeanai_llm_aggregator import (
+    OpenAIAssistantContextAggregator,
+    OpenAIUserContextAggregator,
+)
+
+from dailyai.pipeline.pipeline import Pipeline
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.openai_llm_context import OpenAILLMContext
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.deepgram_ai_services import DeepgramTTSService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from examples.support.runner import configure
+from dailyai.pipeline.frames import (
+    OpenAILLMContextFrame,
+    TranscriptionQueueFrame,
+    Frame,
+    LLMFunctionCallFrame,
+    LLMFunctionStartFrame,
+    AudioFrame,
+)
+from dailyai.services.ai_services import FrameLogger, AIService
+from openai._types import NotGiven, NOT_GIVEN
+
+from openai.types.chat import (
+    ChatCompletionToolParam,
+)
+
+logging.basicConfig(format="%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+sounds = {}
+sound_files = [
+    "clack-short.wav",
+    "clack.wav",
+    "clack-short-quiet.wav",
+    "ding.wav",
+    "ding2.wav",
+]
+
+script_dir = os.path.dirname(__file__)
+
+for file in sound_files:
+    # Build the full path to the sound file
+    full_path = os.path.join(script_dir, "assets", file)
+    # Get the filename without the extension to use as the dictionary key
+    filename = os.path.splitext(os.path.basename(full_path))[0]
+    # Open the sound and convert it to bytes
+    with wave.open(full_path) as audio_file:
+        sounds[file] = audio_file.readframes(-1)
+
+
+steps = [{"prompt": "Start by introducing yourself. Then, ask the user to confirm their identity by telling you their birthday, including the year. When they answer with their birthday, call the verify_birthday function.",
+          "run_async": False,
+          "failed": "The user provided an incorrect birthday. Ask them for their birthday again. When they answer, call the verify_birthday function.",
+          "tools": [{"type": "function",
+                     "function": {"name": "verify_birthday",
+                                  "description": "Use this function to verify the user has provided their correct birthday.",
+                                  "parameters": {"type": "object",
+                                                 "properties": {"birthday": {"type": "string",
+                                                                             "description": "The user's birthdate, including the year. The user can provide it in any format, but convert it to YYYY-MM-DD format to call this function.",
+                                                                             }},
+                                                 },
+                                  },
+                     }],
+          },
+         {"prompt": "Next, thank the user for confirming their identity, then ask the user to list their current prescriptions. Each prescription needs to have a medication name and a dosage. Do not call the list_prescriptions function with any unknown dosages.",
+          "run_async": True,
+          "tools": [{"type": "function",
+                     "function": {"name": "list_prescriptions",
+                                  "description": "Once the user has provided a list of their prescription medications, call this function.",
+                                  "parameters": {"type": "object",
+                                                 "properties": {"prescriptions": {"type": "array",
+                                                                                  "items": {"type": "object",
+                                                                                            "properties": {"medication": {"type": "string",
+                                                                                                                          "description": "The medication's name",
+                                                                                                                          },
+                                                                                                           "dosage": {"type": "string",
+                                                                                                                      "description": "The prescription's dosage",
+                                                                                                                      },
+                                                                                                           },
+                                                                                            },
+                                                                                  }},
+                                                 },
+                                  },
+                     }],
+          },
+         {"prompt": "Next, ask the user if they have any allergies. Once they have listed their allergies or confirmed they don't have any, call the list_allergies function.",
+          "run_async": True,
+          "tools": [{"type": "function",
+                     "function": {"name": "list_allergies",
+                                  "description": "Once the user has provided a list of their allergies, call this function.",
+                                  "parameters": {"type": "object",
+                                                 "properties": {"allergies": {"type": "array",
+                                                                              "items": {"type": "object",
+                                                                                        "properties": {"name": {"type": "string",
+                                                                                                                "description": "What the user is allergic to",
+                                                                                                                }},
+                                                                                        },
+                                                                              }},
+                                                 },
+                                  },
+                     }],
+          },
+         {"prompt": "Now ask the user if they have any medical conditions the doctor should know about. Once they've answered the question, call the list_conditions function.",
+          "run_async": True,
+          "tools": [{"type": "function",
+                     "function": {"name": "list_conditions",
+                                  "description": "Once the user has provided a list of their medical conditions, call this function.",
+                                  "parameters": {"type": "object",
+                                                 "properties": {"conditions": {"type": "array",
+                                                                               "items": {"type": "object",
+                                                                                         "properties": {"name": {"type": "string",
+                                                                                                                 "description": "The user's medical condition",
+                                                                                                                 }},
+                                                                                         },
+                                                                               }},
+                                                 },
+                                  },
+                     },
+                    ],
+          },
+         {"prompt": "Finally, ask the user the reason for their doctor visit today. Once they answer, call the list_visit_reasons function.",
+          "run_async": True,
+          "tools": [{"type": "function",
+                     "function": {"name": "list_visit_reasons",
+                                  "description": "Once the user has provided a list of the reasons they are visiting a doctor today, call this function.",
+                                  "parameters": {"type": "object",
+                                                 "properties": {"visit_reasons": {"type": "array",
+                                                                                  "items": {"type": "object",
+                                                                                            "properties": {"name": {"type": "string",
+                                                                                                                    "description": "The user's reason for visiting the doctor",
+                                                                                                                    }},
+                                                                                            },
+                                                                                  }},
+                                                 },
+                                  },
+                     }],
+          },
+         {"prompt": "Now, thank the user and end the conversation.",
+          "run_async": True,
+          "tools": [],
+          },
+         {"prompt": "",
+          "run_async": True,
+          "tools": []},
+         ]
+current_step = 0
+
+
+class ChecklistProcessor(AIService):
+
+    def __init__(
+        self,
+        context: OpenAILLMContext,
+        llm: AIService,
+        tools: List[ChatCompletionToolParam] | NotGiven = NOT_GIVEN,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self._context: OpenAILLMContext = context
+        self._llm = llm
+        self._id = "You are Jessica, an agent for a company called Tri-County Health Services. Your job is to collect important information from the user before their doctor visit. You're talking to Chad Bailey. You should address the user by their first name and be polite and professional. You're not a medical professional, so you shouldn't provide any advice. Keep your responses short. Your job is to collect information to give to a doctor. Don't make assumptions about what values to plug into functions. Ask for clarification if a user response is ambiguous."
+        self._acks = ["One sec.", "Let me confirm that.", "Thanks.", "OK."]
+
+        # Create an allowlist of functions that the LLM can call
+        self._functions = [
+            "verify_birthday",
+            "list_prescriptions",
+            "list_allergies",
+            "list_conditions",
+            "list_visit_reasons",
+        ]
+
+        self._context.add_message(
+            {"role": "system", "content": f"{self._id} {steps[0]['prompt']}"}
+        )
+
+        if tools:
+            self._context.set_tools(tools)
+
+    def verify_birthday(self, args):
+        return args["birthday"] == "1983-01-01"
+
+    def list_prescriptions(self, args):
+        # print(f"--- Prescriptions: {args['prescriptions']}\n")
+        pass
+
+    def list_allergies(self, args):
+        # print(f"--- Allergies: {args['allergies']}\n")
+        pass
+
+    def list_conditions(self, args):
+        # print(f"--- Medical Conditions: {args['conditions']}")
+        pass
+
+    def list_visit_reasons(self, args):
+        # print(f"Visit Reasons: {args['visit_reasons']}")
+        pass
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        global current_step
+        this_step = steps[current_step]
+        self._context.set_tools(this_step["tools"])
+        if isinstance(frame, LLMFunctionStartFrame):
+            print(f"... Preparing function call: {frame.function_name}")
+            self._function_name = frame.function_name
+            if this_step["run_async"]:
+                # Get the LLM talking about the next step before getting the rest
+                # of the function call completion
+                current_step += 1
+                self._context.add_message(
+                    {"role": "system", "content": steps[current_step]["prompt"]}
+                )
+                yield OpenAILLMContextFrame(self._context)
+
+                local_context = copy.deepcopy(self._context)
+                local_context.set_tool_choice("none")
+                async for frame in llm.process_frame(
+                    OpenAILLMContextFrame(local_context)
+                ):
+                    yield frame
+            else:
+                # Insert a quick response while we run the function
+                yield AudioFrame(sounds["ding2.wav"])
+                pass
+        elif isinstance(frame, LLMFunctionCallFrame):
+
+            if frame.function_name and frame.arguments:
+                print(
+                    f"--> Calling function: {frame.function_name} with arguments:")
+                pretty_json = re.sub(
+                    "\n", "\n    ", json.dumps(
+                        json.loads(
+                            frame.arguments), indent=2))
+                print(f"--> {pretty_json}\n")
+                if frame.function_name not in self._functions:
+                    raise Exception(
+                        f"The LLM tried to call a function named {frame.function_name}, which isn't in the list of known functions. Please check your prompt and/or self._functions."
+                    )
+                fn = getattr(self, frame.function_name)
+                result = fn(json.loads(frame.arguments))
+
+                if not this_step["run_async"]:
+                    if result:
+                        current_step += 1
+                        self._context.add_message(
+                            {"role": "system", "content": steps[current_step]["prompt"]}
+                        )
+                        yield OpenAILLMContextFrame(self._context)
+
+                        local_context = copy.deepcopy(self._context)
+                        local_context.set_tool_choice("none")
+                        async for frame in llm.process_frame(
+                            OpenAILLMContextFrame(local_context)
+                        ):
+                            yield frame
+                    else:
+                        self._context.add_message(
+                            {"role": "system", "content": this_step["failed"]}
+                        )
+                        yield OpenAILLMContextFrame(self._context)
+
+                        local_context = copy.deepcopy(self._context)
+                        local_context.set_tool_choice("none")
+                        async for frame in llm.process_frame(
+                            OpenAILLMContextFrame(local_context)
+                        ):
+                            yield frame
+                    print(f"<-- Verify result: {result}\n")
+
+        else:
+            yield frame
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        global transport
+        global llm
+        global tts
+
+        transport = DailyTransportService(
+            room_url,
+            token,
+            "Intake Bot",
+            5,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            camera_enabled=False,
+            start_transcription=True,
+            vad_enabled=True,
+        )
+
+        messages = []
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
+            model="gpt-4-1106-preview",
+        )
+        # tts = DeepgramTTSService(
+        #     aiohttp_session=session,
+        #     api_key=os.getenv("DEEPGRAM_API_KEY"),
+        #     voice="aura-asteria-en",
+        # )
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id="XrExE9yKIg1WjnnlVkGX",
+        )
+        context = OpenAILLMContext(
+            messages=messages,
+        )
+
+        checklist = ChecklistProcessor(context, llm)
+        fl = FrameLogger("FRAME LOGGER 1:")
+        fl2 = FrameLogger("FRAME LOGGER 2:")
+        pipeline = Pipeline(processors=[fl, llm, fl2, checklist, tts])
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            await pipeline.queue_frames([OpenAILLMContextFrame(context)])
+
+        async def handle_intake():
+            await transport.run_interruptible_pipeline(
+                pipeline,
+                post_processor=OpenAIAssistantContextAggregator(context),
+                pre_processor=OpenAIUserContextAggregator(context),
+            )
+
+        transport.transcription_settings["extra"]["endpointing"] = True
+        transport.transcription_settings["extra"]["punctuate"] = True
+        try:
+            await asyncio.gather(transport.run(), handle_intake())
+        except (asyncio.CancelledError, KeyboardInterrupt):
+            print("whoops")
+            transport.stop()
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/starter-apps/storybot.py

@@ -0,0 +1,294 @@
+import aiohttp
+import asyncio
+import json
+import random
+import logging
+import os
+import re
+import wave
+from typing import AsyncGenerator
+from PIL import Image
+
+from dailyai.pipeline.pipeline import Pipeline
+from dailyai.pipeline.frame_processor import FrameProcessor
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
+from dailyai.services.fal_ai_services import FalImageGenService
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.deepgram_ai_services import DeepgramTTSService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.pipeline.aggregators import (
+    LLMAssistantContextAggregator,
+    LLMContextAggregator,
+    LLMUserContextAggregator,
+    ParallelPipeline,
+    UserResponseAggregator,
+    LLMResponseAggregator,
+)
+from examples.support.runner import configure
+from dailyai.pipeline.frames import (
+    EndPipeFrame,
+    LLMMessagesQueueFrame,
+    TranscriptionQueueFrame,
+    Frame,
+    TextFrame,
+    LLMFunctionCallFrame,
+    LLMFunctionStartFrame,
+    LLMResponseEndFrame,
+    StartFrame,
+    AudioFrame,
+    SpriteFrame,
+    ImageFrame,
+    UserStoppedSpeakingFrame,
+)
+from dailyai.services.ai_services import FrameLogger, AIService
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+sounds = {}
+images = {}
+sound_files = ["talking.wav", "listening.wav", "ding3.wav"]
+image_files = ["grandma-writing.png", "grandma-listening.png"]
+script_dir = os.path.dirname(__file__)
+
+for file in sound_files:
+    # Build the full path to the sound file
+    full_path = os.path.join(script_dir, "assets", file)
+    # Get the filename without the extension to use as the dictionary key
+    filename = os.path.splitext(os.path.basename(full_path))[0]
+    # Open the sound and convert it to bytes
+    with wave.open(full_path) as audio_file:
+        sounds[file] = audio_file.readframes(-1)
+
+for file in image_files:
+    # Build the full path to the image file
+    full_path = os.path.join(script_dir, "assets", file)
+    # Get the filename without the extension to use as the dictionary key
+    filename = os.path.splitext(os.path.basename(full_path))[0]
+    # Open the image and convert it to bytes
+    with Image.open(full_path) as img:
+        images[file] = img.tobytes()
+
+
+class StoryStartFrame(TextFrame):
+    pass
+
+
+class StoryPageFrame(TextFrame):
+    pass
+
+
+class StoryPromptFrame(TextFrame):
+    pass
+
+
+class StoryProcessor(FrameProcessor):
+    def __init__(self, messages, story):
+        self._messages = messages
+        self._text = ""
+        self._story = story
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        """
+        The response from the LLM service looks like:
+        A comment about the user's choice
+        [start] (when the cat starts telling parts of the story)
+        A sentence of the story
+        [break] (between each sentence/'page' of the story)
+        [prompt] (when the cat asks the user to make a decision)
+        Question about the next part of the story
+
+        1. Catch the frames that are generated by the LLM service
+        """
+        if isinstance(frame, UserStoppedSpeakingFrame):
+            yield ImageFrame(None, images["grandma-writing.png"])
+            yield AudioFrame(sounds["talking.wav"])
+
+        elif isinstance(frame, TextFrame):
+            self._text += frame.text
+
+            if re.findall(r".*\[[sS]tart\].*", self._text):
+                # Then we have the intro. Send it to speech ASAP
+                self._text = self._text.replace("[Start]", "")
+                self._text = self._text.replace("[start]", "")
+
+                self._text = self._text.replace("\n", " ")
+                if len(self._text) > 2:
+                    yield ImageFrame(None, images["grandma-writing.png"])
+                    yield StoryStartFrame(self._text)
+                    yield AudioFrame(sounds["ding3.wav"])
+                self._text = ""
+
+            elif re.findall(r".*\[[bB]reak\].*", self._text):
+                # Then it's a page of the story. Get an image too
+                self._text = self._text.replace("[Break]", "")
+                self._text = self._text.replace("[break]", "")
+                self._text = self._text.replace("\n", " ")
+                if len(self._text) > 2:
+                    self._story.append(self._text)
+                    yield StoryPageFrame(self._text)
+                    yield AudioFrame(sounds["ding3.wav"])
+
+                self._text = ""
+            elif re.findall(r".*\[[pP]rompt\].*", self._text):
+                # Then it's question time. Flush any
+                # text here as a story page, then set
+                # the var to get to prompt mode
+                # cb: trying scene now
+                # self.handle_chunk(self._text)
+                self._text = self._text.replace("[Prompt]", "")
+                self._text = self._text.replace("[prompt]", "")
+
+                self._text = self._text.replace("\n", " ")
+                if len(self._text) > 2:
+                    self._story.append(self._text)
+                    yield StoryPageFrame(self._text)
+            else:
+                # After the prompt thing, we'll catch an LLM end to get the
+                # last bit
+                pass
+        elif isinstance(frame, LLMResponseEndFrame):
+            yield ImageFrame(None, images["grandma-writing.png"])
+            yield StoryPromptFrame(self._text)
+            self._text = ""
+            yield frame
+            yield ImageFrame(None, images["grandma-listening.png"])
+            yield AudioFrame(sounds["listening.wav"])
+
+        else:
+            # pass through everything that's not a TextFrame
+            yield frame
+
+
+class StoryImageGenerator(FrameProcessor):
+    def __init__(self, story, llm, img):
+        self._story = story
+        self._llm = llm
+        self._img = img
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, StoryPageFrame):
+            if len(self._story) == 1:
+                prompt = f'You are an illustrator for a children\'s story book. Generate a prompt for DALL-E to create an illustration for the first page of the book, which reads: "{self._story[0]}"\n\n Your response should start with the phrase "Children\'s book illustration of".'
+            else:
+                prompt = f"You are an illustrator for a children's story book. Here is the story so far:\n\n\"{' '.join(self._story[:-1])}\"\n\nGenerate a prompt for DALL-E to create an illustration for the next page. Here's the sentence for the next page:\n\n\"{self._story[-1:][0]}\"\n\n Your response should start with the phrase \"Children's book illustration of\"."
+            msgs = [{"role": "system", "content": prompt}]
+            image_prompt = ""
+            async for f in self._llm.process_frame(LLMMessagesQueueFrame(msgs)):
+                if isinstance(f, TextFrame):
+                    image_prompt += f.text
+            async for f in self._img.process_frame(TextFrame(image_prompt)):
+                yield f
+            # Yield the original StoryPageFrame for basic image/audio sync
+            yield frame
+        else:
+            yield frame
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        messages = [
+            {
+                "role": "system",
+                "content": "You are a storytelling grandma who loves to make up fantastic, fun, and educational stories for children between the ages of 5 and 10 years old. Your stories are full of friendly, magical creatures. Your stories are never scary. Each sentence of your story will become a page in a storybook. Stop after 3-4 sentences and give the child a choice to make that will influence the next part of the story. Once the child responds, start by saying something nice about the choice they made, then include [start] in your response. Include [break] after each sentence of the story. Include [prompt] between the story and the prompt.",
+            }
+        ]
+
+        story = []
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
+            model="gpt-4-1106-preview",
+        )  # gpt-4-1106-preview
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id="Xb7hH8MSUJpSbSDYk0k2",
+        )  # matilda
+        img = FalImageGenService(
+            image_size="1024x1024",
+            aiohttp_session=session,
+            key_id=os.getenv("FAL_KEY_ID"),
+            key_secret=os.getenv("FAL_KEY_SECRET"),
+        )
+        lra = LLMResponseAggregator(messages)
+        ura = UserResponseAggregator(messages)
+        sp = StoryProcessor(messages, story)
+        sig = StoryImageGenerator(story, llm, img)
+
+        transport = DailyTransportService(
+            room_url,
+            token,
+            "Storybot",
+            5,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            camera_enabled=True,
+            camera_width=1024,
+            camera_height=1024,
+            start_transcription=True,
+            vad_enabled=True,
+            vad_stop_s=1.5,
+        )
+
+        start_story_event = asyncio.Event()
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            start_story_event.set()
+
+        async def storytime():
+            await start_story_event.wait()
+
+            # We're being a bit tricky here by using a special system prompt to
+            # ask the user for a story topic. After their intial response, we'll
+            # use a different system prompt to create story pages.
+            intro_messages = [
+                {
+                    "role": "system",
+                    "content": "You are a storytelling grandma who loves to make up fantastic, fun, and educational stories for children between the ages of 5 and 10 years old. Your stories are full of friendly, magical creatures. Your stories are never scary. Begin by asking what a child wants you to tell a story about. Keep your reponse to only a few sentences.",
+                }
+            ]
+            lca = LLMAssistantContextAggregator(messages)
+            local_pipeline = Pipeline(
+                [llm, lca, tts], sink=transport.send_queue)
+            await local_pipeline.queue_frames(
+                [
+                    ImageFrame(None, images["grandma-listening.png"]),
+                    LLMMessagesQueueFrame(intro_messages),
+                    AudioFrame(sounds["listening.wav"]),
+                    EndPipeFrame(),
+                ]
+            )
+            await local_pipeline.run_pipeline()
+
+            fl = FrameLogger("### After Image Generation")
+            pipeline = Pipeline(
+                processors=[
+                    ura,
+                    llm,
+                    sp,
+                    sig,
+                    fl,
+                    tts,
+                    lra,
+                ]
+            )
+            await transport.run_pipeline(
+                pipeline,
+            )
+
+        transport.transcription_settings["extra"]["endpointing"] = True
+        transport.transcription_settings["extra"]["punctuate"] = True
+        try:
+            await asyncio.gather(transport.run(), storytime())
+        except (asyncio.CancelledError, KeyboardInterrupt):
+            print("whoops")
+            transport.stop()
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/starter-apps/translator.py

@@ -0,0 +1,85 @@
+import asyncio
+import aiohttp
+import logging
+import os
+from PIL import Image
+from typing import AsyncGenerator
+
+from dailyai.pipeline.aggregators import (
+    LLMResponseAggregator,
+    UserResponseAggregator,
+    SentenceAggregator,
+)
+from dailyai.pipeline.frames import Frame, LLMMessagesQueueFrame, TextFrame
+from dailyai.pipeline.frame_processor import FrameProcessor
+from dailyai.services.ai_services import AIService, FrameLogger
+from dailyai.pipeline.pipeline import Pipeline
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.azure_ai_services import AzureTTSService
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+"""
+This example looks a bit different than the chatbot example, because it isn't waiting on the user to stop talking to start translating.
+It also isn't saving what the user or bot says into the context object for use in subsequent interactions.
+"""
+
+
+# We need to use a custom service here to yield LLM frames without saving
+# any context
+class TranslationProcessor(FrameProcessor):
+    def __init__(self, language):
+        self._language = language
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, TextFrame):
+            context = [
+                {
+                    "role": "system",
+                    "content": f"You will be provided with a sentence in English, and your task is to translate it into {self._language}.",
+                },
+                {"role": "user", "content": frame.text},
+            ]
+            yield LLMMessagesQueueFrame(context)
+        else:
+            yield frame
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransportService(
+            room_url,
+            token,
+            "Translator",
+            duration_minutes=5,
+            start_transcription=True,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            camera_enabled=False,
+            vad_enabled=True,
+        )
+        tts = AzureTTSService(
+            api_key=os.getenv("AZURE_SPEECH_API_KEY"),
+            region=os.getenv("AZURE_SPEECH_REGION"),
+            voice="es-ES-AlvaroNeural",
+        )
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
+            model="gpt-4-turbo-preview")
+        sa = SentenceAggregator()
+        tp = TranslationProcessor("Spanish")
+        pipeline = Pipeline([sa, tp, llm, tts])
+
+        transport.transcription_settings["extra"]["endpointing"] = True
+        transport.transcription_settings["extra"]["punctuate"] = True
+        await transport.run(pipeline)
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))