From 067ddfe505790a91103bfde42c50b6c416fc9cd6 Mon Sep 17 00:00:00 2001 From: chadbailey59 Date: Wed, 22 Jan 2025 15:20:21 -0600 Subject: [PATCH] Storytelling chatbot updates (#1066) * initial changes for gemini storybot * storybot updates for gemini * more storybot updates * interim interruptible commit * cleanup * cleanup * cleanup * cleanup --- examples/storytelling-chatbot/README.md | 2 +- examples/storytelling-chatbot/env.example | 1 + .../frontend/components/Story.tsx | 10 ++- .../UserInputIndicator.module.css | 17 ---- .../components/UserInputIndicator/index.tsx | 21 ++++- .../storytelling-chatbot/requirements.txt | 2 +- examples/storytelling-chatbot/src/bot.py | 75 ++++++++--------- .../storytelling-chatbot/src/bot_runner.py | 6 +- .../storytelling-chatbot/src/processors.py | 81 +++++++++---------- examples/storytelling-chatbot/src/prompts.py | 55 +++++++------ 10 files changed, 139 insertions(+), 131 deletions(-) diff --git a/examples/storytelling-chatbot/README.md b/examples/storytelling-chatbot/README.md index b2c1e3d8a..355608b20 100644 --- a/examples/storytelling-chatbot/README.md +++ b/examples/storytelling-chatbot/README.md @@ -66,7 +66,7 @@ The build UI files can be found in `frontend/out` Start the API / bot manager: -`python src/bot_runner.py` +`python src/bot_runner.py --host localhost` If you'd like to run a custom domain or port: diff --git a/examples/storytelling-chatbot/env.example b/examples/storytelling-chatbot/env.example index 7060aef52..cdff19149 100644 --- a/examples/storytelling-chatbot/env.example +++ b/examples/storytelling-chatbot/env.example @@ -4,6 +4,7 @@ ELEVENLABS_API_KEY= ELEVENLABS_VOICE_ID= FAL_KEY= OPENAI_API_KEY= +GOOGLE_API_KEY= ENV= # dev | production RUN_AS_VM= # Set this if you want to run bots on process (not launch a new VM) \ No newline at end of file diff --git a/examples/storytelling-chatbot/frontend/components/Story.tsx b/examples/storytelling-chatbot/frontend/components/Story.tsx index 100c83fc1..49e489e56 100644 --- a/examples/storytelling-chatbot/frontend/components/Story.tsx +++ b/examples/storytelling-chatbot/frontend/components/Story.tsx @@ -1,4 +1,4 @@ -import React, { useState } from "react"; +import React, { useState, useEffect } from "react"; import { useDaily, useParticipantIds, @@ -33,7 +33,9 @@ const Story: React.FC = ({ handleLeave }) => { setTimeout(() => daily.setLocalAudio(true), 500); setStoryState("user"); } else { - daily.setLocalAudio(false); + // Uncomment the next line to mute the mic while the + // assistant it talking. Leave it commented to allow for interruptions + // daily.setLocalAudio(false); setStoryState("assistant"); } }, @@ -58,7 +60,7 @@ const Story: React.FC = ({ handleLeave }) => { {participantIds.length >= 1 ? ( ) : ( @@ -71,7 +73,7 @@ const Story: React.FC = ({ handleLeave }) => { )} - + ); }; diff --git a/examples/storytelling-chatbot/frontend/components/UserInputIndicator/UserInputIndicator.module.css b/examples/storytelling-chatbot/frontend/components/UserInputIndicator/UserInputIndicator.module.css index bc57e4d5a..b1c515345 100644 --- a/examples/storytelling-chatbot/frontend/components/UserInputIndicator/UserInputIndicator.module.css +++ b/examples/storytelling-chatbot/frontend/components/UserInputIndicator/UserInputIndicator.module.css @@ -43,25 +43,8 @@ transition: opacity 0.5s ease; } - -@keyframes pulse { - 0% { - outline-width: 6px; - @apply outline-teal-500/10; - } - 50% { - outline-width: 24px; - @apply outline-teal-500/50; - } - 100% { - outline-width: 6px; - @apply outline-teal-500/10; - } -} - .micIconActive{ @apply bg-teal-950 border-teal-500 outline-teal-500/20; - animation: pulse 2s infinite ease-in-out; } .micIconActive svg{ diff --git a/examples/storytelling-chatbot/frontend/components/UserInputIndicator/index.tsx b/examples/storytelling-chatbot/frontend/components/UserInputIndicator/index.tsx index c45d6536d..8120564f9 100644 --- a/examples/storytelling-chatbot/frontend/components/UserInputIndicator/index.tsx +++ b/examples/storytelling-chatbot/frontend/components/UserInputIndicator/index.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect } from "react"; +import React, { useState, useEffect, useRef } from "react"; import { useAppMessage } from "@daily-co/daily-react"; import { DailyEventObjectAppMessage } from "@daily-co/daily-js"; @@ -13,12 +13,31 @@ interface Props { export default function UserInputIndicator({ active }: Props) { const [transcription, setTranscription] = useState([]); + const timeoutRef = useRef(); + + const resetTimeout = () => { + if (timeoutRef.current) { + clearTimeout(timeoutRef.current); + } + timeoutRef.current = setTimeout(() => { + setTranscription([]); + }, 5000); + }; + + useEffect(() => { + return () => { + if (timeoutRef.current) { + clearTimeout(timeoutRef.current); + } + }; + }, []); useAppMessage({ onAppMessage: (e: DailyEventObjectAppMessage) => { if (e.fromId && e.fromId === "transcription") { if (e.data.user_id === "" && e.data.is_final) { setTranscription((t) => [...t, ...e.data.text.split(" ")]); + resetTimeout(); } } }, diff --git a/examples/storytelling-chatbot/requirements.txt b/examples/storytelling-chatbot/requirements.txt index 0cebe6edb..970d3898c 100644 --- a/examples/storytelling-chatbot/requirements.txt +++ b/examples/storytelling-chatbot/requirements.txt @@ -2,4 +2,4 @@ async_timeout fastapi uvicorn python-dotenv -pipecat-ai[daily,elevenlabs,openai,fal] +pipecat-ai[daily,openai,fal,google,cartesia] diff --git a/examples/storytelling-chatbot/src/bot.py b/examples/storytelling-chatbot/src/bot.py index bc4eb62cc..8be1e785d 100644 --- a/examples/storytelling-chatbot/src/bot.py +++ b/examples/storytelling-chatbot/src/bot.py @@ -13,16 +13,23 @@ import aiohttp from dotenv import load_dotenv from loguru import logger from processors import StoryImageProcessor, StoryProcessor -from prompts import CUE_USER_TURN, LLM_BASE_PROMPT, LLM_INTRO_PROMPT +from prompts import CUE_USER_TURN, LLM_BASE_PROMPT from utils.helpers import load_images, load_sounds -from pipecat.frames.frames import EndFrame, LLMMessagesFrame, StopTaskFrame +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import EndFrame, StopTaskFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) +from pipecat.processors.logger import FrameLogger +from pipecat.services.cartesia import CartesiaHttpTTSService, CartesiaTTSService from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.fal import FalImageGenService +from pipecat.services.google import GoogleLLMService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import ( DailyParams, @@ -53,6 +60,7 @@ async def main(room_url, token=None): camera_out_width=768, camera_out_height=768, transcription_enabled=True, + vad_analyzer=SileroVADAnalyzer(), vad_enabled=True, ), ) @@ -61,11 +69,10 @@ async def main(room_url, token=None): # -------------- Services --------------- # - llm_service = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + llm_service = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY")) tts_service = ElevenLabsTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), + api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID") ) fal_service_params = FalImageGenService.InputParams( @@ -74,7 +81,7 @@ async def main(room_url, token=None): fal_service = FalImageGenService( aiohttp_session=session, - model="fal-ai/fast-lightning-sdxl", + model="fal-ai/stable-diffusion-v35-medium", params=fal_service_params, key=os.getenv("FAL_KEY"), ) @@ -97,35 +104,8 @@ async def main(room_url, token=None): runner = PipelineRunner() - # The intro pipeline is used to start - # the story (as per LLM_INTRO_PROMPT) - intro_pipeline = Pipeline([llm_service, tts_service, transport.output()]) - - intro_task = PipelineTask(intro_pipeline) - logger.debug("Waiting for participant...") - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - logger.debug("Participant joined, storytime commence!") - await transport.capture_participant_transcription(participant["id"]) - await intro_task.queue_frames( - [ - images["book1"], - LLMMessagesFrame([LLM_INTRO_PROMPT]), - DailyTransportMessageFrame(CUE_USER_TURN), - sounds["listening"], - images["book2"], - StopTaskFrame(), - ] - ) - - # We run the intro pipeline. This will start the transport. The intro - # task will exit after StopTaskFrame is processed. - await runner.run(intro_task) - - # The main story pipeline is used to continue the story based on user - # input. main_pipeline = Pipeline( [ transport.input(), @@ -139,11 +119,32 @@ async def main(room_url, token=None): ] ) - main_task = PipelineTask(main_pipeline) + main_task = PipelineTask( + main_pipeline, + PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + logger.debug("Participant joined, storytime commence!") + await transport.capture_participant_transcription(participant["id"]) + await main_task.queue_frames( + [ + images["book1"], + context_aggregator.user().get_context_frame(), + DailyTransportMessageFrame(CUE_USER_TURN), + # sounds["listening"], + images["book2"], + ] + ) @transport.event_handler("on_participant_left") async def on_participant_left(transport, participant, reason): - await intro_task.queue_frame(EndFrame()) await main_task.queue_frame(EndFrame()) @transport.event_handler("on_call_state_updated") diff --git a/examples/storytelling-chatbot/src/bot_runner.py b/examples/storytelling-chatbot/src/bot_runner.py index 97a60b72e..2d949c768 100644 --- a/examples/storytelling-chatbot/src/bot_runner.py +++ b/examples/storytelling-chatbot/src/bot_runner.py @@ -114,7 +114,7 @@ async def start_bot(request: Request) -> JSONResponse: else: try: subprocess.Popen( - [f"python3 -m bot -u {room.url} -t {token}"], + [f"python -m bot -u {room.url} -t {token}"], shell=True, bufsize=1, cwd=os.path.dirname(os.path.abspath(__file__)), @@ -175,7 +175,7 @@ async def virtualize_bot(room_url: str, token: str): image = data[0]["config"]["image"] # Machine configuration - cmd = f"python3 src/bot.py -u {room_url} -t {token}" + cmd = f"python src/bot.py -u {room_url} -t {token}" cmd = cmd.split() worker_props = { "config": { @@ -215,7 +215,7 @@ async def virtualize_bot(room_url: str, token: str): if __name__ == "__main__": # Check environment variables required_env_vars = [ - "OPENAI_API_KEY", + "GOOGLE_API_KEY", "DAILY_API_KEY", "FAL_KEY", "ELEVENLABS_VOICE_ID", diff --git a/examples/storytelling-chatbot/src/processors.py b/examples/storytelling-chatbot/src/processors.py index 096efd577..6d30b543c 100644 --- a/examples/storytelling-chatbot/src/processors.py +++ b/examples/storytelling-chatbot/src/processors.py @@ -37,8 +37,7 @@ class StoryPromptFrame(TextFrame): class StoryImageProcessor(FrameProcessor): - """ - Processor for image prompt frames that will be sent to the FAL service. + """Processor for image prompt frames that will be sent to the FAL service. This processor is responsible for consuming frames of type `StoryImageFrame`. It processes them by passing it to the FAL service. @@ -68,8 +67,7 @@ class StoryImageProcessor(FrameProcessor): class StoryProcessor(FrameProcessor): - """ - Primary frame processor. It takes the frames generated by the LLM + """Primary frame processor. It takes the frames generated by the LLM and processes them into image prompts and story pages (sentences). For a clearer picture of how this works, reference prompts.py @@ -97,44 +95,10 @@ class StoryProcessor(FrameProcessor): await self.push_frame(sounds["talking"]) elif isinstance(frame, TextFrame): - # We want to look for sentence breaks in the text - # but since TextFrames are streamed from the LLM - # we need to keep a buffer of the text we've seen so far + # Add new text to the buffer self._text += frame.text - - # IMAGE PROMPT - # Looking for: < [image prompt] > in the LLM response - # We prompted our LLM to add an image prompt in the response - # so we use regex matching to find it and yield a StoryImageFrame - if re.search(r"<.*?>", self._text): - if not re.search(r"<.*?>.*?>", self._text): - # Pass any frames until we have a closing bracket - # otherwise the image prompt will be passed to TTS - pass - # Extract the image prompt from the text using regex - image_prompt = re.search(r"<(.*?)>", self._text).group(1) - # Remove the image prompt from the text - self._text = re.sub(r"<.*?>", "", self._text, count=1) - # Process the image prompt frame - await self.push_frame(StoryImageFrame(image_prompt)) - - # STORY PAGE - # Looking for: [break] in the LLM response - # We prompted our LLM to add a [break] after each sentence - # so we use regex matching to find it in the LLM response - if re.search(r".*\[[bB]reak\].*", self._text): - # Remove the [break] token from the text - # so it isn't spoken out loud by the TTS - self._text = re.sub(r"\[[bB]reak\]", "", self._text, flags=re.IGNORECASE) - self._text = self._text.replace("\n", " ") - if len(self._text) > 2: - # Append the sentence to the story - self._story.append(self._text) - await self.push_frame(StoryPageFrame(self._text)) - # Assert that it's the LLMs turn, until we're finished - await self.push_frame(DailyTransportMessageFrame(CUE_ASSISTANT_TURN)) - # Clear the buffer - self._text = "" + # Process any complete patterns in the order they appear + await self.process_text_content() # End of a full LLM response # Driven by the prompt, the LLM should have asked the user for input @@ -150,3 +114,38 @@ class StoryProcessor(FrameProcessor): # Anything that is not a TextFrame pass through else: await self.push_frame(frame) + + async def process_text_content(self): + """Process text content in order of appearance, handling both image prompts and story breaks.""" + while True: + # Find the first occurrence of each pattern + image_match = re.search(r"<(.*?)>", self._text) + break_match = re.search(r"\[[bB]reak\]", self._text) + + # If neither pattern is found, we're done processing + if not image_match and not break_match: + break + + # Find which pattern comes first in the text + image_pos = image_match.start() if image_match else float("inf") + break_pos = break_match.start() if break_match else float("inf") + + if image_pos < break_pos: + # Process image prompt first + image_prompt = image_match.group(1) + # Remove the image prompt from the text + self._text = self._text[: image_match.start()] + self._text[image_match.end() :] + await self.push_frame(StoryImageFrame(image_prompt)) + else: + # Process story break first + parts = re.split(r"\[[bB]reak\]", self._text, flags=re.IGNORECASE, maxsplit=1) + before_break = parts[0].replace("\n", " ").strip() + + if len(before_break) > 2: + self._story.append(before_break) + await self.push_frame(StoryPageFrame(before_break)) + # await self.push_frame(sounds["ding"]) + await self.push_frame(DailyTransportMessageFrame(CUE_ASSISTANT_TURN)) + + # Keep the remainder (if any) in the buffer + self._text = parts[1].strip() if len(parts) > 1 else "" diff --git a/examples/storytelling-chatbot/src/prompts.py b/examples/storytelling-chatbot/src/prompts.py index 08abbc93c..186828932 100644 --- a/examples/storytelling-chatbot/src/prompts.py +++ b/examples/storytelling-chatbot/src/prompts.py @@ -1,31 +1,34 @@ -LLM_INTRO_PROMPT = { - "role": "system", - "content": "You are a creative storyteller who loves to tell whimsical, fantastical stories. \ - Your goal is to craft an engaging and fun story. \ - Start by asking the user what kind of story they'd like to hear. Don't provide any examples. \ - Keep your response to only a few sentences.", -} - - LLM_BASE_PROMPT = { "role": "system", - "content": "You are a creative storyteller who loves tell whimsical, fantastical stories. \ - Your goal is to craft an engaging and fun story. \ - Keep all responses short and no more than a few sentences. Include [break] after each sentence of the story. \ - \ - Start each sentence with an image prompt, wrapped in triangle braces, that I can use to generate an illustration representing the upcoming scene. \ - Image prompts should always be wrapped in triangle braces, like this: . \ - You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence. \ - For any recurring characters, you should provide a description of them in the image prompt each time, for example: . \ - Please do not include any character names in the image prompts, just their descriptions. \ - Image prompts should focus on key visual attributes of all characters each time, for example . \ - Please use the following structure for your image prompts: characters, setting, action, and mood. \ - Image prompts should be less than 150-200 characters and start in lowercase. \ - \ - Responses should use the format: <...> story sentence [break] <...> story sentence [break] ... \ - After each response, ask me how I'd like the story to continue and wait for my input. \ - Please ensure your responses are less than 3-4 sentences long. \ - Please refrain from using any explicit language or content. Do not tell scary stories.", + "content": """You are a creative storyteller who loves tell whimsical, fantastical stories. + Your goal is to craft an engaging and fun story. + Keep all responses short and no more than a few sentences. + Start by asking the user what kind of story they'd like to hear. Don't provide any examples. + After they've answered the question, start telling the story. Include [break] after each sentence of the story. + + Start each sentence with an image prompt, wrapped in triangle braces, that I can use to generate an illustration representing the upcoming scene. + Image prompts should always be wrapped in triangle braces, like this: . + You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence. + For any recurring characters, you should provide a description of them in the image prompt each time, for example: . + Please do not include any character names in the image prompts, just their descriptions. + Image prompts should focus on key visual attributes of all characters each time, for example . + Please use the following structure for your image prompts: characters, setting, action, and mood. + Image prompts should be less than 150-200 characters and start in lowercase. + + STORY SENTENCE OUTPUT FORMAT: + + story sentence 1 [break] + + story sentence 2 [break] + + story sentence 3 [break] + How would you like the story to continue? + END OF EXAMPLE OUTPUT + + Generate three story sentences, then ask what should happen next and wait for my input. You can propose an idea for how the story should proceed, but make sure to tell me I can suggest whatever I want. \ + Please ensure your responses are less than 5 sentences long. \ + Please refrain from using any explicit language or content. Do not tell scary stories. + Once you've started telling the story, EVERY RESPONSE should follow the story sentence output format. It is VERY IMPORTANT that you continue to include and [break] between story sentences. DO NOT RESPOND without image descriptions and break tags.""", }