wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Storytelling chatbot updates (#1066)

Browse Source 
* initial changes for gemini storybot

* storybot updates for gemini

* more storybot updates

* interim interruptible commit

* cleanup

* cleanup

* cleanup

* cleanup
This commit is contained in:
chadbailey59
2025-01-22 15:20:21 -06:00
committed by GitHub
parent a64df978e7
commit 067ddfe505
10 changed files with 139 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/storytelling-chatbot/README.md
View File

@@ -66,7 +66,7 @@ The build UI files can be found in `frontend/out`
Start the API / bot manager:
`python src/bot_runner.py`
`python src/bot_runner.py --host localhost`
If you'd like to run a custom domain or port:

1
examples/storytelling-chatbot/env.example
View File

@@ -4,6 +4,7 @@ ELEVENLABS_API_KEY=
ELEVENLABS_VOICE_ID=
FAL_KEY=
OPENAI_API_KEY=
GOOGLE_API_KEY=
ENV= # dev | production
RUN_AS_VM= # Set this if you want to run bots on process (not launch a new VM)

10
examples/storytelling-chatbot/frontend/components/Story.tsx
View File

@@ -1,4 +1,4 @@
import React, { useState } from "react";
import React, { useState, useEffect } from "react";
import {
useDaily,
useParticipantIds,
@@ -33,7 +33,9 @@ const Story: React.FC<StoryProps> = ({ handleLeave }) => {
setTimeout(() => daily.setLocalAudio(true), 500);
setStoryState("user");
} else {
daily.setLocalAudio(false);
// Uncomment the next line to mute the mic while the
// assistant it talking. Leave it commented to allow for interruptions
// daily.setLocalAudio(false);
setStoryState("assistant");
}
},
@@ -58,7 +60,7 @@ const Story: React.FC<StoryProps> = ({ handleLeave }) => {
{participantIds.length >= 1 ? (
<VideoTile
sessionId={participantIds[0]}
inactive={storyState === "user"}
inactive={false}
/>
) : (
<span className="p-3 rounded-full bg-gray-900/60 animate-pulse">
@@ -71,7 +73,7 @@ const Story: React.FC<StoryProps> = ({ handleLeave }) => {
)}
<DailyAudio />
</div>
<UserInputIndicator active={storyState === "user"} />
<UserInputIndicator active={true} />
</div>
);
};

17
examples/storytelling-chatbot/frontend/components/UserInputIndicator/UserInputIndicator.module.css
View File

@@ -43,25 +43,8 @@
transition: opacity 0.5s ease;
}
@keyframes pulse {
0% {
outline-width: 6px;
@apply outline-teal-500/10;
}
50% {
outline-width: 24px;
@apply outline-teal-500/50;
}
100% {
outline-width: 6px;
@apply outline-teal-500/10;
}
}
.micIconActive{
@apply bg-teal-950 border-teal-500 outline-teal-500/20;
animation: pulse 2s infinite ease-in-out;
}
.micIconActive svg{

21
examples/storytelling-chatbot/frontend/components/UserInputIndicator/index.tsx
View File

@@ -1,4 +1,4 @@
import React, { useState, useEffect } from "react";
import React, { useState, useEffect, useRef } from "react";
import { useAppMessage } from "@daily-co/daily-react";
import { DailyEventObjectAppMessage } from "@daily-co/daily-js";
@@ -13,12 +13,31 @@ interface Props {
export default function UserInputIndicator({ active }: Props) {
const [transcription, setTranscription] = useState<string[]>([]);
const timeoutRef = useRef<NodeJS.Timeout>();
const resetTimeout = () => {
if (timeoutRef.current) {
clearTimeout(timeoutRef.current);
}
timeoutRef.current = setTimeout(() => {
setTranscription([]);
}, 5000);
};
useEffect(() => {
return () => {
if (timeoutRef.current) {
clearTimeout(timeoutRef.current);
}
};
}, []);
useAppMessage({
onAppMessage: (e: DailyEventObjectAppMessage<any>) => {
if (e.fromId && e.fromId === "transcription") {
if (e.data.user_id === "" && e.data.is_final) {
setTranscription((t) => [...t, ...e.data.text.split(" ")]);
resetTimeout();
}
}
},

2
examples/storytelling-chatbot/requirements.txt
View File

@@ -2,4 +2,4 @@ async_timeout
fastapi
uvicorn
python-dotenv
pipecat-ai[daily,elevenlabs,openai,fal]
pipecat-ai[daily,openai,fal,google,cartesia]

75
examples/storytelling-chatbot/src/bot.py
View File

@@ -13,16 +13,23 @@ import aiohttp
from dotenv import load_dotenv
from loguru import logger
from processors import StoryImageProcessor, StoryProcessor
from prompts import CUE_USER_TURN, LLM_BASE_PROMPT, LLM_INTRO_PROMPT
from prompts import CUE_USER_TURN, LLM_BASE_PROMPT
from utils.helpers import load_images, load_sounds
from pipecat.frames.frames import EndFrame, LLMMessagesFrame, StopTaskFrame
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import EndFrame, StopTaskFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
OpenAILLMContextFrame,
)
from pipecat.processors.logger import FrameLogger
from pipecat.services.cartesia import CartesiaHttpTTSService, CartesiaTTSService
from pipecat.services.elevenlabs import ElevenLabsTTSService
from pipecat.services.fal import FalImageGenService
from pipecat.services.google import GoogleLLMService
from pipecat.services.openai import OpenAILLMService
from pipecat.transports.services.daily import (
DailyParams,
@@ -53,6 +60,7 @@ async def main(room_url, token=None):
camera_out_width=768,
camera_out_height=768,
transcription_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
vad_enabled=True,
),
)
@@ -61,11 +69,10 @@ async def main(room_url, token=None):
# -------------- Services --------------- #
llm_service = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
llm_service = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"))
tts_service = ElevenLabsTTSService(
api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID")
)
fal_service_params = FalImageGenService.InputParams(
@@ -74,7 +81,7 @@ async def main(room_url, token=None):
fal_service = FalImageGenService(
aiohttp_session=session,
model="fal-ai/fast-lightning-sdxl",
model="fal-ai/stable-diffusion-v35-medium",
params=fal_service_params,
key=os.getenv("FAL_KEY"),
)
@@ -97,35 +104,8 @@ async def main(room_url, token=None):
runner = PipelineRunner()
# The intro pipeline is used to start
# the story (as per LLM_INTRO_PROMPT)
intro_pipeline = Pipeline([llm_service, tts_service, transport.output()])
intro_task = PipelineTask(intro_pipeline)
logger.debug("Waiting for participant...")
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
logger.debug("Participant joined, storytime commence!")
await transport.capture_participant_transcription(participant["id"])
await intro_task.queue_frames(
[
images["book1"],
LLMMessagesFrame([LLM_INTRO_PROMPT]),
DailyTransportMessageFrame(CUE_USER_TURN),
sounds["listening"],
images["book2"],
StopTaskFrame(),
]
)
# We run the intro pipeline. This will start the transport. The intro
# task will exit after StopTaskFrame is processed.
await runner.run(intro_task)
# The main story pipeline is used to continue the story based on user
# input.
main_pipeline = Pipeline(
[
transport.input(),
@@ -139,11 +119,32 @@ async def main(room_url, token=None):
]
)
main_task = PipelineTask(main_pipeline)
main_task = PipelineTask(
main_pipeline,
PipelineParams(
allow_interruptions=True,
enable_metrics=True,
enable_usage_metrics=True,
report_only_initial_ttfb=True,
),
)
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
logger.debug("Participant joined, storytime commence!")
await transport.capture_participant_transcription(participant["id"])
await main_task.queue_frames(
[
images["book1"],
context_aggregator.user().get_context_frame(),
DailyTransportMessageFrame(CUE_USER_TURN),
# sounds["listening"],
images["book2"],
]
)
@transport.event_handler("on_participant_left")
async def on_participant_left(transport, participant, reason):
await intro_task.queue_frame(EndFrame())
await main_task.queue_frame(EndFrame())
@transport.event_handler("on_call_state_updated")

6
examples/storytelling-chatbot/src/bot_runner.py
View File

@@ -114,7 +114,7 @@ async def start_bot(request: Request) -> JSONResponse:
else:
try:
subprocess.Popen(
[f"python3 -m bot -u {room.url} -t {token}"],
[f"python -m bot -u {room.url} -t {token}"],
shell=True,
bufsize=1,
cwd=os.path.dirname(os.path.abspath(__file__)),
@@ -175,7 +175,7 @@ async def virtualize_bot(room_url: str, token: str):
image = data[0]["config"]["image"]
# Machine configuration
cmd = f"python3 src/bot.py -u {room_url} -t {token}"
cmd = f"python src/bot.py -u {room_url} -t {token}"
cmd = cmd.split()
worker_props = {
"config": {
@@ -215,7 +215,7 @@ async def virtualize_bot(room_url: str, token: str):
if __name__ == "__main__":
# Check environment variables
required_env_vars = [
"OPENAI_API_KEY",
"GOOGLE_API_KEY",
"DAILY_API_KEY",
"FAL_KEY",
"ELEVENLABS_VOICE_ID",

81
examples/storytelling-chatbot/src/processors.py
View File

@@ -37,8 +37,7 @@ class StoryPromptFrame(TextFrame):
class StoryImageProcessor(FrameProcessor):
"""
Processor for image prompt frames that will be sent to the FAL service.
"""Processor for image prompt frames that will be sent to the FAL service.
This processor is responsible for consuming frames of type `StoryImageFrame`.
It processes them by passing it to the FAL service.
@@ -68,8 +67,7 @@ class StoryImageProcessor(FrameProcessor):
class StoryProcessor(FrameProcessor):
"""
Primary frame processor. It takes the frames generated by the LLM
"""Primary frame processor. It takes the frames generated by the LLM
and processes them into image prompts and story pages (sentences).
For a clearer picture of how this works, reference prompts.py
@@ -97,44 +95,10 @@ class StoryProcessor(FrameProcessor):
await self.push_frame(sounds["talking"])
elif isinstance(frame, TextFrame):
# We want to look for sentence breaks in the text
# but since TextFrames are streamed from the LLM
# we need to keep a buffer of the text we've seen so far
# Add new text to the buffer
self._text += frame.text
# IMAGE PROMPT
# Looking for: < [image prompt] > in the LLM response
# We prompted our LLM to add an image prompt in the response
# so we use regex matching to find it and yield a StoryImageFrame
if re.search(r"<.*?>", self._text):
if not re.search(r"<.*?>.*?>", self._text):
# Pass any frames until we have a closing bracket
# otherwise the image prompt will be passed to TTS
pass
# Extract the image prompt from the text using regex
image_prompt = re.search(r"<(.*?)>", self._text).group(1)
# Remove the image prompt from the text
self._text = re.sub(r"<.*?>", "", self._text, count=1)
# Process the image prompt frame
await self.push_frame(StoryImageFrame(image_prompt))
# STORY PAGE
# Looking for: [break] in the LLM response
# We prompted our LLM to add a [break] after each sentence
# so we use regex matching to find it in the LLM response
if re.search(r".*\[[bB]reak\].*", self._text):
# Remove the [break] token from the text
# so it isn't spoken out loud by the TTS
self._text = re.sub(r"\[[bB]reak\]", "", self._text, flags=re.IGNORECASE)
self._text = self._text.replace("\n", " ")
if len(self._text) > 2:
# Append the sentence to the story
self._story.append(self._text)
await self.push_frame(StoryPageFrame(self._text))
# Assert that it's the LLMs turn, until we're finished
await self.push_frame(DailyTransportMessageFrame(CUE_ASSISTANT_TURN))
# Clear the buffer
self._text = ""
# Process any complete patterns in the order they appear
await self.process_text_content()
# End of a full LLM response
# Driven by the prompt, the LLM should have asked the user for input
@@ -150,3 +114,38 @@ class StoryProcessor(FrameProcessor):
# Anything that is not a TextFrame pass through
else:
await self.push_frame(frame)
async def process_text_content(self):
"""Process text content in order of appearance, handling both image prompts and story breaks."""
while True:
# Find the first occurrence of each pattern
image_match = re.search(r"<(.*?)>", self._text)
break_match = re.search(r"\[[bB]reak\]", self._text)
# If neither pattern is found, we're done processing
if not image_match and not break_match:
break
# Find which pattern comes first in the text
image_pos = image_match.start() if image_match else float("inf")
break_pos = break_match.start() if break_match else float("inf")
if image_pos < break_pos:
# Process image prompt first
image_prompt = image_match.group(1)
# Remove the image prompt from the text
self._text = self._text[: image_match.start()] + self._text[image_match.end() :]
await self.push_frame(StoryImageFrame(image_prompt))
else:
# Process story break first
parts = re.split(r"\[[bB]reak\]", self._text, flags=re.IGNORECASE, maxsplit=1)
before_break = parts[0].replace("\n", " ").strip()
if len(before_break) > 2:
self._story.append(before_break)
await self.push_frame(StoryPageFrame(before_break))
# await self.push_frame(sounds["ding"])
await self.push_frame(DailyTransportMessageFrame(CUE_ASSISTANT_TURN))
# Keep the remainder (if any) in the buffer
self._text = parts[1].strip() if len(parts) > 1 else ""

55
examples/storytelling-chatbot/src/prompts.py
View File

@@ -1,31 +1,34 @@
LLM_INTRO_PROMPT = {
"role": "system",
"content": "You are a creative storyteller who loves to tell whimsical, fantastical stories. \
Your goal is to craft an engaging and fun story. \
Start by asking the user what kind of story they'd like to hear. Don't provide any examples. \
Keep your response to only a few sentences.",
}
LLM_BASE_PROMPT = {
"role": "system",
"content": "You are a creative storyteller who loves tell whimsical, fantastical stories. \
Your goal is to craft an engaging and fun story. \
Keep all responses short and no more than a few sentences. Include [break] after each sentence of the story. \
\
Start each sentence with an image prompt, wrapped in triangle braces, that I can use to generate an illustration representing the upcoming scene. \
Image prompts should always be wrapped in triangle braces, like this: <image prompt goes here>. \
You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence. \
For any recurring characters, you should provide a description of them in the image prompt each time, for example: <a brown fluffy dog ...>. \
Please do not include any character names in the image prompts, just their descriptions. \
Image prompts should focus on key visual attributes of all characters each time, for example <a brown fluffy dog and the tiny red cat ...>. \
Please use the following structure for your image prompts: characters, setting, action, and mood. \
Image prompts should be less than 150-200 characters and start in lowercase. \
\
Responses should use the format: <...> story sentence [break] <...> story sentence [break] ... \
After each response, ask me how I'd like the story to continue and wait for my input. \
Please ensure your responses are less than 3-4 sentences long. \
Please refrain from using any explicit language or content. Do not tell scary stories.",
"content": """You are a creative storyteller who loves tell whimsical, fantastical stories.
Your goal is to craft an engaging and fun story.
Keep all responses short and no more than a few sentences.
Start by asking the user what kind of story they'd like to hear. Don't provide any examples.
After they've answered the question, start telling the story. Include [break] after each sentence of the story.
Start each sentence with an image prompt, wrapped in triangle braces, that I can use to generate an illustration representing the upcoming scene.
Image prompts should always be wrapped in triangle braces, like this: <image prompt goes here>.
You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence.
For any recurring characters, you should provide a description of them in the image prompt each time, for example: <a brown fluffy dog ...>.
Please do not include any character names in the image prompts, just their descriptions.
Image prompts should focus on key visual attributes of all characters each time, for example <a brown fluffy dog and the tiny red cat ...>.
Please use the following structure for your image prompts: characters, setting, action, and mood.
Image prompts should be less than 150-200 characters and start in lowercase.
STORY SENTENCE OUTPUT FORMAT:
<image description 1>
story sentence 1 [break]
<image description 2>
story sentence 2 [break]
<image description 3>
story sentence 3 [break]
How would you like the story to continue?
END OF EXAMPLE OUTPUT
Generate three story sentences, then ask what should happen next and wait for my input. You can propose an idea for how the story should proceed, but make sure to tell me I can suggest whatever I want. \
Please ensure your responses are less than 5 sentences long. \
Please refrain from using any explicit language or content. Do not tell scary stories.
Once you've started telling the story, EVERY RESPONSE should follow the story sentence output format. It is VERY IMPORTANT that you continue to include <image descriptions> and [break] between story sentences. DO NOT RESPOND without image descriptions and break tags.""",
}