added examples back

This commit is contained in:
Jon Taylor
2024-05-13 17:09:46 +01:00
parent 27ba50cbbf
commit f1c02f8554
154 changed files with 7918 additions and 1287 deletions

View File

@@ -0,0 +1,2 @@
frontend/node_modules
frontend/out

158
examples/storytelling-chatbot/.gitignore vendored Normal file
View File

@@ -0,0 +1,158 @@
node_modules/
.idea/
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
.DS_Store
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
read.html

View File

@@ -0,0 +1,54 @@
FROM python:3.11-bullseye
ARG DEBIAN_FRONTEND=noninteractive
ARG USE_PERSISTENT_DATA
ENV PYTHONUNBUFFERED=1
ENV NODE_MAJOR=20
# Expose FastAPI port
ENV FAST_API_PORT=7860
EXPOSE 7860
# Install system dependencies
RUN apt-get update && apt-get install --no-install-recommends -y \
build-essential \
git \
ffmpeg \
google-perftools \
ca-certificates curl gnupg \
&& apt-get clean && rm -rf /var/lib/apt/lists/*
# Install Node.js
RUN mkdir -p /etc/apt/keyrings
RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list > /dev/null
RUN apt-get update && apt-get install nodejs -y
# Set up a new user named "user" with user ID 1000
RUN useradd -m -u 1000 user
# Set home to the user's home directory
ENV HOME=/home/user \
PATH=/home/user/.local/bin:$PATH \
PYTHONPATH=$HOME/app \
PYTHONUNBUFFERED=1
# Switch to the "user" user
USER user
# Set the working directory to the user's home directory
WORKDIR $HOME/app
# Install Python dependencies
COPY ./requirements.txt requirements.txt
RUN pip3 install --no-cache-dir --upgrade -r requirements.txt
# Copy everything else
COPY --chown=user ./src/ src/
# Copy frontend app and build
COPY --chown=user ./frontend/ frontend/
RUN cd frontend && npm install && npm run build
# Start the FastAPI server
CMD python3 src/server.py --port ${FAST_API_PORT}

View File

@@ -0,0 +1,83 @@
[![Try](https://img.shields.io/badge/try_it-here-blue)](https://storytelling-chatbot.fly.dev)
# Storytelling Chatbot
<img src="image.png" width="420px">
This example shows how to build a voice-driven interactive storytelling experience.
It periodically prompts the user for input for a 'choose your own adventure' style experience.
We add visual elements to the story by generating images at lightning speed using Fal.
---
### It uses the following AI services:
**Deepgram - Speech-to-Text**
Transcribes inbound participant voice media to text.
**OpenAI (GPT4) - LLM**
Our creative writer LLM. You can see the context used to prompt it [here](src/prompts.py)
**ElevenLabs - Text-to-Speech**
Converts and streams the LLM response from text to audio
**Fal.ai - Image Generation**
Adds pictures to our story (really fast!) Prompting is quite key for style consistency, so we task the LLM to turn each story page into a short image prompt.
---
## Setup
**Install requirements**
```shell
pip install -r requirements.txt
```
**Create environment file and set variables:**
```shell
mv env.example .env
```
**Build the frontend:**
This project uses a custom frontend, which needs to built. Note: this is done automatically as part of the Docker deployment.
```shell
cd frontend/
npm install / yarn
npm run build
```
The build UI files can be found in `frontend/out`
## Running it locally
Start the API / bot manager:
`python src/server.py`
If you'd like to run a custom domain or port:
`python src/server.py --host somehost --p 7777`
➡️ Open the host URL in your browser
> [!IMPORTANT]
> Whilst working on the frontend code, please `yarn run dev`
> and open the NextJS hosted service vs. the Python server.
> (Usually localhost:3000.)
---
## Improvements to make
- Wait for track_started event to avoid rushed intro
- Show 5 minute timer on the UI

View File

@@ -0,0 +1,6 @@
ELEVENLABS_API_KEY=
ELEVENLABS_VOICE_ID=
FAL_KEY=
DAILY_API_URL=api.daily.co/v1
DAILY_API_KEY=
OPENAI_API_KEY=

View File

@@ -0,0 +1,3 @@
{
"extends": "next/core-web-vitals"
}

View File

@@ -0,0 +1,36 @@
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
# dependencies
/node_modules
/.pnp
.pnp.js
.yarn/install-state.gz
# testing
/coverage
# next.js
/.next/
/out/
# production
/build
# misc
.DS_Store
*.pem
# debug
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# local env files
.env*.local
# vercel
.vercel
# typescript
*.tsbuildinfo
next-env.d.ts

View File

@@ -0,0 +1,36 @@
This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
## Getting Started
First, run the development server:
```bash
npm run dev
# or
yarn dev
# or
pnpm dev
# or
bun dev
```
Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
## Learn More
To learn more about Next.js, take a look at the following resources:
- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
## Deploy on Vercel
The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 KiB

View File

@@ -0,0 +1,109 @@
@tailwind base;
@tailwind components;
@tailwind utilities;
@layer base {
:root {
--background: 0 0% 100%;
--foreground: 224 71.4% 4.1%;
--card: 0 0% 100%;
--card-foreground: 224 71.4% 4.1%;
--popover: 0 0% 100%;
--popover-foreground: 224 71.4% 4.1%;
--primary: 220.9 39.3% 11%;
--primary-foreground: 210 20% 98%;
--secondary: 220 14.3% 95.9%;
--secondary-foreground: 220.9 39.3% 11%;
--muted: 220 14.3% 95.9%;
--muted-foreground: 220 8.9% 46.1%;
--accent: 220 14.3% 95.9%;
--accent-foreground: 220.9 39.3% 11%;
--destructive: 0 84.2% 60.2%;
--destructive-foreground: 210 20% 98%;
--border: 220 13% 91%;
--input: 220 13% 91%;
--ring: 224 71.4% 4.1%;
--radius: 0.5rem;
}
.dark {
--background: 224 71.4% 4.1%;
--foreground: 210 20% 98%;
--card: 224 71.4% 4.1%;
--card-foreground: 210 20% 98%;
--popover: 224 71.4% 4.1%;
--popover-foreground: 210 20% 98%;
--primary: 210 20% 98%;
--primary-foreground: 220.9 39.3% 11%;
--secondary: 215 27.9% 16.9%;
--secondary-foreground: 210 20% 98%;
--muted: 215 27.9% 16.9%;
--muted-foreground: 217.9 10.6% 64.9%;
--accent: 215 27.9% 16.9%;
--accent-foreground: 210 20% 98%;
--destructive: 0 62.8% 30.6%;
--destructive-foreground: 210 20% 98%;
--border: 215 27.9% 16.9%;
--input: 215 27.9% 16.9%;
--ring: 216 12.2% 83.9%;
}
}
@layer base {
* {
@apply border-border;
}
body {
@apply bg-background text-foreground;
}
}
body{
background: url("/bg.jpg") no-repeat center center;
background-size: cover;
}
.cardShadow{
box-shadow: 0px 124px 35px 0px rgba(0, 0, 0, 0.00), 0px 79px 32px 0px rgba(0, 0, 0, 0.01), 0px 45px 27px 0px rgba(0, 0, 0, 0.05), 0px 20px 20px 0px rgba(0, 0, 0, 0.09), 0px 5px 11px 0px rgba(0, 0, 0, 0.10);
}
@keyframes fadeInSlideUp {
0% {
opacity: 0;
transform: translateY(50px);
}
100% {
opacity: 1;
transform: translateY(0);
}
}
.fade-in {
animation: fadeIn 1s ease-out;
}
@keyframes fadeIn {
0% {
opacity: 0;
}
100% {
opacity: 1;
}
}

View File

@@ -0,0 +1,46 @@
import React from "react";
import "./globals.css";
import type { Metadata } from "next";
import { Space_Grotesk, Space_Mono } from "next/font/google";
import { cn } from "@/app/utils";
// Font
const sans = Space_Grotesk({
subsets: ["latin"],
weight: ["400", "500", "600"],
variable: "--font-sans",
});
const mono = Space_Mono({
subsets: ["latin"],
weight: ["400", "700"],
variable: "--font-mono",
});
export const metadata: Metadata = {
title: "Storytelling Chatbot - Daily AI",
description: "Built with git.new/ai",
metadataBase: new URL(process.env.SITE_URL || "http://localhost:3000"),
};
export default function RootLayout({
children,
}: Readonly<{
children: React.ReactNode;
}>) {
return (
<html lang="en">
<body
className={cn(
"min-h-screen bg-background font-sans antialiased flex flex-col",
sans.variable,
mono.variable
)}
>
<main className="flex flex-1">{children}</main>
</body>
</html>
);
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 MiB

View File

@@ -0,0 +1,16 @@
"use client";
import React from "react";
import { DailyProvider, useCallObject } from "@daily-co/daily-react";
import App from "../components/App";
export default function Home() {
const callObject = useCallObject({});
return (
<DailyProvider callObject={callObject}>
<App />
</DailyProvider>
);
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.4 MiB

View File

@@ -0,0 +1,6 @@
import { type ClassValue, clsx } from "clsx"
import { twMerge } from "tailwind-merge"
export function cn(...inputs: ClassValue[]) {
return twMerge(clsx(inputs))
}

View File

@@ -0,0 +1,17 @@
{
"$schema": "https://ui.shadcn.com/schema.json",
"style": "default",
"rsc": true,
"tsx": true,
"tailwind": {
"config": "tailwind.config.ts",
"css": "app/globals.css",
"baseColor": "gray",
"cssVariables": true,
"prefix": ""
},
"aliases": {
"components": "@/components",
"utils": "@/app"
}
}

View File

@@ -0,0 +1,90 @@
"use client";
import React, { useState } from "react";
import { useDaily } from "@daily-co/daily-react";
import Setup from "./Setup";
import Story from "./Story";
type State =
| "idle"
| "connecting"
| "connected"
| "started"
| "finished"
| "error";
export default function Call() {
const daily = useDaily();
const [state, setState] = useState<State>("idle");
const [room, setRoom] = useState<string | null>(null);
async function start() {
setState("connecting");
if (!daily) return;
// Create a new room for the story session
try {
const response = await fetch("/create", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
room_url: process.env.NEXT_PUBLIC_ROOM_URL || null,
}),
});
const { room_url, token } = await response.json();
// Keep a reference to the room url for later
setRoom(room_url);
// Join the WebRTC session
await daily.join({
url: room_url,
token,
videoSource: false,
startAudioOff: true,
});
setState("connected");
// Disable local audio, the bot will say hello first
daily.setLocalAudio(false);
// Start the bot
const resp = await fetch("/start", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
room_url,
}),
});
setState("started");
} catch (error) {
setState("error");
leave();
}
}
async function leave() {
await daily?.leave();
setState("finished");
}
if (state === "error") {
return <div>An Error occured</div>;
}
if (state === "started") {
return <Story handleLeave={() => leave()} />;
}
return <Setup handleStart={() => start()} />;
}

View File

@@ -0,0 +1,69 @@
import {
useAudioLevel,
useAudioTrack,
useLocalSessionId,
} from "@daily-co/daily-react";
import { useCallback, useRef } from "react";
export const AudioIndicator: React.FC = () => {
const localSessionId = useLocalSessionId();
const audioTrack = useAudioTrack(localSessionId);
const volRef = useRef<HTMLDivElement>(null);
useAudioLevel(
audioTrack?.persistentTrack,
useCallback((volume) => {
// this volume number will be between 0 and 1
// give it a minimum scale of 0.15 to not completely disappear 👻
if (volRef.current) {
const v = volume * 1.75;
volRef.current.style.transform = `scale(${Math.max(0.1, v)})`;
}
}, [])
);
// Your audio track's audio volume visualized in a small circle,
// whose size changes depending on the volume level
return (
<>
<div className="vol bg-teal-700" ref={volRef} />
<style jsx>{`
.vol {
position: absolute;
overflow: hidden;
inset: 0px;
z-index: 0;
border-radius: 999px;
transition: all 0.1s ease;
transform: scale(0);
}
`}</style>
</>
);
};
export const AudioIndicatorBar: React.FC = () => {
const localSessionId = useLocalSessionId();
const audioTrack = useAudioTrack(localSessionId);
const volRef = useRef<HTMLDivElement>(null);
useAudioLevel(
audioTrack?.persistentTrack,
useCallback((volume) => {
if (volRef.current)
volRef.current.style.width = Math.max(2, volume * 100) + "%";
}, [])
);
return (
<div className="flex-1 bg-gray-200 h-[8px] rounded-full overflow-hidden">
<div
className="bg-green-500 h-[8px] w-[0] rounded-full transition-all duration-100 ease"
ref={volRef}
/>
</div>
);
};
export default AudioIndicator;

View File

@@ -0,0 +1,139 @@
"use client";
import { useEffect } from "react";
import { DailyMeetingState } from "@daily-co/daily-js";
import { useDaily, useDevices } from "@daily-co/daily-react";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { IconMicrophone, IconDeviceSpeaker } from "@tabler/icons-react";
import { AudioIndicatorBar } from "../AudioIndicator";
interface Props {}
export default function DevicePicker({}: Props) {
const daily = useDaily();
const {
currentMic,
hasMicError,
micState,
microphones,
setMicrophone,
currentSpeaker,
speakers,
setSpeaker,
} = useDevices();
const handleMicrophoneChange = (value: string) => {
setMicrophone(value);
};
const handleSpeakerChange = (value: string) => {
setSpeaker(value);
};
useEffect(() => {
if (microphones.length > 0 || !daily || daily.isDestroyed()) return;
const meetingState = daily.meetingState();
const meetingStatesBeforeJoin: DailyMeetingState[] = [
"new",
"loading",
"loaded",
];
if (meetingStatesBeforeJoin.includes(meetingState)) {
daily.startCamera({ startVideoOff: true, startAudioOff: false });
}
}, [daily, microphones]);
return (
<div className="flex flex-col gap-5">
<section>
<label className="uppercase text-sm tracking-wider text-gray-500">
Microphone:
</label>
<div className="flex flex-row gap-4 items-center mt-2">
<IconMicrophone size={24} />
<div className="flex flex-col flex-1 gap-3">
<Select onValueChange={handleMicrophoneChange}>
<SelectTrigger className="">
<SelectValue
placeholder={
hasMicError ? "error" : currentMic?.device?.label
}
/>
</SelectTrigger>
<SelectContent>
{hasMicError && (
<option value="error" disabled>
No microphone access.
</option>
)}
{microphones.map((m) => (
<SelectItem key={m.device.deviceId} value={m.device.deviceId}>
{m.device.label}
</SelectItem>
))}
</SelectContent>
</Select>
<AudioIndicatorBar />
</div>
</div>
</section>
<section>
<label className="uppercase text-sm tracking-wider text-gray-500">
Speakers:
</label>
<div className="flex flex-row gap-4 items-center mt-2">
<IconDeviceSpeaker size={24} />
<Select onValueChange={handleSpeakerChange}>
<SelectTrigger className="">
<SelectValue placeholder={currentSpeaker?.device?.label} />
</SelectTrigger>
<SelectContent>
{speakers.map((m) => (
<SelectItem key={m.device.deviceId} value={m.device.deviceId}>
{m.device.label}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
</section>
{hasMicError && (
<div className="error">
{micState === "blocked" ? (
<p>
Please check your browser and system permissions. Make sure that
this app is allowed to access your microphone.
</p>
) : micState === "in-use" ? (
<p>
Your microphone is being used by another app. Please close any
other apps using your microphone and restart this app.
</p>
) : micState === "not-found" ? (
<p>
No microphone seems to be connected. Please connect a microphone.
</p>
) : micState === "not-supported" ? (
<p>
This app is not supported on your device. Please update your
software or use a different device.
</p>
) : (
<p>
There seems to be an issue accessing your microphone. Try
restarting the app or consult a system administrator.
</p>
)}
</div>
)}
</div>
);
}

View File

@@ -0,0 +1,34 @@
import {
useDaily,
useLocalSessionId,
useMediaTrack,
} from "@daily-co/daily-react";
import { useCallback } from "react";
import { IconMicrophone, IconMicrophoneOff } from "@tabler/icons-react";
export const MicToggle: React.FC = () => {
const daily = useDaily();
const localSessionId = useLocalSessionId();
const audioTrack = useMediaTrack(localSessionId, "audio");
const isMicMuted =
audioTrack.state === "blocked" || audioTrack.state === "off";
const handleClick = useCallback(() => {
if (!daily) return;
daily.setLocalAudio(isMicMuted);
}, [daily, isMicMuted]);
const text = isMicMuted ? (
<IconMicrophone size={21} />
) : (
<IconMicrophoneOff size={21} />
);
return (
<button className="MicToggle UIButton" onClick={handleClick}>
{text}
</button>
);
};
export default MicToggle;

View File

@@ -0,0 +1,93 @@
import React from "react";
import { Button } from "@/components/ui/button";
import DevicePicker from "@/components/DevicePicker";
import { IconEar, IconLoader2 } from "@tabler/icons-react";
type SetupProps = {
handleStart: () => void;
};
const buttonLabel = {
intro: "Next",
setup: "Let's begin!",
loading: "Joining...",
};
export const Setup: React.FC<SetupProps> = ({ handleStart }) => {
const [state, setState] = React.useState<"intro" | "setup" | "loading">(
"intro"
);
return (
<div className="w-full flex flex-col items-center justify-between">
<div className="bg-white rounded-3xl cardAnim cardShadow p-9 max-w-screen-sm mx-auto outline outline-[5px] outline-gray-600/10 my-auto">
<div className="flex flex-col gap-6">
<h1 className="text-4xl font-bold text-pretty tracking-tighter mb-4">
Welcome to <span className="text-sky-500">Storytime</span>
</h1>
{state === "intro" ? (
<>
<p className="text-gray-600 leading-relaxed text-pretty">
This app demos a voice-controlled storytelling chatbot. It will
start with the bot asking you what kind of story you&apos;d like
to hear (e.g. a fairy tale, a mystery, etc.). After each scene,
the bot will pause to ask for your input. Direct the story any
way you choose!
</p>
<p className="flex flex-row gap-2 text-gray-600 font-medium">
<IconEar size={24} /> For best results, try in a quiet
environment!
</p>
</>
) : (
<>
<p className="text-gray-600 leading-relaxed text-pretty">
Since you&apos;ll be talking to Storybot, we need to make sure
it can hear you! Please configure your microphone and speakers
below.
</p>
<DevicePicker />
</>
)}
<hr className="border-gray-150 my-2" />
<Button
size="lg"
disabled={state === "loading"}
onClick={() => {
if (state === "intro") {
setState("setup");
} else {
setState("loading");
handleStart();
}
}}
>
{state === "loading" && (
<IconLoader2
size={21}
stroke={2}
className="mr-2 h-4 w-4 animate-spin"
/>
)}
{buttonLabel[state]}
</Button>
</div>
</div>
<footer className="flex-0 text-center font-mono text-sm text-gray-100 py-6">
<span className="bg-gray-800/70 px-3 py-1 rounded-md">
Created with{" "}
<a
href="https://git.new/ai"
className="text-violet-300 underline decoration-violet-400 hover:text-violet-100"
>
git.new/ai
</a>
</span>
</footer>
</div>
);
};
export default Setup;

View File

@@ -0,0 +1,79 @@
import React, { useState } from "react";
import {
useDaily,
useParticipantIds,
useAppMessage,
DailyAudio,
} from "@daily-co/daily-react";
import { IconLogout, IconLoader2 } from "@tabler/icons-react";
import VideoTile from "@/components/VideoTile";
import { Button } from "@/components/ui/button";
import UserInputIndicator from "@/components/UserInputIndicator";
import WaveText from "@/components/WaveText";
interface StoryProps {
handleLeave: () => void;
}
const Story: React.FC<StoryProps> = ({ handleLeave }) => {
const daily = useDaily();
const participantIds = useParticipantIds({ filter: "remote" });
const [storyState, setStoryState] = useState<"user" | "assistant">(
"assistant"
);
useAppMessage({
onAppMessage: (e) => {
if (!daily || !e.data?.cue) return;
// Determine the UI state from the cue sent by the bot
if (e.data?.cue === "user_turn") {
// Delay enabling local mic input to avoid feedback from LLM
setTimeout(() => daily.setLocalAudio(true), 500);
setStoryState("user");
} else {
daily.setLocalAudio(false);
setStoryState("assistant");
}
},
});
return (
<div className="w-full flex flex-col flex-1 self-stretch">
{/* Absolute elements */}
<div className="absolute top-20 w-full text-center z-50">
<WaveText active={storyState === "user"} />
</div>
<header className="flex absolute top-0 w-full z-50 p-6 justify-end">
<Button variant="secondary" onClick={() => handleLeave()}>
<IconLogout size={21} className="mr-2" />
Exit
</Button>
</header>
<div className="absolute inset-0 bg-gray-800 bg-opacity-90 z-10 fade-in"></div>
{/* Static elements */}
<div className="relative z-20 flex-1 flex items-center justify-center">
{participantIds.length >= 1 ? (
<VideoTile
sessionId={participantIds[0]}
inactive={storyState === "user"}
/>
) : (
<span className="p-3 rounded-full bg-gray-900/60 animate-pulse">
<IconLoader2
size={42}
stroke={2}
className="animate-spin text-white z-20 self-center"
/>
</span>
)}
<DailyAudio />
</div>
<UserInputIndicator active={storyState === "user"} />
</div>
);
};
export default Story;

View File

@@ -0,0 +1,42 @@
.container{
position: absolute;
color:white;
z-index: 50;
margin: 0 auto;
display: flex;
flex-direction: column;
@apply gap-4 inset-6;
align-items: center;
justify-content: end;
text-align: center;
}
.transcript{
@apply font-semibold;
}
.transcript span{
box-decoration-break: clone;
@apply bg-gray-900/80 rounded-md px-4 py-2;
}
.sentence{
opacity: 1;
animation: fadeOut 2.5s linear forwards;
animation-delay: 1s;
}
@keyframes fadeOut {
0% {
opacity: 1;
transform: scale(1);
}
20% {
transform: scale(1);
filter: blur(0);
}
100% {
transform: scale(0.8) translateY(-50%);
filter: blur(25px);
opacity: 0;
}
}

View File

@@ -0,0 +1,55 @@
"use client";
import React, { useEffect, useRef, useState } from "react";
import { useAppMessage } from "@daily-co/daily-react";
import { DailyEventObjectAppMessage } from "@daily-co/daily-js";
import styles from "./StoryTranscript.module.css";
export default function StoryTranscript() {
const [partialText, setPartialText] = useState<string>("");
const [sentences, setSentences] = useState<string[]>([]);
const intervalRef = useRef<any | null>(null);
useEffect(() => {
clearInterval(intervalRef.current);
intervalRef.current = setInterval(() => {
if (sentences.length > 2) {
setSentences((s) => s.slice(1));
}
}, 2500);
return () => clearInterval(intervalRef.current);
}, [sentences]);
useAppMessage({
onAppMessage: (e: DailyEventObjectAppMessage<any>) => {
if (e.fromId && e.fromId === "transcription") {
// Check for LLM transcripts only
if (e.data.user_id !== "") {
setPartialText(e.data.text);
if (e.data.is_final) {
setPartialText("");
setSentences((s) => [...s, e.data.text]);
}
}
}
},
});
return (
<div className={styles.container}>
{sentences.map((sentence, index) => (
<p key={index} className={`${styles.transcript} ${styles.sentence}`}>
<span>{sentence}</span>
</p>
))}
{partialText && (
<p className={`${styles.transcript}`}>
<span>{partialText}</span>
</p>
)}
</div>
);
}

View File

@@ -0,0 +1,82 @@
.panel{
width: 100%;
pointer-events: none;
color: #ffffff;
text-align: center;
position: relative;
@apply pb-8;
}
.panel::after{
content: "";
position: absolute;
inset: 0px;
z-index: 10;
opacity: 0;
transition: all 0.3s ease;
@apply bg-gradient-to-t from-gray-950 to-transparent;
}
.active::after{ opacity: 1;}
.micIcon{
position: relative;
width: 120px;
height: 120px;
border-radius: 120px;
border: 6px solid;
outline: 6px solid;
display: flex;
justify-content: center;
align-items: center;
margin: 0 auto;
z-index: 20;
transition: all 0.5s ease;
@apply bg-gray-700/60 border-gray-600 outline-gray-900/20;
}
.micIcon svg{
position: relative;
z-index: 20;
opacity: 0.25;
transition: opacity 0.5s ease;
}
@keyframes pulse {
0% {
outline-width: 6px;
@apply outline-teal-500/10;
}
50% {
outline-width: 24px;
@apply outline-teal-500/50;
}
100% {
outline-width: 6px;
@apply outline-teal-500/10;
}
}
.micIconActive{
@apply bg-teal-950 border-teal-500 outline-teal-500/20;
animation: pulse 2s infinite ease-in-out;
}
.micIconActive svg{
opacity: 1;
}
.transcript{
flex: 0;
align-self: center;
opacity: 0.25;
transition: opacity 1s ease;
transition-delay: 2.5s;
@apply bg-gray-900/90 font-medium py-1 px-2 rounded-sm mt-4;
}
.active .transcript{
opacity: 1;
}

View File

@@ -0,0 +1,48 @@
import React, { useState, useEffect } from "react";
import { useAppMessage } from "@daily-co/daily-react";
import { DailyEventObjectAppMessage } from "@daily-co/daily-js";
import styles from "./UserInputIndicator.module.css";
import { IconMicrophone } from "@tabler/icons-react";
import { TypewriterEffect } from "../ui/typewriter";
import AudioIndicator from "../AudioIndicator";
interface Props {
active: boolean;
}
export default function UserInputIndicator({ active }: Props) {
const [transcription, setTranscription] = useState<string[]>([]);
useAppMessage({
onAppMessage: (e: DailyEventObjectAppMessage<any>) => {
if (e.fromId && e.fromId === "transcription") {
if (e.data.user_id === "" && e.data.is_final) {
setTranscription((t) => [...t, ...e.data.text.split(" ")]);
}
}
},
});
useEffect(() => {
if (active) return;
const t = setTimeout(() => setTranscription([]), 4000);
return () => clearTimeout(t);
}, [active]);
return (
<div className={`${styles.panel} ${active ? styles.active : ""}`}>
<div className="relative z-20 flex flex-col">
<div
className={`${styles.micIcon} ${active ? styles.micIconActive : ""}`}
>
<IconMicrophone size={42} />
{active && <AudioIndicator />}
</div>
<footer className={styles.transcript}>
<TypewriterEffect words={transcription} />
</footer>
</div>
</div>
);
}

View File

@@ -0,0 +1,34 @@
.container{
position: relative;
animation: fadeIn 3s ease;
transition: all 3s ease-out;
}
.videoTile{
@apply bg-gray-950;
width: 560px;
height: 560px;
mask-image: url('/alpha-mask.gif');
mask-size: cover;
mask-repeat: no-repeat;
margin:0 auto;
z-index: 10;
position: relative;
}
.inactive{
opacity: 0.7;
filter:blur(3px);
transform: scale(0.95)
}
@keyframes fadeIn {
0% {
filter: blur(100px);
opacity: 0;
}
100% {
filter: blur(0px);
opacity: 1;
}
}

View File

@@ -0,0 +1,27 @@
import React from "react";
import styles from "./VideoTile.module.css";
import { DailyVideo } from "@daily-co/daily-react";
import StoryTranscript from "@/components/StoryTranscript";
interface Props {
sessionId: string;
inactive: boolean;
}
const VideoTile = ({ sessionId, inactive }: Props) => {
return (
<div className={`${styles.container} ${inactive ? styles.inactive : ""} `}>
<StoryTranscript />
<div className={styles.videoTile}>
<DailyVideo
sessionId={sessionId}
type={"video"}
className="aspect-square"
/>
</div>
</div>
);
};
export default VideoTile;

View File

@@ -0,0 +1,68 @@
.waveText{
color: white;
text-shadow: 3px 3px 0px rgba(0,0,0,0.5);
opacity: 0;
transition: opacity 2s ease;
position: relative;
@apply text-4xl font-bold;
}
.active{
opacity: 1;
}
@keyframes jump {
0% {
opacity: 0.5;
transform:translateY(0px)
}
50% {
opacity: 1;
transform:translateY(-30px);
}
100% {
opacity: 0.55;
transform:translateY(0px)
}
}
.waveText span{
display:inline-block;
animation:jump 2s infinite ease-in-out;
}
.waveText span:nth-child(1) {
animation-delay:0s;
}
.waveText span:nth-child(1) {
animation-delay:0.1s;
}
.waveText span:nth-child(2) {
animation-delay:0.2s;
}
.waveText span:nth-child(3) {
animation-delay:0.3s;
}
.waveText span:nth-child(4) {
animation-delay:0.4s;
}
.waveText span:nth-child(5) {
animation-delay:0.5s;
}
.waveText span:nth-child(6) {
animation-delay:0.6s;
}
.waveText span:nth-child(7) {
animation-delay:0.7s;
}
.waveText span:nth-child(8) {
animation-delay:0.8s;
}
.waveText span:nth-child(9) {
animation-delay:0.9s;
}
.waveText span:nth-child(10) {
animation-delay:1s;
}

View File

@@ -0,0 +1,23 @@
import React from "react";
import styles from "./WaveText.module.css";
interface Props {
active: boolean;
}
export default function WaveText({ active }: Props) {
return (
<div className={`${styles.waveText} ${active ? styles.active : ""}`}>
<span>W</span>
<span>h</span>
<span>a</span>
<span>t</span>
<span>&nbsp;&nbsp;</span>
<span>n</span>
<span>e</span>
<span>x</span>
<span>t</span>
<span>?</span>
</div>
);
}

View File

@@ -0,0 +1,56 @@
import * as React from "react";
import { Slot } from "@radix-ui/react-slot";
import { cva, type VariantProps } from "class-variance-authority";
import { cn } from "@/app/utils";
const buttonVariants = cva(
"inline-flex items-center justify-center whitespace-nowrap rounded-md text-md font-medium ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50",
{
variants: {
variant: {
default: "bg-primary text-primary-foreground hover:bg-primary/90",
destructive:
"bg-destructive text-destructive-foreground hover:bg-destructive/90",
outline:
"border border-input bg-background hover:bg-accent hover:text-accent-foreground",
secondary:
"bg-secondary text-secondary-foreground hover:bg-secondary/80",
ghost: "hover:bg-accent hover:text-accent-foreground",
link: "text-primary underline-offset-4 hover:underline",
},
size: {
default: "h-11 rounded-lg px-6 py-2",
sm: "h-9 rounded-md px-3",
lg: "h-12 rounded-xl px-8",
icon: "h-10 w-10",
},
},
defaultVariants: {
variant: "default",
size: "default",
},
}
);
export interface ButtonProps
extends React.ButtonHTMLAttributes<HTMLButtonElement>,
VariantProps<typeof buttonVariants> {
asChild?: boolean;
}
const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
({ className, variant, size, asChild = false, ...props }, ref) => {
const Comp = asChild ? Slot : "button";
return (
<Comp
className={cn(buttonVariants({ variant, size, className }))}
ref={ref}
{...props}
/>
);
}
);
Button.displayName = "Button";
export { Button, buttonVariants };

View File

@@ -0,0 +1,160 @@
"use client";
import * as React from "react";
import * as SelectPrimitive from "@radix-ui/react-select";
import { IconCheck, IconChevronDown, IconChevronUp } from "@tabler/icons-react";
import { cn } from "@/app/utils";
const Select = SelectPrimitive.Root;
const SelectGroup = SelectPrimitive.Group;
const SelectValue = SelectPrimitive.Value;
const SelectTrigger = React.forwardRef<
React.ElementRef<typeof SelectPrimitive.Trigger>,
React.ComponentPropsWithoutRef<typeof SelectPrimitive.Trigger>
>(({ className, children, ...props }, ref) => (
<SelectPrimitive.Trigger
ref={ref}
className={cn(
"flex gap-3 h-12 w-full items-center justify-between rounded-xl border border-input bg-background px-4 py-2 text-sm ring-offset-background placeholder:text-muted-foreground focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 [&>span]:line-clamp-1",
className
)}
{...props}
>
{children}
<SelectPrimitive.Icon asChild>
<IconChevronDown className="h-4 w-4 opacity-50" />
</SelectPrimitive.Icon>
</SelectPrimitive.Trigger>
));
SelectTrigger.displayName = SelectPrimitive.Trigger.displayName;
const SelectScrollUpButton = React.forwardRef<
React.ElementRef<typeof SelectPrimitive.ScrollUpButton>,
React.ComponentPropsWithoutRef<typeof SelectPrimitive.ScrollUpButton>
>(({ className, ...props }, ref) => (
<SelectPrimitive.ScrollUpButton
ref={ref}
className={cn(
"flex cursor-default items-center justify-center py-1",
className
)}
{...props}
>
<IconChevronUp className="h-4 w-4" />
</SelectPrimitive.ScrollUpButton>
));
SelectScrollUpButton.displayName = SelectPrimitive.ScrollUpButton.displayName;
const SelectScrollDownButton = React.forwardRef<
React.ElementRef<typeof SelectPrimitive.ScrollDownButton>,
React.ComponentPropsWithoutRef<typeof SelectPrimitive.ScrollDownButton>
>(({ className, ...props }, ref) => (
<SelectPrimitive.ScrollDownButton
ref={ref}
className={cn(
"flex cursor-default items-center justify-center py-1",
className
)}
{...props}
>
<IconChevronDown className="h-4 w-4" />
</SelectPrimitive.ScrollDownButton>
));
SelectScrollDownButton.displayName =
SelectPrimitive.ScrollDownButton.displayName;
const SelectContent = React.forwardRef<
React.ElementRef<typeof SelectPrimitive.Content>,
React.ComponentPropsWithoutRef<typeof SelectPrimitive.Content>
>(({ className, children, position = "popper", ...props }, ref) => (
<SelectPrimitive.Portal>
<SelectPrimitive.Content
ref={ref}
className={cn(
"relative z-50 max-h-96 min-w-[8rem] overflow-hidden rounded-md border bg-popover text-popover-foreground shadow-md data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2",
position === "popper" &&
"data-[side=bottom]:translate-y-1 data-[side=left]:-translate-x-1 data-[side=right]:translate-x-1 data-[side=top]:-translate-y-1",
className
)}
position={position}
{...props}
>
<SelectScrollUpButton />
<SelectPrimitive.Viewport
className={cn(
"p-1",
position === "popper" &&
"h-[var(--radix-select-trigger-height)] w-full min-w-[var(--radix-select-trigger-width)]"
)}
>
{children}
</SelectPrimitive.Viewport>
<SelectScrollDownButton />
</SelectPrimitive.Content>
</SelectPrimitive.Portal>
));
SelectContent.displayName = SelectPrimitive.Content.displayName;
const SelectLabel = React.forwardRef<
React.ElementRef<typeof SelectPrimitive.Label>,
React.ComponentPropsWithoutRef<typeof SelectPrimitive.Label>
>(({ className, ...props }, ref) => (
<SelectPrimitive.Label
ref={ref}
className={cn("py-1.5 pl-8 pr-2 text-sm font-semibold", className)}
{...props}
/>
));
SelectLabel.displayName = SelectPrimitive.Label.displayName;
const SelectItem = React.forwardRef<
React.ElementRef<typeof SelectPrimitive.Item>,
React.ComponentPropsWithoutRef<typeof SelectPrimitive.Item>
>(({ className, children, ...props }, ref) => (
<SelectPrimitive.Item
ref={ref}
className={cn(
"relative flex w-full cursor-default select-none items-center rounded-sm py-1.5 pl-8 pr-2 text-sm outline-none focus:bg-accent focus:text-accent-foreground data-[disabled]:pointer-events-none data-[disabled]:opacity-50",
className
)}
{...props}
>
<span className="absolute left-2 flex h-3.5 w-3.5 items-center justify-center">
<SelectPrimitive.ItemIndicator>
<IconCheck className="h-4 w-4" />
</SelectPrimitive.ItemIndicator>
</span>
<SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
</SelectPrimitive.Item>
));
SelectItem.displayName = SelectPrimitive.Item.displayName;
const SelectSeparator = React.forwardRef<
React.ElementRef<typeof SelectPrimitive.Separator>,
React.ComponentPropsWithoutRef<typeof SelectPrimitive.Separator>
>(({ className, ...props }, ref) => (
<SelectPrimitive.Separator
ref={ref}
className={cn("-mx-1 my-1 h-px bg-muted", className)}
{...props}
/>
));
SelectSeparator.displayName = SelectPrimitive.Separator.displayName;
export {
Select,
SelectGroup,
SelectValue,
SelectTrigger,
SelectContent,
SelectLabel,
SelectItem,
SelectSeparator,
SelectScrollUpButton,
SelectScrollDownButton,
};

View File

@@ -0,0 +1,61 @@
"use client";
import { cn } from "@/app/utils";
import { motion } from "framer-motion";
export const TypewriterEffect = ({
words,
className,
}: {
words: string[];
className?: string;
cursorClassName?: string;
}) => {
const renderWords = () => {
return (
<div>
{words.map((word, idx) => {
return (
<div key={`word-${idx}`} className="inline-block">
{word.split("").map((char, index) => (
<span key={`char-${index}`}>{char}</span>
))}
&nbsp;
</div>
);
})}
</div>
);
};
return (
<div className={cn("flex", className)}>
{words.length < 1 ? (
<span>...</span>
) : (
<motion.div
className="overflow-hidden"
initial={{
width: "0%",
}}
whileInView={{
width: "fit-content",
}}
transition={{
duration: 0.5,
ease: "linear",
delay: 1,
}}
>
<div
style={{
whiteSpace: "nowrap",
}}
>
{renderWords()}
</div>
</motion.div>
)}
</div>
);
};

View File

@@ -0,0 +1,2 @@
NEXT_PUBLIC_ROOM_URL=
SITE_URL=

View File

@@ -0,0 +1,15 @@
/** @type {import('next').NextConfig} */
const nextConfig = {
output: "export",
async rewrites() {
return [
{
source: "/:path*",
destination: "http://localhost:7860/:path*",
},
];
},
};
export default nextConfig;

View File

@@ -0,0 +1,38 @@
{
"name": "frontend",
"version": "0.1.0",
"private": true,
"scripts": {
"dev": "next dev",
"build": "next build",
"start": "next start",
"lint": "next lint"
},
"dependencies": {
"@daily-co/daily-js": "^0.62.0",
"@daily-co/daily-react": "^0.18.0",
"@radix-ui/react-select": "^2.0.0",
"@radix-ui/react-slot": "^1.0.2",
"@tabler/icons-react": "^3.1.0",
"class-variance-authority": "^0.7.0",
"clsx": "^2.1.0",
"framer-motion": "^11.0.27",
"next": "14.1.4",
"react": "^18",
"react-dom": "^18",
"recoil": "^0.7.7",
"tailwind-merge": "^2.2.2",
"tailwindcss-animate": "^1.0.7"
},
"devDependencies": {
"@types/node": "^20",
"@types/react": "^18",
"@types/react-dom": "^18",
"autoprefixer": "^10.0.1",
"eslint": "^8",
"eslint-config-next": "14.1.4",
"postcss": "^8",
"tailwindcss": "^3.4.3",
"typescript": "^5"
}
}

View File

@@ -0,0 +1,6 @@
module.exports = {
plugins: {
tailwindcss: {},
autoprefixer: {},
},
};

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 788 KiB

View File

@@ -0,0 +1,86 @@
import type { Config } from "tailwindcss";
import { fontFamily } from "tailwindcss/defaultTheme";
const config = {
darkMode: ["class"],
content: [
"./pages/**/*.{ts,tsx}",
"./components/**/*.{ts,tsx}",
"./app/**/*.{ts,tsx}",
"./src/**/*.{ts,tsx}",
],
prefix: "",
theme: {
container: {
center: true,
padding: "2rem",
screens: {
"2xl": "1400px",
},
},
extend: {
fontFamily: {
sans: ["var(--font-sans)", ...fontFamily.sans],
mono: ["var(--font-mono)", ...fontFamily.mono],
},
colors: {
border: "hsl(var(--border))",
input: "hsl(var(--input))",
ring: "hsl(var(--ring))",
background: "hsl(var(--background))",
foreground: "hsl(var(--foreground))",
primary: {
DEFAULT: "hsl(var(--primary))",
foreground: "hsl(var(--primary-foreground))",
},
secondary: {
DEFAULT: "hsl(var(--secondary))",
foreground: "hsl(var(--secondary-foreground))",
},
destructive: {
DEFAULT: "hsl(var(--destructive))",
foreground: "hsl(var(--destructive-foreground))",
},
muted: {
DEFAULT: "hsl(var(--muted))",
foreground: "hsl(var(--muted-foreground))",
},
accent: {
DEFAULT: "hsl(var(--accent))",
foreground: "hsl(var(--accent-foreground))",
},
popover: {
DEFAULT: "hsl(var(--popover))",
foreground: "hsl(var(--popover-foreground))",
},
card: {
DEFAULT: "hsl(var(--card))",
foreground: "hsl(var(--card-foreground))",
},
},
borderRadius: {
lg: "var(--radius)",
md: "calc(var(--radius) - 2px)",
sm: "calc(var(--radius) - 4px)",
},
keyframes: {
"accordion-down": {
from: { height: "0" },
to: { height: "var(--radix-accordion-content-height)" },
},
"accordion-up": {
from: { height: "var(--radix-accordion-content-height)" },
to: { height: "0" },
},
},
animation: {
"accordion-down": "accordion-down 0.2s ease-out",
"accordion-up": "accordion-up 0.2s ease-out",
},
},
},
plugins: [require("tailwindcss-animate")],
} satisfies Config;
export default config;

View File

@@ -0,0 +1,40 @@
{
"compilerOptions": {
"lib": [
"dom",
"dom.iterable",
"esnext"
],
"allowJs": true,
"skipLibCheck": true,
"strict": true,
"noEmit": true,
"esModuleInterop": true,
"module": "esnext",
"moduleResolution": "bundler",
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": "preserve",
"incremental": true,
"plugins": [
{
"name": "next"
}
],
"paths": {
"@/*": [
"./*"
]
}
},
"include": [
"next-env.d.ts",
"**/*.ts",
"**/*.tsx",
".next/types/**/*.ts",
"build/types/**/*.ts"
],
"exclude": [
"node_modules"
]
}

File diff suppressed because it is too large Load Diff

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 MiB

View File

@@ -0,0 +1,5 @@
dailyai[daily,openai,fal]==0.0.8
fastapi
uvicorn
requests
python-dotenv

Binary file not shown.

After

Width:  |  Height:  |  Size: 803 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 835 KiB

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@@ -0,0 +1,172 @@
import asyncio
import aiohttp
import logging
import os
import argparse
from dailyai.pipeline.pipeline import Pipeline
from dailyai.pipeline.frames import (
AudioFrame,
ImageFrame,
EndPipeFrame,
LLMMessagesFrame,
SendAppMessageFrame
)
from dailyai.pipeline.aggregators import (
LLMUserResponseAggregator,
LLMAssistantResponseAggregator,
)
from dailyai.transports.daily_transport import DailyTransport
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.open_ai_services import OpenAILLMService
from dailyai.services.fal_ai_services import FalImageGenService
from processors import StoryProcessor, StoryImageProcessor
from prompts import LLM_BASE_PROMPT, LLM_INTRO_PROMPT, CUE_USER_TURN
from utils.helpers import load_sounds, load_images
from dotenv import load_dotenv
load_dotenv(override=True)
logging.basicConfig(format=f"[STORYBOT] %(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.INFO)
sounds = load_sounds(["listening.wav"])
images = load_images(["book1.png", "book2.png"])
async def main(room_url, token=None):
async with aiohttp.ClientSession() as session:
# -------------- Transport --------------- #
transport = DailyTransport(
room_url,
token,
"Storytelling Bot",
duration_minutes=5,
start_transcription=True,
mic_enabled=True,
mic_sample_rate=16000,
vad_enabled=True,
camera_framerate=30,
camera_bitrate=680000,
camera_enabled=True,
camera_width=768,
camera_height=768,
)
logger.debug("Transport created for room:" + room_url)
# -------------- Services --------------- #
llm_service = OpenAILLMService(
api_key=os.getenv("OPENAI_API_KEY"),
model="gpt-4-turbo"
)
tts_service = ElevenLabsTTSService(
aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
)
fal_service_params = FalImageGenService.InputParams(
image_size={
"width": 768,
"height": 768
}
)
fal_service = FalImageGenService(
aiohttp_session=session,
model="fal-ai/fast-lightning-sdxl",
params=fal_service_params,
key=os.getenv("FAL_KEY"),
)
# --------------- Setup ----------------- #
message_history = [LLM_BASE_PROMPT]
story_pages = []
# We need aggregators to keep track of user and LLM responses
llm_responses = LLMAssistantResponseAggregator(message_history)
user_responses = LLMUserResponseAggregator(message_history)
# -------------- Processors ------------- #
story_processor = StoryProcessor(message_history, story_pages)
image_processor = StoryImageProcessor(fal_service)
# -------------- Story Loop ------------- #
logger.debug("Waiting for participant...")
start_storytime_event = asyncio.Event()
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport, participant):
logger.debug("Participant joined, storytime commence!")
start_storytime_event.set()
# The storytime coroutine will wait for the start_storytime_event
# to be set before starting the storytime pipeline
async def storytime():
await start_storytime_event.wait()
# The intro pipeline is used to start
# the story (as per LLM_INTRO_PROMPT)
intro_pipeline = Pipeline(processors=[
llm_service,
tts_service,
], sink=transport.send_queue)
await intro_pipeline.queue_frames(
[
ImageFrame(images['book1'], (768, 768)),
LLMMessagesFrame([LLM_INTRO_PROMPT]),
SendAppMessageFrame(CUE_USER_TURN, None),
AudioFrame(sounds["listening"]),
ImageFrame(images['book2'], (768, 768)),
EndPipeFrame(),
]
)
# We start the pipeline as soon as the user joins
await intro_pipeline.run_pipeline()
# The main story pipeline is used to continue the
# story based on user input
pipeline = Pipeline(processors=[
user_responses,
llm_service,
story_processor,
image_processor,
tts_service,
llm_responses,
])
await transport.run_pipeline(pipeline)
transport.transcription_settings["extra"]["endpointing"] = True
transport.transcription_settings["extra"]["punctuate"] = True
try:
await asyncio.gather(transport.run(), storytime())
except (asyncio.CancelledError, KeyboardInterrupt):
transport.stop()
logger.debug("Pipeline finished. Exiting.")
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Daily Storyteller Bot")
parser.add_argument("-u", type=str, help="Room URL")
parser.add_argument("-t", type=str, help="Token")
config = parser.parse_args()
asyncio.run(main(config.u, config.t))

View File

@@ -0,0 +1,148 @@
from typing import AsyncGenerator
import re
from dailyai.pipeline.frames import TextFrame, Frame, AudioFrame
from dailyai.pipeline.frame_processor import FrameProcessor
from dailyai.pipeline.frames import (
Frame,
TextFrame,
SendAppMessageFrame,
LLMResponseEndFrame,
UserStoppedSpeakingFrame,
)
from utils.helpers import load_sounds
from prompts import IMAGE_GEN_PROMPT, CUE_USER_TURN, CUE_ASSISTANT_TURN
import asyncio
sounds = load_sounds(["talking.wav", "listening.wav", "ding.wav"])
# -------------- Frame Types ------------- #
class StoryPageFrame(TextFrame):
# Frame for each sentence in the story before a [break]
pass
class StoryImageFrame(TextFrame):
# Frame for trigger image generation
pass
class StoryPromptFrame(TextFrame):
# Frame for prompting the user for input
pass
# ------------ Frame Processors ----------- #
class StoryImageProcessor(FrameProcessor):
"""
Processor for image prompt frames that will be sent to the FAL service.
This processor is responsible for consuming frames of type `StoryImageFrame`.
It processes the by passing it to the FAL service
The processed frames are then yielded back.
Attributes:
_fal_service (FALService): The FAL service, generates the images (fast fast!).
"""
def __init__(self, fal_service):
self._fal_service = fal_service
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, StoryImageFrame):
try:
async with asyncio.timeout(7):
async for i in self._fal_service.process_frame(TextFrame(IMAGE_GEN_PROMPT % frame.text)):
yield i
except TimeoutError:
pass
pass
else:
yield frame
class StoryProcessor(FrameProcessor):
"""
Primary frame processor. It takes the frames generated by the LLM
and processes them into image prompts and story pages (sentences.)
For a clearer picture of how this works, reference prompts.py
Attributes:
_messages (list): A list of llm messages.
_text (str): A buffer to store the text from text frames.
_story (list): A list to store the story sentences, or 'pages'.
Methods:
process_frame: Processes a frame and removes any [break] or [image] tokens.
"""
def __init__(self, messages, story):
self._messages = messages
self._text = ""
self._story = story
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, UserStoppedSpeakingFrame):
# Send an app message to the UI
yield SendAppMessageFrame(CUE_ASSISTANT_TURN, None)
yield AudioFrame(sounds["talking"])
elif isinstance(frame, TextFrame):
# We want to look for sentence breaks in the text
# but since TextFrames are streamed from the LLM
# we need to keep a buffer of the text we've seen so far
self._text += frame.text
# IMAGE PROMPT
# Looking for: < [image prompt] > in the LLM response
# We prompted our LLM to add an image prompt in the response
# so we use regex matching to find it and yield a StoryImageFrame
if re.search(r"<.*?>", self._text):
if not re.search(r"<.*?>.*?>", self._text):
# Pass any frames until we have a closing bracket
# otherwise the image prompt will be passed to TTS
pass
# Extract the image prompt from the text using regex
image_prompt = re.search(r"<(.*?)>", self._text).group(1)
# Remove the image prompt from the text
self._text = re.sub(r"<.*?>", '', self._text, count=1)
# Process the image prompt frame
yield StoryImageFrame(image_prompt)
# STORY PAGE
# Looking for: [break] in the LLM response
# We prompted our LLM to add a [break] after each sentence
# so we use regex matching to find it in the LLM response
if re.search(r".*\[[bB]reak\].*", self._text):
# Remove the [break] token from the text
# so it isn't spoken out loud by the TTS
self._text = re.sub(r'\[[bB]reak\]', '',
self._text, flags=re.IGNORECASE)
self._text = self._text.replace("\n", " ")
if len(self._text) > 2:
# Append the sentence to the story
self._story.append(self._text)
yield StoryPageFrame(self._text)
# Assert that it's the LLMs turn, until we're finished
yield SendAppMessageFrame(CUE_ASSISTANT_TURN, None)
# Clear the buffer
self._text = ""
# End of LLM response
# Driven by the prompt, the LLM should have asked the user for input
elif isinstance(frame, LLMResponseEndFrame):
# We use a different frame type, as to avoid image generation ingest
yield StoryPromptFrame(self._text)
self._text = ""
yield frame
# Send an app message to the UI
yield SendAppMessageFrame(CUE_USER_TURN, None)
yield AudioFrame(sounds["listening"])
# Anything that is not a TextFrame pass through
else:
yield frame

View File

@@ -0,0 +1,35 @@
LLM_INTRO_PROMPT = {
"role": "system",
"content": "You are a creative storyteller who loves to tell whimsical, fantastical stories. \
Your goal is to craft an engaging and fun story. \
Start by asking the user what kind of story they'd like to hear. Don't provide any examples. \
Keep your reponse to only a few sentences."
}
LLM_BASE_PROMPT = {
"role": "system",
"content": "You are a creative storyteller who loves tell whimsical, fantastical stories. \
Your goal is to craft an engaging and fun story. \
Keep all responses short and no more than a few sentences. Include [break] after each sentence of the story. \
\
Start each sentence with an image prompt, wrapped in triangle braces, that I can use to generate an illustration representing the upcoming scene. \
Image prompts should always be wrapped in triangle braces, like this: <image prompt goes here>. \
You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence. \
For any recurring characters, you should provide a description of them in the image prompt each time, for example: <a brown fluffy dog ...>. \
Please do not include any character names in the image prompts, just their descriptions. \
Image prompts should focus on key visual attributes of all characters each time, for example <a brown fluffy dog and the tiny red cat ...>. \
Please use the following structure for your image prompts: characters, setting, action, and mood. \
Image prompts should be less than 150-200 characters and start in lowercase. \
\
Responses should use the format: <...> story sentence [break] <...> story sentence [break] ... \
After each response, ask me how I'd like the story to continue and wait for my input. \
Please ensure your responses are less than 3-4 sentences long. \
Please refrain from using any explicit language or content. Do not tell scary stories."
}
IMAGE_GEN_PROMPT = "illustrative art of %s. In the style of Studio Ghibli. colorful, whimsical, painterly, concept art."
CUE_USER_TURN = {"cue": "user_turn"}
CUE_ASSISTANT_TURN = {"cue": "assistant_turn"}

View File

@@ -0,0 +1,175 @@
import os
import argparse
import subprocess
import atexit
from pathlib import Path
from typing import Optional
from fastapi import FastAPI, Request, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse, JSONResponse, RedirectResponse
from utils.daily_helpers import create_room as _create_room, get_token, get_name_from_url
MAX_BOTS_PER_ROOM = 1
# Bot sub-process dict for status reporting and concurrency control
bot_procs = {}
def cleanup():
# Clean up function, just to be extra safe
for proc in bot_procs.values():
proc.terminate()
proc.wait()
atexit.register(cleanup)
app = FastAPI()
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# Mount the static directory
STATIC_DIR = "frontend/out"
app.mount("/static", StaticFiles(directory=STATIC_DIR, html=True), name="static")
@app.post("/create")
async def create_room(request: Request) -> JSONResponse:
data = await request.json()
if data.get('room_url') is not None:
room_url = data.get('room_url')
room_name = get_name_from_url(room_url)
else:
room_url, room_name = _create_room()
token = get_token(room_url)
return JSONResponse({"room_url": room_url, "room_name": room_name, "token": token})
@app.post("/start")
async def start_agent(request: Request) -> JSONResponse:
data = await request.json()
# Is this a webhook creation request?
if "test" in data:
return JSONResponse({"test": True})
# Ensure the room property is present
room_url = data.get('room_url')
if not room_url:
raise HTTPException(
status_code=500,
detail="Missing 'room' property in request data. Cannot start agent without a target room!")
# Check if there is already an existing process running in this room
num_bots_in_room = sum(
1 for proc in bot_procs.values() if proc[1] == room_url and proc[0].poll() is None)
if num_bots_in_room >= MAX_BOTS_PER_ROOM:
raise HTTPException(
status_code=500, detail=f"Max bot limited reach for room: {room_url}")
# Get the token for the room
token = get_token(room_url)
if not token:
raise HTTPException(
status_code=500, detail=f"Failed to get token for room: {room_url}")
# Spawn a new agent, and join the user session
# Note: this is mostly for demonstration purposes (refer to 'deployment' in README)
try:
proc = subprocess.Popen(
[
f"python3 -m bot -u {room_url} -t {token}"
],
shell=True,
bufsize=1,
cwd=os.path.dirname(os.path.abspath(__file__))
)
bot_procs[proc.pid] = (proc, room_url)
except Exception as e:
raise HTTPException(
status_code=500, detail=f"Failed to start subprocess: {e}")
return JSONResponse({"bot_id": proc.pid, "room_url": room_url})
@app.get("/status/{pid}")
def get_status(pid: int):
# Look up the subprocess
proc = bot_procs.get(pid)
# If the subprocess doesn't exist, return an error
if not proc:
raise HTTPException(
status_code=404, detail=f"Bot with process id: {pid} not found")
# Check the status of the subprocess
if proc[0].poll() is None:
status = "running"
else:
status = "finished"
return JSONResponse({"bot_id": pid, "status": status})
@app.get("/{path_name:path}", response_class=FileResponse)
async def catch_all(path_name: Optional[str] = ""):
if path_name == "":
return FileResponse(f"{STATIC_DIR}/index.html")
file_path = Path(STATIC_DIR) / (path_name or "")
if file_path.is_file():
return file_path
html_file_path = file_path.with_suffix(".html")
if html_file_path.is_file():
return FileResponse(html_file_path)
raise HTTPException(status_code=450, detail="Incorrect API call")
if __name__ == "__main__":
# Check environment variables
required_env_vars = ['OPENAI_API_KEY', 'DAILY_API_KEY',
'FAL_KEY', 'ELEVENLABS_VOICE_ID', 'ELEVENLABS_API_KEY']
for env_var in required_env_vars:
if env_var not in os.environ:
raise Exception(f"Missing environment variable: {env_var}.")
import uvicorn
default_host = os.getenv("HOST", "0.0.0.0")
default_port = int(os.getenv("FAST_API_PORT", "7860"))
parser = argparse.ArgumentParser(
description="Daily Storyteller FastAPI server")
parser.add_argument("--host", type=str,
default=default_host, help="Host address")
parser.add_argument("--port", type=int,
default=default_port, help="Port number")
parser.add_argument("--reload", action="store_true",
help="Reload code on change")
config = parser.parse_args()
uvicorn.run(
"server:app",
host=config.host,
port=config.port,
reload=config.reload
)

View File

@@ -0,0 +1,109 @@
import urllib.parse
import os
import time
import urllib
import requests
from dotenv import load_dotenv
load_dotenv()
daily_api_path = os.getenv("DAILY_API_URL")
daily_api_key = os.getenv("DAILY_API_KEY")
def create_room() -> tuple[str, str]:
"""
Helper function to create a Daily room.
# See: https://docs.daily.co/reference/rest-api/rooms
Returns:
tuple: A tuple containing the room URL and room name.
Raises:
Exception: If the request to create the room fails or if the response does not contain the room URL or room name.
"""
room_props = {
"exp": time.time() + 60 * 60, # 1 hour
"enable_chat": True,
"enable_emoji_reactions": True,
"eject_at_room_exp": True,
"enable_prejoin_ui": False, # Important for the bot to be able to join headlessly
}
res = requests.post(
f"https://{daily_api_path}/rooms",
headers={"Authorization": f"Bearer {daily_api_key}"},
json={
"properties": room_props
},
)
if res.status_code != 200:
raise Exception(f"Unable to create room: {res.text}")
data = res.json()
room_url: str = data.get("url")
room_name: str = data.get("name")
if room_url is None or room_name is None:
raise Exception("Missing room URL or room name in response")
return room_url, room_name
def get_name_from_url(room_url: str) -> str:
"""
Extracts the name from a given room URL.
Args:
room_url (str): The URL of the room.
Returns:
str: The extracted name from the room URL.
"""
return urllib.parse.urlparse(room_url).path[1:]
def get_token(room_url: str) -> str:
"""
Retrieves a meeting token for the specified Daily room URL.
# See: https://docs.daily.co/reference/rest-api/meeting-tokens
Args:
room_url (str): The URL of the Daily room.
Returns:
str: The meeting token.
Raises:
Exception: If no room URL is specified or if no Daily API key is specified.
Exception: If there is an error creating the meeting token.
"""
if not room_url:
raise Exception(
"No Daily room specified. You must specify a Daily room in order a token to be generated.")
if not daily_api_key:
raise Exception(
"No Daily API key specified. set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers.")
expiration: float = time.time() + 60 * 60
room_name = get_name_from_url(room_url)
res: requests.Response = requests.post(
f"https://{daily_api_path}/meeting-tokens",
headers={
"Authorization": f"Bearer {daily_api_key}"},
json={
"properties": {
"room_name": room_name,
"is_owner": True, # Owner tokens required for transcription
"exp": expiration}},
)
if res.status_code != 200:
raise Exception(
f"Failed to create meeting token: {res.status_code} {res.text}")
token: str = res.json()["token"]
return token

View File

@@ -0,0 +1,33 @@
import os
import wave
from PIL import Image
script_dir = os.path.dirname(__file__)
def load_images(image_files):
images = {}
for file in image_files:
# Build the full path to the image file
full_path = os.path.join(script_dir, "../assets", file)
# Get the filename without the extension to use as the dictionary key
filename = os.path.splitext(os.path.basename(full_path))[0]
# Open the image and convert it to bytes
with Image.open(full_path) as img:
images[filename] = img.tobytes()
return images
def load_sounds(sound_files):
sounds = {}
for file in sound_files:
# Build the full path to the sound file
full_path = os.path.join(script_dir, "../assets", file)
# Get the filename without the extension to use as the dictionary key
filename = os.path.splitext(os.path.basename(full_path))[0]
# Open the sound and convert it to bytes
with wave.open(full_path) as audio_file:
sounds[filename] = audio_file.readframes(-1)
return sounds