This commit is contained in:
lukasIO
2024-01-09 15:05:20 +01:00
commit eae180722e
40 changed files with 7568 additions and 0 deletions

23
.env.example Normal file
View File

@@ -0,0 +1,23 @@
# LiveKit API Configuration
LIVEKIT_API_KEY=YOUR_API_KEY
LIVEKIT_API_SECRET=YOUR_API_SECRET
LIVEKIT_URL=YOUR_LIVEKIT_URL
# Public configuration
NEXT_PUBLIC_LIVEKIT_URL=YOUR_LIVEKIT_URL
# Application Configuration
NEXT_PUBLIC_APP_CONFIG="
title: 'LiveKit Agent Playground'
description: 'LiveKit Agent Playground allows you to test your LiveKit Agent integration by connecting to your LiveKit Cloud or self-hosted instance.'
github_link: 'https://github.com/livekit-examples/agent-playground'
theme_color: 'cyan'
outputs:
audio: true # Enable or disable audio output
video: true # Enable or disable video output
chat: true # Enable or disable chat feature
inputs:
mic: true # Enable or disable microphone input
camera: true # Enable or disable camera input
sip: true # Enable or disable SIP input
"

3
.eslintrc.json Normal file
View File

@@ -0,0 +1,3 @@
{
"extends": "next/core-web-vitals"
}

36
.gitignore vendored Normal file
View File

@@ -0,0 +1,36 @@
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
# dependencies
/node_modules
/.pnp
.pnp.js
.yarn/install-state.gz
# testing
/coverage
# next.js
/.next/
/out/
# production
/build
# misc
.DS_Store
*.pem
# debug
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# local env files
.env*.local
# vercel
.vercel
# typescript
*.tsbuildinfo
next-env.d.ts

53
README.md Normal file
View File

@@ -0,0 +1,53 @@
This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
## Getting Started
First, run the development server:
```bash
npm run dev
# or
yarn dev
# or
pnpm dev
# or
bun dev
```
Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
You can start editing the page by modifying `pages/index.tsx`. The page auto-updates as you edit the file.
[API routes](https://nextjs.org/docs/api-routes/introduction) can be accessed on [http://localhost:3000/api/hello](http://localhost:3000/api/hello). This endpoint can be edited in `pages/api/hello.ts`.
The `pages/api` directory is mapped to `/api/*`. Files in this directory are treated as [API routes](https://nextjs.org/docs/api-routes/introduction) instead of React pages.
This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
## Features
- Render video, audio and chat from your agent
- Send video, audio, or text to your agent
- Configurable settings panel to work with your agent
## Notes
- Update `.env` to include
```
LIVEKIT_API_KEY=<your API KEY>
LIVEKIT_API_SECRET=<Your API Secret>
LIVEKIT_URL=<Your Cloud URL>
NEXT_PUBLIC_LIVEKIT_URL=<Your Cloud URL>
```
- This playground is currently work in progress. There are known layout/responsive bugs and some features are under tested.
- The playground was tested against the kitt example in `https://github.com/livekit/python-agents`.
- Feel free to ask questions, request features at #team-agents.
- Feel free to add features yourself if there's something you want to see
## Known issues
- "Disconnect" in header is not yet functional
- Layout can break on smaller screens.
- Mobile device sizes not supported currently

6
next.config.js Normal file
View File

@@ -0,0 +1,6 @@
/** @type {import('next').NextConfig} */
const nextConfig = {
reactStrictMode: false,
};
module.exports = nextConfig;

5345
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

34
package.json Normal file
View File

@@ -0,0 +1,34 @@
{
"name": "kitt-web-v2",
"version": "0.1.0",
"private": true,
"scripts": {
"dev": "next dev",
"build": "next build",
"start": "next start",
"lint": "next lint"
},
"dependencies": {
"@livekit/components-react": "latest",
"framer-motion": "^10.16.16",
"js-yaml": "^4.1.0",
"livekit-client": "^1.15.4",
"livekit-server-sdk": "^1.2.7",
"next": "14.0.1",
"qrcode.react": "^3.1.0",
"react": "^18",
"react-dom": "^18"
},
"devDependencies": {
"@types/js-yaml": "^4.0.9",
"@types/node": "^20.10.4",
"@types/react": "^18.2.43",
"@types/react-dom": "^18",
"autoprefixer": "^10.4.16",
"eslint": "^8",
"eslint-config-next": "14.0.1",
"postcss": "^8.4.31",
"tailwindcss": "^3.3.5",
"typescript": "^5.3.3"
}
}

6
postcss.config.js Normal file
View File

@@ -0,0 +1,6 @@
module.exports = {
plugins: {
tailwindcss: {},
autoprefixer: {},
},
}

BIN
public/favicon.ico Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

16
public/logo.svg Normal file
View File

@@ -0,0 +1,16 @@
<svg width="164" height="32" viewBox="0 0 164 32" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_101_119699)">
<path d="M19.2006 12.7998H12.7996V19.2008H19.2006V12.7998Z" fill="currentColor"/>
<path d="M25.6014 6.40137H19.2004V12.8024H25.6014V6.40137Z" fill="currentColor"/>
<path d="M25.6014 19.2002H19.2004V25.6012H25.6014V19.2002Z" fill="currentColor"/>
<path d="M32 0H25.599V6.401H32V0Z" fill="currentColor"/>
<path d="M32 25.5986H25.599V31.9996H32V25.5986Z" fill="currentColor"/>
<path d="M6.401 25.599V19.2005V12.7995V6.401V0H0V6.401V12.7995V19.2005V25.599V32H6.401H12.7995H19.2005V25.599H12.7995H6.401Z" fill="white"/>
</g>
<path d="M44.23 12.5L46.49 4.9H48.52L50.78 12.5H49.7L49.07 10.34H45.94L45.31 12.5H44.23ZM46.2 9.44H48.81L47.51 4.95L46.2 9.44ZM55.382 12.61C53.622 12.61 52.372 11.18 52.372 8.7C52.372 6.22 53.632 4.79 55.572 4.79C57.142 4.79 58.102 5.71 58.502 7.25H57.452C57.152 6.21 56.542 5.64 55.572 5.64C54.232 5.64 53.422 6.75 53.422 8.7C53.422 10.65 54.222 11.76 55.542 11.76C56.742 11.76 57.562 10.88 57.592 9.31H55.322V8.41H58.602V12.5H57.722V10.59C57.492 11.54 56.792 12.61 55.382 12.61ZM61.2639 12.5V4.9H66.0939V5.8H62.2739V8.13H66.0239V9.03H62.2739V11.6H66.1939V12.5H61.2639ZM68.7959 12.5V4.9H70.8359L73.2059 12.44V4.9H74.2159V12.5H72.1759L69.8059 4.96V12.5H68.7959ZM79.0078 12.5V5.8H76.4778V4.9H82.5478V5.8H80.0178V12.5H79.0078ZM45.12 27.5V19.9H47.74C49.51 19.9 50.58 20.75 50.58 22.28C50.58 23.81 49.51 24.66 47.74 24.66H46.13V27.5H45.12ZM46.13 23.76H47.77C48.9 23.76 49.53 23.28 49.53 22.28C49.53 21.28 48.9 20.8 47.77 20.8H46.13V23.76ZM53.592 27.5V19.9H54.602V26.6H58.482V27.5H53.592ZM60.2339 27.5L62.4939 19.9H64.5239L66.7839 27.5H65.7039L65.0739 25.34H61.9439L61.3139 27.5H60.2339ZM62.2039 24.44H64.8139L63.5139 19.95L62.2039 24.44ZM71.0059 27.5V24.23L68.3559 19.9H69.5059L71.5059 23.17L73.5059 19.9H74.6659L72.0159 24.23V27.5H71.0059ZM79.3878 27.61C77.6278 27.61 76.3778 26.18 76.3778 23.7C76.3778 21.22 77.6378 19.79 79.5778 19.79C81.1478 19.79 82.1078 20.71 82.5078 22.25H81.4578C81.1578 21.21 80.5478 20.64 79.5778 20.64C78.2378 20.64 77.4278 21.75 77.4278 23.7C77.4278 25.65 78.2278 26.76 79.5478 26.76C80.7478 26.76 81.5678 25.88 81.5978 24.31H79.3278V23.41H82.6078V27.5H81.7278V25.59C81.4978 26.54 80.7978 27.61 79.3878 27.61ZM85.0798 27.5V19.9H87.7398C89.3998 19.9 90.4398 20.78 90.4398 22.15C90.4398 23.2 89.8298 23.96 88.8098 24.26L90.4898 27.5H89.3098L87.7298 24.4H86.0898V27.5H85.0798ZM86.0898 23.5H87.7598C88.7698 23.5 89.3898 23.02 89.3898 22.15C89.3898 21.28 88.7698 20.8 87.7598 20.8H86.0898V23.5ZM95.5117 27.61C93.5717 27.61 92.3117 26.18 92.3117 23.7C92.3117 21.22 93.5717 19.79 95.5117 19.79C97.4517 19.79 98.7117 21.22 98.7117 23.7C98.7117 26.18 97.4517 27.61 95.5117 27.61ZM93.3617 23.7C93.3617 25.65 94.1717 26.76 95.5117 26.76C96.8517 26.76 97.6617 25.65 97.6617 23.7C97.6617 21.75 96.8517 20.64 95.5117 20.64C94.1717 20.64 93.3617 21.75 93.3617 23.7ZM103.514 27.61C101.624 27.61 100.804 26.54 100.804 24.55V24.53V19.9H101.814V24.59V24.61C101.814 26.13 102.424 26.76 103.514 26.76C104.604 26.76 105.214 26.12 105.214 24.59V19.9H106.224V24.53C106.224 26.53 105.404 27.61 103.514 27.61ZM108.806 27.5V19.9H110.846L113.216 27.44V19.9H114.226V27.5H112.186L109.816 19.96V27.5H108.806ZM116.868 27.5V19.9H118.928C121.148 19.9 122.628 21.23 122.628 23.7C122.628 26.17 121.148 27.5 118.928 27.5H116.868ZM117.878 26.6H118.958C120.608 26.6 121.578 25.57 121.578 23.7C121.578 21.84 120.608 20.8 118.958 20.8H117.878V26.6Z" fill="white" fillOpacity="0.8"/>
<defs>
<clipPath id="clip0_101_119699">
<rect width="32" height="32" fill="white"/>
</clipPath>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 3.5 KiB

1
public/next.svg Normal file
View File

@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>

After

Width:  |  Height:  |  Size: 1.3 KiB

1
public/vercel.svg Normal file
View File

@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 283 64"><path fill="black" d="M141 16c-11 0-19 7-19 18s9 18 20 18c7 0 13-3 16-7l-7-5c-2 3-6 4-9 4-5 0-9-3-10-7h28v-3c0-11-8-18-19-18zm-9 15c1-4 4-7 9-7s8 3 9 7h-18zm117-15c-11 0-19 7-19 18s9 18 20 18c6 0 12-3 16-7l-8-5c-2 3-5 4-8 4-5 0-9-3-11-7h28l1-3c0-11-8-18-19-18zm-10 15c2-4 5-7 10-7s8 3 9 7h-19zm-39 3c0 6 4 10 10 10 4 0 7-2 9-5l8 5c-3 5-9 8-17 8-11 0-19-7-19-18s8-18 19-18c8 0 14 3 17 8l-8 5c-2-3-5-5-9-5-6 0-10 4-10 10zm83-29v46h-9V5h9zM37 0l37 64H0L37 0zm92 5-27 48L74 5h10l18 30 17-30h10zm59 12v10l-3-1c-6 0-10 4-10 10v15h-9V17h9v9c0-5 6-9 13-9z"/></svg>

After

Width:  |  Height:  |  Size: 629 B

View File

@@ -0,0 +1,62 @@
import { Button } from "./button/Button";
import { useRef } from "react";
type PlaygroundConnectProps = {
accentColor: string;
onConnectClicked: (url: string, roomToken: string) => void;
};
export const PlaygroundConnect = ({
accentColor,
onConnectClicked,
}: PlaygroundConnectProps) => {
const urlInput = useRef<HTMLInputElement>(null);
const tokenInput = useRef<HTMLTextAreaElement>(null);
return (
<div className="flex left-0 top-0 w-full h-full bg-black/80 items-center justify-center text-center">
<div className="flex flex-col gap-4 p-8 bg-gray-950 w-full max-w-[400px] rounded-lg text-white border border-gray-900">
<div className="flex flex-col gap-2">
<h1 className="text-xl">Connect to playground</h1>
<p className="text-sm text-gray-500">
Connect LiveKit Agent Playground with a custom server using LiveKit
Cloud or LiveKit Server.
</p>
</div>
<div className="flex flex-col gap-2 my-4">
<input
ref={urlInput}
className="text-white text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-2"
placeholder="wss://url"
></input>
<textarea
ref={tokenInput}
className="text-white text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-2"
placeholder="room token..."
></textarea>
</div>
<Button
accentColor={accentColor}
className="w-full"
onClick={() => {
if (urlInput.current && tokenInput.current) {
onConnectClicked(
urlInput.current.value,
tokenInput.current.value
);
}
}}
>
Connect
</Button>
<a
href="https://kitt.livekit.io/"
className={`text-xs text-${accentColor}-500 hover:underline`}
>
Dont have a URL or token? Try out our KITT example to see agents in
action!
</a>
</div>
</div>
);
};

View File

@@ -0,0 +1,27 @@
import React, { ButtonHTMLAttributes, ReactNode } from "react";
type ButtonProps = {
accentColor: string;
children: ReactNode;
className?: string;
disabled?: boolean;
} & ButtonHTMLAttributes<HTMLButtonElement>;
export const Button: React.FC<ButtonProps> = ({
accentColor,
children,
className,
disabled,
...allProps
}) => {
return (
<button
className={`flex flex-row ${
disabled ? "pointer-events-none" : ""
} text-gray-950 text-sm justify-center border border-transparent bg-${accentColor}-500 px-3 py-1 rounded-md transition ease-out duration-250 hover:bg-transparent hover:shadow-${accentColor} hover:border-${accentColor}-500 hover:text-${accentColor}-500 active:scale-[0.98] ${className}`}
{...allProps}
>
{children}
</button>
);
};

View File

@@ -0,0 +1,31 @@
export const LoadingSVG = ({
diameter = 20,
strokeWidth = 4,
}: {
diameter?: number;
strokeWidth?: number;
}) => (
<svg
className="animate-spin"
fill="none"
viewBox="0 0 24 24"
style={{
width: `${diameter}px`,
height: `${diameter}px`,
}}
>
<circle
className="opacity-25"
cx="12"
cy="12"
r="10"
stroke="currentColor"
strokeWidth={strokeWidth}
></circle>
<path
className="opacity-75"
fill="currentColor"
d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
></path>
</svg>
);

View File

@@ -0,0 +1,32 @@
type ChatMessageProps = {
message: string;
accentColor: string;
name: string;
isSelf: boolean;
};
export const ChatMessage = ({
name,
message,
accentColor,
isSelf,
}: ChatMessageProps) => {
return (
<div className="flex flex-col gap-1">
<div
className={`text-${
isSelf ? "gray-700" : accentColor + "-800 text-ts-" + accentColor
} uppercase text-xs`}
>
{name}
</div>
<div
className={`pr-4 text-${
isSelf ? "gray-300" : accentColor + "-500"
} text-sm ${isSelf ? "" : "drop-shadow-" + accentColor}`}
>
{message}
</div>
</div>
);
};

View File

@@ -0,0 +1,118 @@
import { useWindowResize } from "@/hooks/useWindowResize";
import { useEffect, useRef, useState } from "react";
type ChatMessageInput = {
placeholder: string;
accentColor: string;
height: number;
onSend: (message: string) => void;
};
export const ChatMessageInput = ({
placeholder,
accentColor,
height,
onSend,
}: ChatMessageInput) => {
const [message, setMessage] = useState("");
const [inputTextWidth, setInputTextWidth] = useState(0);
const [inputWidth, setInputWidth] = useState(0);
const hiddenInputRef = useRef<HTMLSpanElement>(null);
const inputRef = useRef<HTMLInputElement>(null);
const windowSize = useWindowResize();
const [isTyping, setIsTyping] = useState(false);
const [inputHasFocus, setInputHasFocus] = useState(false);
const handleSend = () => {
if (message === "") {
return;
}
onSend(message);
setMessage("");
};
useEffect(() => {
setIsTyping(true);
const timeout = setTimeout(() => {
setIsTyping(false);
}, 500);
return () => clearTimeout(timeout);
}, [message]);
useEffect(() => {
if (hiddenInputRef.current) {
setInputTextWidth(hiddenInputRef.current.clientWidth);
}
}, [hiddenInputRef, message]);
useEffect(() => {
if (inputRef.current) {
setInputWidth(inputRef.current.clientWidth);
}
}, [hiddenInputRef, message, windowSize.width]);
return (
<div
className="flex flex-col gap-2 border-t border-t-gray-800"
style={{ height: height }}
>
<div className="flex flex-row pt-3 gap-2 items-center relative">
<div
className={`w-2 h-4 bg-${inputHasFocus ? accentColor : "gray"}-${
inputHasFocus ? 500 : 800
} ${inputHasFocus ? "shadow-" + accentColor : ""} absolute left-2 ${
!isTyping && inputHasFocus ? "cursor-animation" : ""
}`}
style={{
transform:
"translateX(" +
(message.length > 0
? Math.min(inputTextWidth, inputWidth - 20) - 4
: 0) +
"px)",
}}
></div>
<input
ref={inputRef}
className={`w-full text-xs caret-transparent bg-transparent opacity-25 text-gray-300 p-2 pr-6 rounded-sm focus:opacity-100 focus:outline-none focus:border-${accentColor}-700 focus:ring-1 focus:ring-${accentColor}-700`}
style={{
paddingLeft: message.length > 0 ? "12px" : "24px",
caretShape: "block",
}}
placeholder={placeholder}
value={message}
onChange={(e) => {
setMessage(e.target.value);
}}
onFocus={() => {
setInputHasFocus(true);
}}
onBlur={() => {
setInputHasFocus(false);
}}
onKeyDown={(e) => {
if (e.key === "Enter") {
handleSend();
}
}}
></input>
<span
ref={hiddenInputRef}
className="absolute top-0 left-0 text-xs pl-3 text-amber-500 pointer-events-none opacity-0"
>
{message.replaceAll(" ", "\u00a0")}
</span>
<button
onClick={handleSend}
className={`text-xs uppercase text-${accentColor}-500 hover:bg-${accentColor}-950 p-2 rounded-md opacity-${
message.length > 0 ? 100 : 25
} pointer-events-${message.length > 0 ? "auto" : "none"}`}
>
Send
</button>
</div>
</div>
);
};

View File

@@ -0,0 +1,56 @@
import { useEffect, useRef, useState } from "react";
import { ChatMessage } from "@/components/chat/ChatMessage";
import { ChatMessageInput } from "@/components/chat/ChatMessageInput";
const inputHeight = 48;
export type ChatMessageType = {
name: string;
message: string;
isSelf: boolean;
};
type ChatTileProps = {
messages: ChatMessageType[];
accentColor: string;
onSend: (message: string) => void;
};
export const ChatTile = ({ messages, accentColor, onSend }: ChatTileProps) => {
const containerRef = useRef<HTMLDivElement>(null);
useEffect(() => {
if (containerRef.current) {
containerRef.current.scrollTop = containerRef.current.scrollHeight;
}
}, [containerRef, messages]);
return (
<div className="flex flex-col gap-4 w-full h-full">
<div
ref={containerRef}
className="overflow-y-auto"
style={{
height: `calc(100% - ${inputHeight}px)`,
}}
>
<div className="flex flex-col min-h-full justify-end gap-6">
{messages.map((message, index) => (
<ChatMessage
key={index}
name={message.name}
message={message.message}
isSelf={message.isSelf}
accentColor={accentColor}
/>
))}
</div>
</div>
<ChatMessageInput
height={inputHeight}
placeholder="Type a message"
accentColor={accentColor}
onSend={onSend}
/>
</div>
);
};

View File

@@ -0,0 +1,49 @@
import { useState } from "react";
type ColorPickerProps = {
colors: string[];
selectedColor: string;
onSelect: (color: string) => void;
};
export const ColorPicker = ({
colors,
selectedColor,
onSelect,
}: ColorPickerProps) => {
const [isHovering, setIsHovering] = useState(false);
const onMouseEnter = () => {
setIsHovering(true);
};
const onMouseLeave = () => {
setIsHovering(false);
};
return (
<div
className="flex flex-row gap-1 py-2 flex-wrap"
onMouseEnter={onMouseEnter}
onMouseLeave={onMouseLeave}
>
{colors.map((color) => {
const isSelected = color === selectedColor;
const saturation = !isHovering && !isSelected ? "saturate-[0.25]" : "";
const borderColor = isSelected
? `border border-${color}-800`
: "border-transparent";
const opacity = isSelected ? `opacity-100` : "opacity-20";
return (
<div
key={color}
className={`${saturation} rounded-md p-1 border-2 ${borderColor} cursor-pointer hover:opacity-100 transition transition-all duration-200 ${opacity} hover:scale-[1.05]`}
onClick={() => {
onSelect(color);
}}
>
<div className={`w-5 h-5 bg-${color}-500 rounded-sm`}></div>
</div>
);
})}
</div>
);
};

View File

@@ -0,0 +1,26 @@
import { useRef } from "react";
import { AgentMultibandAudioVisualizer } from "../visualization/AgentMultibandAudioVisualizer";
type AudioInputTileProps = {
frequencies: Float32Array[];
};
export const AudioInputTile = ({ frequencies }: AudioInputTileProps) => {
return (
<div
className={`flex flex-row gap-2 h-[100px] items-center w-full justify-center border rounded-sm border-gray-800 bg-gray-900`}
>
<AgentMultibandAudioVisualizer
state="speaking"
barWidth={4}
minBarHeight={2}
maxBarHeight={50}
accentColor={"gray"}
accentShade={400}
frequencies={frequencies}
borderRadius={2}
gap={4}
/>
</div>
);
};

View File

@@ -0,0 +1,40 @@
import { ReactNode } from "react";
import { PlaygroundDeviceSelector } from "@/components/playground/PlaygroundDeviceSelector";
import { TrackToggle } from "@livekit/components-react";
import { Track } from "livekit-client";
type ConfigurationPanelItemProps = {
title: string;
children?: ReactNode;
deviceSelectorKind?: MediaDeviceKind;
};
export const ConfigurationPanelItem: React.FC<ConfigurationPanelItemProps> = ({
children,
title,
deviceSelectorKind,
}) => {
return (
<div className="w-full text-gray-300 py-4 border-b border-b-gray-800 relative">
<div className="flex flex-row justify-between items-center px-4 text-xs uppercase tracking-wider">
<h3>{title}</h3>
{deviceSelectorKind && (
<span className="flex flex-row gap-2">
<TrackToggle
className="px-2 py-1 bg-gray-900 text-gray-300 border border-gray-800 rounded-sm hover:bg-gray-800"
source={
deviceSelectorKind === "audioinput"
? Track.Source.Microphone
: Track.Source.Camera
}
/>
<PlaygroundDeviceSelector kind={deviceSelectorKind} />
</span>
)}
</div>
<div className="px-4 py-2 text-xs text-gray-500 leading-normal">
{children}
</div>
</div>
);
};

View File

@@ -0,0 +1,22 @@
import { ReactNode } from "react";
type NameValueRowProps = {
name: string;
value?: ReactNode;
valueColor?: string;
};
export const NameValueRow: React.FC<NameValueRowProps> = ({
name,
value,
valueColor = "gray-300",
}) => {
return (
<div className="flex flex-row w-full items-baseline text-sm">
<div className="grow shrink-0 text-gray-500">{name}</div>
<div className={`text-xs uppercase shrink text-${valueColor} text-right`}>
{value}
</div>
</div>
);
};

View File

@@ -0,0 +1,487 @@
"use client";
import {
VideoTrack,
useConnectionState,
useDataChannel,
useLocalParticipant,
useRemoteParticipants,
useRoomContext,
useTracks,
} from "@livekit/components-react";
import {
ConnectionState,
DataPacket_Kind,
LocalParticipant,
Track,
} from "livekit-client";
import { ColorPicker } from "@/components/colorPicker/ColorPicker";
import { ConfigurationPanelItem } from "@/components/config/ConfigurationPanelItem";
import { LoadingSVG } from "@/components/button/LoadingSVG";
import { NameValueRow } from "@/components/config/NameValueRow";
import { PlaygroundHeader } from "@/components/playground/PlaygroundHeader";
import {
PlaygroundTab,
PlaygroundTabbedTile,
PlaygroundTile,
} from "@/components/playground/PlaygroundTile";
import { ReactNode, useCallback, useEffect, useMemo, useState } from "react";
import { useMultibandTrackVolume } from "@/hooks/useTrackVolume";
import { QRCodeSVG } from "qrcode.react";
import { AudioInputTile } from "@/components/config/AudioInputTile";
import { ChatMessageType, ChatTile } from "@/components/chat/ChatTile";
import { AgentMultibandAudioVisualizer } from "@/components/visualization/AgentMultibandAudioVisualizer";
export enum PlaygroundOutputs {
Video,
Audio,
Chat,
}
export interface PlaygroundProps {
logo?: ReactNode;
title?: string;
githubLink?: string;
description?: ReactNode;
themeColors: string[];
defaultColor: string;
outputs?: PlaygroundOutputs[];
showQR?: boolean;
onConnect: (connect: boolean, opts?: { token: string; url: string }) => void;
metadata?: { name: string; value: string }[];
}
const headerHeight = 56;
export default function Playground({
logo,
title,
githubLink,
description,
outputs,
showQR,
themeColors,
defaultColor,
onConnect,
metadata,
}: PlaygroundProps) {
const [agentState, setAgentState] = useState<
"listening" | "speaking" | "thinking" | "offline"
>("offline");
const [userState, setUserState] = useState<"silent" | "speaking">("silent");
const [themeColor, setThemeColor] = useState(defaultColor);
const [messages, setMessages] = useState<ChatMessageType[]>([]);
const localParticipant = useLocalParticipant();
const roomContext = useRoomContext();
const visualizerState = useMemo(() => {
if (agentState === "thinking") {
return "thinking";
} else if (agentState === "speaking") {
return "talking";
}
return "idle";
}, [agentState]);
const roomState = useConnectionState();
const tracks = useTracks();
const agentAudioTrack = tracks.find(
(trackRef) =>
trackRef.publication.kind === Track.Kind.Audio &&
trackRef.participant.isAgent
);
const agentVideoTrack = tracks.find(
(trackRef) =>
trackRef.publication.kind === Track.Kind.Video &&
trackRef.participant.isAgent
);
const subscribedVolumes = useMultibandTrackVolume(
agentAudioTrack?.publication.track,
5
);
const localTracks = tracks.filter(
({ participant }) => participant instanceof LocalParticipant
);
const localVideoTrack = localTracks.find(
({ source }) => source === Track.Source.Camera
);
const localMicTrack = localTracks.find(
({ source }) => source === Track.Source.Microphone
);
const localMultibandVolume = useMultibandTrackVolume(
localMicTrack?.publication.track,
20
);
const isAgentConnected = !!useRemoteParticipants().find((p) => p.isAgent);
useEffect(() => {
if (isAgentConnected && agentState === "offline") {
setAgentState("listening");
} else if (!isAgentConnected) {
setAgentState("offline");
}
}, [isAgentConnected, agentState]);
const onDataReceived = useCallback(
(msg: any) => {
const decoded = JSON.parse(new TextDecoder("utf-8").decode(msg.payload));
if (decoded.type === "state") {
const { agent_state, user_state } = decoded;
setAgentState(agent_state);
setUserState(user_state);
} else if (decoded.type === "transcription") {
setMessages([
...messages,
{ name: "You", message: decoded.text, isSelf: true },
]);
} else if (decoded.type === "agent_chat_message") {
setMessages([
...messages,
{ name: "Agent", message: decoded.text, isSelf: false },
]);
}
console.log("data received", decoded, msg.from);
},
[messages]
);
useDataChannel(onDataReceived);
const videoTileContent = useMemo(() => {
return (
<div className="flex flex-col w-full grow text-gray-950 bg-black rounded-sm border border-gray-800 relative">
{agentVideoTrack ? (
<VideoTrack
trackRef={agentVideoTrack}
className="absolute top-1/2 -translate-y-1/2 object-cover object-position-center w-full h-full"
/>
) : (
<div className="flex flex-col items-center justify-center gap-2 text-gray-700 text-center h-full w-full">
<LoadingSVG />
Waiting for video track
</div>
)}
</div>
);
}, [agentVideoTrack]);
const audioTileContent = useMemo(() => {
return (
<div className="flex items-center justify-center w-full">
{agentAudioTrack ? (
<AgentMultibandAudioVisualizer
state={agentState}
barWidth={30}
minBarHeight={30}
maxBarHeight={150}
accentColor={themeColor}
accentShade={500}
frequencies={subscribedVolumes}
borderRadius={12}
gap={16}
/>
) : (
<div className="flex flex-col items-center gap-2 text-gray-700 text-center w-full">
<LoadingSVG />
Waiting for audio track
</div>
)}
</div>
);
}, [agentAudioTrack, subscribedVolumes, themeColor, agentState]);
const chatTileContent = useMemo(() => {
return (
<ChatTile
messages={messages}
accentColor={themeColor}
onSend={(message) => {
if (roomContext.state === ConnectionState.Disconnected) {
return;
}
setMessages([
...messages,
{ name: "You", message: message, isSelf: true },
]);
const data = {
type: "user_chat_message",
text: message,
};
const encoder = new TextEncoder();
localParticipant.localParticipant?.publishData(
encoder.encode(JSON.stringify(data)),
DataPacket_Kind.RELIABLE
);
}}
/>
);
}, [
localParticipant.localParticipant,
messages,
roomContext.state,
themeColor,
]);
const settingsTileContent = useMemo(() => {
return (
<div className="flex flex-col gap-4 h-full w-full items-start overflow-y-auto">
{description && (
<ConfigurationPanelItem title="Description">
{description}
</ConfigurationPanelItem>
)}
<ConfigurationPanelItem title="Settings">
<div className="flex flex-col gap-2">
<NameValueRow
name="Agent URL"
value={process.env.NEXT_PUBLIC_LIVEKIT_URL}
/>
{metadata?.map((data, index) => (
<NameValueRow
key={data.name + index}
name={data.name}
value={data.value}
/>
))}
</div>
</ConfigurationPanelItem>
<ConfigurationPanelItem title="Status">
<div className="flex flex-col gap-2">
<NameValueRow
name="Room connected"
value={
roomState === ConnectionState.Connecting ? (
<LoadingSVG diameter={16} strokeWidth={2} />
) : (
roomState
)
}
valueColor={
roomState === ConnectionState.Connected
? `${themeColor}-500`
: "gray-500"
}
/>
<NameValueRow
name="Agent connected"
value={
isAgentConnected ? (
"true"
) : roomState === ConnectionState.Connected ? (
<LoadingSVG diameter={12} strokeWidth={2} />
) : (
"false"
)
}
valueColor={isAgentConnected ? `${themeColor}-500` : "gray-500"}
/>
<NameValueRow
name="Agent status"
value={
agentState !== "offline" && agentState !== "speaking" ? (
<div className="flex gap-2 items-center">
<LoadingSVG diameter={12} strokeWidth={2} />
{agentState}
</div>
) : (
agentState
)
}
valueColor={
agentState === "speaking" ? `${themeColor}-500` : "gray-500"
}
/>
<NameValueRow
name="User status"
value={userState}
valueColor={
userState === "silent" ? "gray-500" : `${themeColor}-500`
}
/>
</div>
</ConfigurationPanelItem>
{localVideoTrack && (
<ConfigurationPanelItem
title="Camera"
deviceSelectorKind="videoinput"
>
<div className="relative">
<VideoTrack
className="rounded-sm border border-gray-800 opacity-70 w-full"
trackRef={localVideoTrack}
/>
</div>
</ConfigurationPanelItem>
)}
{localMicTrack && (
<ConfigurationPanelItem
title="Microphone"
deviceSelectorKind="audioinput"
>
<AudioInputTile frequencies={localMultibandVolume} />
</ConfigurationPanelItem>
)}
<div className="w-full">
<ConfigurationPanelItem title="Color">
<ColorPicker
colors={themeColors}
selectedColor={themeColor}
onSelect={(color) => {
setThemeColor(color);
}}
/>
</ConfigurationPanelItem>
</div>
{showQR && (
<div className="w-full">
<ConfigurationPanelItem title="QR Code">
<QRCodeSVG value={window.location.href} width="128" />
</ConfigurationPanelItem>
</div>
)}
</div>
);
}, [
agentState,
description,
isAgentConnected,
localMicTrack,
localMultibandVolume,
localVideoTrack,
metadata,
roomState,
themeColor,
themeColors,
userState,
showQR,
]);
let mobileTabs: PlaygroundTab[] = [];
if (outputs?.includes(PlaygroundOutputs.Video)) {
mobileTabs.push({
title: "Video",
content: (
<PlaygroundTile
className="w-full h-full grow"
childrenClassName="justify-center"
>
{videoTileContent}
</PlaygroundTile>
),
});
}
if (outputs?.includes(PlaygroundOutputs.Audio)) {
mobileTabs.push({
title: "Audio",
content: (
<PlaygroundTile
className="w-full h-full grow"
childrenClassName="justify-center"
>
{audioTileContent}
</PlaygroundTile>
),
});
}
if (outputs?.includes(PlaygroundOutputs.Chat)) {
mobileTabs.push({
title: "Chat",
content: chatTileContent,
});
}
mobileTabs.push({
title: "Settings",
content: (
<PlaygroundTile
padding={false}
backgroundColor="gray-950"
className="h-full w-full basis-1/4 items-start overflow-y-auto flex"
childrenClassName="h-full grow items-start"
>
{settingsTileContent}
</PlaygroundTile>
),
});
return (
<>
<PlaygroundHeader
title={title}
logo={logo}
githubLink={githubLink}
height={headerHeight}
accentColor={themeColor}
connectionState={roomState}
onConnectClicked={() =>
onConnect(roomState === ConnectionState.Disconnected)
}
/>
<div
className={`flex gap-4 py-4 grow w-full selection:bg-${themeColor}-900`}
style={{ height: `calc(100% - ${headerHeight}px)` }}
>
<div className="flex flex-col grow basis-1/2 gap-4 h-full lg:hidden">
<PlaygroundTabbedTile
className="h-full"
tabs={mobileTabs}
initialTab={mobileTabs.length - 1}
/>
</div>
<div
className={`flex-col grow basis-1/2 gap-4 h-full hidden lg:${
!outputs?.includes(PlaygroundOutputs.Audio) &&
!outputs?.includes(PlaygroundOutputs.Video)
? "hidden"
: "flex"
}`}
>
{outputs?.includes(PlaygroundOutputs.Video) && (
<PlaygroundTile
title="Video"
className="w-full h-full grow"
childrenClassName="justify-center"
>
{videoTileContent}
</PlaygroundTile>
)}
{outputs?.includes(PlaygroundOutputs.Audio) && (
<PlaygroundTile
title="Audio"
className="w-full h-full grow"
childrenClassName="justify-center"
>
{audioTileContent}
</PlaygroundTile>
)}
</div>
{outputs?.includes(PlaygroundOutputs.Chat) && (
<PlaygroundTile
title="Chat"
className="h-full grow basis-1/4 hidden lg:flex"
>
{chatTileContent}
</PlaygroundTile>
)}
<PlaygroundTile
padding={false}
backgroundColor="gray-950"
className="h-full w-full basis-1/4 items-start overflow-y-auto hidden max-w-[480px] lg:flex"
childrenClassName="h-full grow items-start"
>
{settingsTileContent}
</PlaygroundTile>
</div>
</>
);
}

View File

@@ -0,0 +1,94 @@
import { useMediaDeviceSelect } from "@livekit/components-react";
import { useEffect, useState } from "react";
type PlaygroundDeviceSelectorProps = {
kind: MediaDeviceKind;
};
export const PlaygroundDeviceSelector = ({
kind,
}: PlaygroundDeviceSelectorProps) => {
const [showMenu, setShowMenu] = useState(false);
const deviceSelect = useMediaDeviceSelect({ kind: kind });
const [selectedDeviceName, setSelectedDeviceName] = useState("");
useEffect(() => {
deviceSelect.devices.forEach((device) => {
if (device.deviceId === deviceSelect.activeDeviceId) {
setSelectedDeviceName(device.label);
}
});
}, [deviceSelect.activeDeviceId, deviceSelect.devices, selectedDeviceName]);
useEffect(() => {
const handleClickOutside = (event: MouseEvent) => {
if (showMenu) {
setShowMenu(false);
}
};
document.addEventListener("click", handleClickOutside);
return () => {
document.removeEventListener("click", handleClickOutside);
};
}, [showMenu]);
return (
<div>
<button
className="flex gap-2 items-center px-2 py-1 bg-gray-900 text-gray-300 border border-gray-800 rounded-sm hover:bg-gray-800"
onClick={(e) => {
setShowMenu(!showMenu);
e.stopPropagation();
}}
>
<span className="max-w-[80px] overflow-ellipsis overflow-hidden whitespace-nowrap">
{selectedDeviceName}
</span>
<ChevronSVG />
</button>
<div
className="absolute right-4 top-12 bg-gray-800 text-gray-300 border border-gray-800 rounded-sm z-10"
style={{
display: showMenu ? "block" : "none",
}}
>
{deviceSelect.devices.map((device, index) => {
return (
<div
onClick={() => {
deviceSelect.setActiveMediaDevice(device.deviceId);
setShowMenu(false);
}}
className={`${
device.deviceId === deviceSelect.activeDeviceId
? "text-white"
: "text-gray-500"
} bg-gray-900 text-xs py-2 px-2 cursor-pointer hover:bg-gray-800 hover:text-white`}
key={index}
>
{device.label}
</div>
);
})}
</div>
</div>
);
};
const ChevronSVG = () => (
<svg
xmlns="http://www.w3.org/2000/svg"
width="16"
height="16"
viewBox="0 0 16 16"
fill="none"
>
<path
fillRule="evenodd"
clipRule="evenodd"
d="M3 5H5V7H3V5ZM7 9V7H5V9H7ZM9 9V11H7V9H9ZM11 7V9H9V7H11ZM11 7V5H13V7H11Z"
fill="currentColor"
fillOpacity="0.8"
/>
</svg>
);

View File

@@ -0,0 +1,122 @@
import { Button } from "@/components/button/Button";
import { ConnectionState } from "livekit-client";
import { LoadingSVG } from "@/components/button/LoadingSVG";
import { ReactNode } from "react";
type PlaygroundHeader = {
logo?: ReactNode;
title?: ReactNode;
githubLink?: string;
height: number;
accentColor: string;
connectionState: ConnectionState;
onConnectClicked: () => void;
};
export const PlaygroundHeader = ({
logo,
title,
githubLink,
accentColor,
height,
onConnectClicked,
connectionState,
}: PlaygroundHeader) => {
return (
<div
className={`flex gap-4 pt-4 text-${accentColor}-500 justify-between items-center shrink-0`}
style={{
height: height + "px",
}}
>
<div className="flex items-center gap-3 basis-2/3">
<div className="flex lg:basis-1/2">
<a href="https://livekit.io">{logo ?? <LKLogo />}</a>
</div>
<div className="lg:basis-1/2 lg:text-center text-xs lg:text-base lg:font-semibold text-white">
{title}
</div>
</div>
<div className="flex basis-1/3 justify-end items-center gap-4">
{githubLink && (
<a
href={githubLink}
target="_blank"
className={`text-white hover:text-white/80`}
>
<GithubSVG />
</a>
)}
<Button
accentColor={
connectionState === ConnectionState.Connected ? "red" : accentColor
}
disabled={connectionState === ConnectionState.Connecting}
onClick={() => {
onConnectClicked();
}}
>
{connectionState === ConnectionState.Connecting ? (
<LoadingSVG />
) : connectionState === ConnectionState.Connected ? (
"Disconnect"
) : (
"Connect"
)}
</Button>
</div>
</div>
);
};
const LKLogo = () => (
<svg
width="28"
height="28"
viewBox="0 0 32 32"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<g clipPath="url(#clip0_101_119699)">
<path
d="M19.2006 12.7998H12.7996V19.2008H19.2006V12.7998Z"
fill="currentColor"
/>
<path
d="M25.6014 6.40137H19.2004V12.8024H25.6014V6.40137Z"
fill="currentColor"
/>
<path
d="M25.6014 19.2002H19.2004V25.6012H25.6014V19.2002Z"
fill="currentColor"
/>
<path d="M32 0H25.599V6.401H32V0Z" fill="currentColor" />
<path d="M32 25.5986H25.599V31.9996H32V25.5986Z" fill="currentColor" />
<path
d="M6.401 25.599V19.2005V12.7995V6.401V0H0V6.401V12.7995V19.2005V25.599V32H6.401H12.7995H19.2005V25.599H12.7995H6.401Z"
fill="white"
/>
</g>
<defs>
<clipPath id="clip0_101_119699">
<rect width="32" height="32" fill="white" />
</clipPath>
</defs>
</svg>
);
const GithubSVG = () => (
<svg
width="24"
height="24"
viewBox="0 0 98 96"
xmlns="http://www.w3.org/2000/svg"
>
<path
fillRule="evenodd"
clipRule="evenodd"
d="M48.854 0C21.839 0 0 22 0 49.217c0 21.756 13.993 40.172 33.405 46.69 2.427.49 3.316-1.059 3.316-2.362 0-1.141-.08-5.052-.08-9.127-13.59 2.934-16.42-5.867-16.42-5.867-2.184-5.704-5.42-7.17-5.42-7.17-4.448-3.015.324-3.015.324-3.015 4.934.326 7.523 5.052 7.523 5.052 4.367 7.496 11.404 5.378 14.235 4.074.404-3.178 1.699-5.378 3.074-6.6-10.839-1.141-22.243-5.378-22.243-24.283 0-5.378 1.94-9.778 5.014-13.2-.485-1.222-2.184-6.275.486-13.038 0 0 4.125-1.304 13.426 5.052a46.97 46.97 0 0 1 12.214-1.63c4.125 0 8.33.571 12.213 1.63 9.302-6.356 13.427-5.052 13.427-5.052 2.67 6.763.97 11.816.485 13.038 3.155 3.422 5.015 7.822 5.015 13.2 0 18.905-11.404 23.06-22.324 24.283 1.78 1.548 3.316 4.481 3.316 9.126 0 6.6-.08 11.897-.08 13.526 0 1.304.89 2.853 3.316 2.364 19.412-6.52 33.405-24.935 33.405-46.691C97.707 22 75.788 0 48.854 0z"
fill="currentColor"
/>
</svg>
);

View File

@@ -0,0 +1,104 @@
import { ReactNode, useState } from "react";
const titleHeight = 32;
type PlaygroundTileProps = {
title?: string;
children?: ReactNode;
className?: string;
childrenClassName?: string;
padding?: boolean;
backgroundColor?: string;
};
export type PlaygroundTab = {
title: string;
content: ReactNode;
};
export type PlaygroundTabbedTileProps = {
tabs: PlaygroundTab[];
initialTab?: number;
} & PlaygroundTileProps;
export const PlaygroundTile: React.FC<PlaygroundTileProps> = ({
children,
title,
className,
childrenClassName,
padding = true,
backgroundColor = "transparent",
}) => {
const contentPadding = padding ? 4 : 0;
return (
<div
className={`flex flex-col border rounded-sm border-gray-800 text-gray-500 bg-${backgroundColor} ${className}`}
>
{title && (
<div
className="flex items-center justify-center text-xs uppercase py-2 border-b border-b-gray-800 tracking-wider"
style={{
height: `${titleHeight}px`,
}}
>
<h2>{title}</h2>
</div>
)}
<div
className={`flex flex-col items-center grow w-full ${childrenClassName}`}
style={{
height: `calc(100% - ${title ? titleHeight + "px" : "0px"})`,
padding: `${contentPadding * 4}px`,
}}
>
{children}
</div>
</div>
);
};
export const PlaygroundTabbedTile: React.FC<PlaygroundTabbedTileProps> = ({
tabs,
initialTab = 0,
className,
childrenClassName,
backgroundColor = "transparent",
}) => {
const contentPadding = 4;
const [activeTab, setActiveTab] = useState(initialTab);
return (
<div
className={`flex flex-col h-full border rounded-sm border-gray-800 text-gray-500 bg-${backgroundColor} ${className}`}
>
<div
className="flex items-center justify-start text-xs uppercase border-b border-b-gray-800 tracking-wider"
style={{
height: `${titleHeight}px`,
}}
>
{tabs.map((tab, index) => (
<button
key={index}
className={`px-4 py-2 rounded-sm hover:bg-gray-800 hover:text-gray-300 border-r border-r-gray-800 ${
index === activeTab
? `bg-gray-900 text-gray-300`
: `bg-transparent text-gray-500`
}`}
onClick={() => setActiveTab(index)}
>
{tab.title}
</button>
))}
</div>
<div
className={`w-full ${childrenClassName}`}
style={{
height: `calc(100% - ${titleHeight}px)`,
padding: `${contentPadding * 4}px`,
}}
>
{tabs[activeTab].content}
</div>
</div>
);
};

View File

@@ -0,0 +1,37 @@
export type ToastType = "error" | "success" | "info";
export type ToastProps = {
message: string;
type: ToastType;
onDismiss: () => void;
};
export const PlaygroundToast = ({ message, type, onDismiss }: ToastProps) => {
const color =
type === "error" ? "red" : type === "success" ? "green" : "amber";
return (
<div
className={`absolute text-sm break-words px-4 pr-12 py-2 bg-${color}-950 rounded-sm border border-${color}-800 text-${color}-400 top-4 left-4 right-4`}
>
<button
className={`absolute right-2 border border-transparent rounded-md px-2 hover:bg-${color}-900 hover:text-${color}-300`}
onClick={onDismiss}
>
<svg
xmlns="http://www.w3.org/2000/svg"
width="16"
height="16"
viewBox="0 0 24 24"
fill="none"
>
<path
fill-rule="evenodd"
clip-rule="evenodd"
d="M5.29289 5.29289C5.68342 4.90237 6.31658 4.90237 6.70711 5.29289L12 10.5858L17.2929 5.29289C17.6834 4.90237 18.3166 4.90237 18.7071 5.29289C19.0976 5.68342 19.0976 6.31658 18.7071 6.70711L13.4142 12L18.7071 17.2929C19.0976 17.6834 19.0976 18.3166 18.7071 18.7071C18.3166 19.0976 17.6834 19.0976 17.2929 18.7071L12 13.4142L6.70711 18.7071C6.31658 19.0976 5.68342 19.0976 5.29289 18.7071C4.90237 18.3166 4.90237 17.6834 5.29289 17.2929L10.5858 12L5.29289 6.70711C4.90237 6.31658 4.90237 5.68342 5.29289 5.29289Z"
fill="currentColor"
/>
</svg>
</button>
{message}
</div>
);
};

View File

@@ -0,0 +1,113 @@
import { useEffect, useState } from "react";
type AgentMultibandAudioVisualizerProps = {
state: "listening" | "speaking" | "thinking" | "offline";
barWidth: number;
minBarHeight: number;
maxBarHeight: number;
accentColor: string;
accentShade?: number;
frequencies: Float32Array[];
borderRadius: number;
gap: number;
};
export const AgentMultibandAudioVisualizer = ({
state,
barWidth,
minBarHeight,
maxBarHeight,
accentColor,
accentShade,
frequencies,
borderRadius,
gap,
}: AgentMultibandAudioVisualizerProps) => {
const summedFrequencies = frequencies.map((bandFrequencies) => {
const sum = bandFrequencies.reduce((a, b) => a + b, 0);
return Math.sqrt(sum / bandFrequencies.length);
});
const [thinkingIndex, setThinkingIndex] = useState(
Math.floor(summedFrequencies.length / 2)
);
const [thinkingDirection, setThinkingDirection] = useState<"left" | "right">(
"right"
);
useEffect(() => {
if (state !== "thinking") {
setThinkingIndex(Math.floor(summedFrequencies.length / 2));
return;
}
const timeout = setTimeout(() => {
if (thinkingDirection === "right") {
if (thinkingIndex === summedFrequencies.length - 1) {
setThinkingDirection("left");
setThinkingIndex((prev) => prev - 1);
} else {
setThinkingIndex((prev) => prev + 1);
}
} else {
if (thinkingIndex === 0) {
setThinkingDirection("right");
setThinkingIndex((prev) => prev + 1);
} else {
setThinkingIndex((prev) => prev - 1);
}
}
}, 200);
return () => clearTimeout(timeout);
}, [state, summedFrequencies.length, thinkingDirection, thinkingIndex]);
return (
<div
className={`flex flex-row items-center`}
style={{
gap: gap + "px",
}}
>
{summedFrequencies.map((frequency, index) => {
const isCenter = index === Math.floor(summedFrequencies.length / 2);
let color = accentColor;
let shadow = `shadow-lg-${accentColor}`;
let transform;
if (state === "listening") {
color = isCenter ? `${accentColor}-${accentShade}` : "gray-950";
shadow = !isCenter ? "" : shadow;
transform = !isCenter ? "scale(1.0)" : "scale(1.2)";
} else if (state === "speaking") {
color = `${accentColor}${accentShade ? "-" + accentShade : ""}`;
} else if (state === "thinking") {
color =
index === thinkingIndex
? `${accentColor}-${accentShade}`
: "gray-950";
shadow = "";
transform = thinkingIndex !== index ? "scale(1)" : "scale(1.1)";
}
return (
<div
className={`bg-${color} ${shadow} ${
isCenter && state === "listening" ? "animate-pulse" : ""
}`}
key={"frequency-" + index}
style={{
height:
minBarHeight + frequency * (maxBarHeight - minBarHeight) + "px",
borderRadius: borderRadius + "px",
width: barWidth + "px",
transition:
"background-color 0.35s ease-out, transform 0.25s ease-out",
transform: transform,
}}
></div>
);
})}
</div>
);
};

View File

@@ -0,0 +1,57 @@
import jsYaml from "js-yaml";
import { useEffect, useState } from "react";
const APP_CONFIG = process.env.NEXT_PUBLIC_APP_CONFIG;
export type AppConfig = {
title: string;
description: string;
github_link?: string;
theme_color?: string;
outputs: {
audio: boolean;
video: boolean;
chat: boolean;
};
inputs: {
mic: boolean;
camera: boolean;
};
show_qr?: boolean;
};
// Fallback if NEXT_PUBLIC_APP_CONFIG is not set
const defaultConfig: AppConfig = {
title: "Agent Playground",
description: "A playground for testing LiveKit agents",
theme_color: "cyan",
outputs: {
audio: true,
video: true,
chat: true,
},
inputs: {
mic: true,
camera: true,
},
show_qr: false,
};
export const useAppConfig = (): AppConfig => {
const [config, setConfig] = useState<any>(null);
useEffect(() => {
try {
if (APP_CONFIG) {
const parsedConfig = jsYaml.load(APP_CONFIG);
setConfig(parsedConfig);
console.log("parsedConfig:", parsedConfig);
} else {
setConfig(defaultConfig);
}
} catch (error) {
console.error("Error parsing NEXT_PUBLIC_APP_CONFIG:", error);
}
}, []);
return config;
};

View File

@@ -0,0 +1,112 @@
import { Track } from "livekit-client";
import { useEffect, useState } from "react";
export const useTrackVolume = (track?: Track) => {
const [volume, setVolume] = useState(0);
useEffect(() => {
if (!track || !track.mediaStream) {
return;
}
const ctx = new AudioContext();
const source = ctx.createMediaStreamSource(track.mediaStream);
const analyser = ctx.createAnalyser();
analyser.fftSize = 32;
analyser.smoothingTimeConstant = 0;
source.connect(analyser);
const bufferLength = analyser.frequencyBinCount;
const dataArray = new Uint8Array(bufferLength);
const updateVolume = () => {
analyser.getByteFrequencyData(dataArray);
let sum = 0;
for (let i = 0; i < dataArray.length; i++) {
const a = dataArray[i];
sum += a * a;
}
setVolume(Math.sqrt(sum / dataArray.length) / 255);
};
const interval = setInterval(updateVolume, 1000 / 30);
return () => {
source.disconnect();
clearInterval(interval);
};
}, [track, track?.mediaStream]);
return volume;
};
const normalizeFrequencies = (frequencies: Float32Array) => {
const normalizeDb = (value: number) => {
const minDb = -100;
const maxDb = -10;
let db = 1 - (Math.max(minDb, Math.min(maxDb, value)) * -1) / 100;
db = Math.sqrt(db);
return db;
};
// Normalize all frequency values
return frequencies.map((value) => {
if (value === -Infinity) {
return 0;
}
return normalizeDb(value);
});
};
export const useMultibandTrackVolume = (
track?: Track,
bands: number = 5,
loPass: number = 100,
hiPass: number = 600
) => {
const [frequencyBands, setFrequencyBands] = useState<Float32Array[]>([]);
useEffect(() => {
if (!track || !track.mediaStream) {
return;
}
const ctx = new AudioContext();
const source = ctx.createMediaStreamSource(track.mediaStream);
const analyser = ctx.createAnalyser();
analyser.fftSize = 2048;
source.connect(analyser);
const bufferLength = analyser.frequencyBinCount;
const dataArray = new Float32Array(bufferLength);
const updateVolume = () => {
analyser.getFloatFrequencyData(dataArray);
let frequencies: Float32Array = new Float32Array(dataArray.length);
for (let i = 0; i < dataArray.length; i++) {
frequencies[i] = dataArray[i];
}
frequencies = frequencies.slice(loPass, hiPass);
const normalizedFrequencies = normalizeFrequencies(frequencies);
const chunkSize = Math.ceil(normalizedFrequencies.length / bands);
const chunks: Float32Array[] = [];
for (let i = 0; i < bands; i++) {
chunks.push(
normalizedFrequencies.slice(i * chunkSize, (i + 1) * chunkSize)
);
}
setFrequencyBands(chunks);
};
const interval = setInterval(updateVolume, 10);
return () => {
source.disconnect();
clearInterval(interval);
};
}, [track, track?.mediaStream, loPass, hiPass, bands]);
return frequencyBands;
};

View File

@@ -0,0 +1,27 @@
import { useEffect, useState } from "react";
export const useWindowResize = () => {
const [size, setSize] = useState({
width: 0,
height: 0,
});
useEffect(() => {
const handleResize = () => {
setSize({
width: window.innerWidth,
height: window.innerHeight,
});
};
handleResize();
window.addEventListener("resize", handleResize);
return () => {
window.removeEventListener("resize", handleResize);
};
}, []);
return size;
};

16
src/lib/types.ts Normal file
View File

@@ -0,0 +1,16 @@
import { LocalAudioTrack, LocalVideoTrack } from "livekit-client";
export interface SessionProps {
roomName: string;
identity: string;
audioTrack?: LocalAudioTrack;
videoTrack?: LocalVideoTrack;
region?: string;
turnServer?: RTCIceServer;
forceRelay?: boolean;
}
export interface TokenResult {
identity: string;
accessToken: string;
}

12
src/lib/util.ts Normal file
View File

@@ -0,0 +1,12 @@
export function generateRandomAlphanumeric(length: number): string {
const characters =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
let result = "";
const charactersLength = characters.length;
for (let i = 0; i < length; i++) {
result += characters.charAt(Math.floor(Math.random() * charactersLength));
}
return result;
}

6
src/pages/_app.tsx Normal file
View File

@@ -0,0 +1,6 @@
import "@/styles/globals.css";
import type { AppProps } from "next/app";
export default function App({ Component, pageProps }: AppProps) {
return <Component {...pageProps} />;
}

13
src/pages/_document.tsx Normal file
View File

@@ -0,0 +1,13 @@
import { Html, Head, Main, NextScript } from "next/document";
export default function Document() {
return (
<Html lang="en">
<Head />
<body>
<Main />
<NextScript />
</body>
</Html>
);
}

78
src/pages/api/token.ts Normal file
View File

@@ -0,0 +1,78 @@
import { NextApiRequest, NextApiResponse } from "next";
import { AccessToken } from "livekit-server-sdk";
import type { AccessTokenOptions, VideoGrant } from "livekit-server-sdk";
import { TokenResult } from "../../lib/types";
const apiKey = process.env.LIVEKIT_API_KEY;
const apiSecret = process.env.LIVEKIT_API_SECRET;
const createToken = (userInfo: AccessTokenOptions, grant: VideoGrant) => {
const at = new AccessToken(apiKey, apiSecret, userInfo);
at.addGrant(grant);
return at.toJwt();
};
const roomPattern = /\w{4}\-\w{4}/;
export default async function handleToken(
req: NextApiRequest,
res: NextApiResponse
) {
try {
const { roomName, identity, name, metadata } = req.query;
if (typeof identity !== "string" || typeof roomName !== "string") {
res.statusMessage =
"identity and roomName have to be specified in the request";
res.status(403).end();
return;
}
if (!apiKey || !apiSecret) {
res.statusMessage = "Environment variables aren't set up correctly";
res.status(500).end();
return;
}
if (Array.isArray(name)) {
throw Error("provide max one name");
}
if (Array.isArray(metadata)) {
throw Error("provide max one metadata string");
}
// enforce room name to be xxxx-xxxx
// this is simple & naive way to prevent user from guessing room names
// please use your own authentication mechanisms in your own app
if (!roomName.match(roomPattern)) {
res.statusMessage = "Invalid roomName";
res.status(400).end();
return;
}
// if (!userSession.isAuthenticated) {
// res.status(403).end();
// return;
// }
const grant: VideoGrant = {
room: roomName,
roomJoin: true,
canPublish: true,
canPublishData: true,
canSubscribe: true,
};
const token = createToken({ identity, name, metadata }, grant);
const result: TokenResult = {
identity,
accessToken: token,
};
res.status(200).json(result);
} catch (e) {
res.statusMessage = (e as Error).message;
res.status(500).end();
}
}

150
src/pages/index.tsx Normal file
View File

@@ -0,0 +1,150 @@
import Head from "next/head";
import { useCallback, useEffect, useMemo, useState } from "react";
import { Inter } from "next/font/google";
import { generateRandomAlphanumeric } from "@/lib/util";
import { motion, AnimatePresence } from "framer-motion";
import {
LiveKitRoom,
RoomAudioRenderer,
useToken,
} from "@livekit/components-react";
import Playground, {
PlaygroundOutputs,
} from "@/components/playground/Playground";
import { useAppConfig } from "@/hooks/useAppConfig";
import { PlaygroundConnect } from "@/components/PlaygroundConnect";
import { PlaygroundToast, ToastType } from "@/components/toast/PlaygroundToast";
const themeColors = [
"cyan",
"green",
"amber",
"blue",
"violet",
"rose",
"pink",
"teal",
];
const inter = Inter({ subsets: ["latin"] });
export default function Home() {
const [toastMessage, setToastMessage] = useState<{
message: string;
type: ToastType;
} | null>(null);
const [shouldConnect, setShouldConnect] = useState(false);
const [liveKitUrl, setLiveKitUrl] = useState(
process.env.NEXT_PUBLIC_LIVEKIT_URL
);
const [customToken, setCustomToken] = useState<string>();
const [roomName, setRoomName] = useState(
[generateRandomAlphanumeric(4), generateRandomAlphanumeric(4)].join("-")
);
const tokenOptions = useMemo(() => {
return {
userInfo: { identity: generateRandomAlphanumeric(16) },
};
}, []);
// set a new room name each time the user disconnects so that a new token gets fetched behind the scenes for a different room
useEffect(() => {
if (shouldConnect === false) {
setRoomName(
[generateRandomAlphanumeric(4), generateRandomAlphanumeric(4)].join("-")
);
}
}, [shouldConnect]);
const token = useToken("/api/token", roomName, tokenOptions);
const appConfig = useAppConfig();
const outputs = [
appConfig?.outputs.audio && PlaygroundOutputs.Audio,
appConfig?.outputs.video && PlaygroundOutputs.Video,
appConfig?.outputs.chat && PlaygroundOutputs.Chat,
].filter((item) => typeof item !== "boolean") as PlaygroundOutputs[];
const handleConnect = useCallback(
(connect: boolean, opts?: { url: string; token: string }) => {
if (connect && opts) {
setLiveKitUrl(opts.url);
setCustomToken(opts.token);
}
setShouldConnect(connect);
},
[]
);
return (
<>
<Head>
<title>Agent Playground</title>
<meta name="description" content="Generated by create next app" />
<meta
name="viewport"
content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no"
/>
<meta name="apple-mobile-web-app-capable" content="yes" />
<meta name="apple-mobile-web-app-status-bar-style" content="black" />
<link rel="icon" href="/favicon.ico" />
</Head>
<main className="relative flex flex-col justify-center px-4 items-center h-full w-full bg-black repeating-square-background">
<AnimatePresence>
{toastMessage && (
<motion.div
className="left-0 right-0 top-0 absolute z-10"
initial={{ opacity: 0, translateY: -50 }}
animate={{ opacity: 1, translateY: 0 }}
exit={{ opacity: 0, translateY: -50 }}
>
<PlaygroundToast
message={toastMessage.message}
type={toastMessage.type}
onDismiss={() => {
setToastMessage(null);
}}
/>
</motion.div>
)}
</AnimatePresence>
{liveKitUrl ? (
<LiveKitRoom
className="flex flex-col h-full w-full"
serverUrl={liveKitUrl}
token={customToken ?? token}
audio={appConfig?.inputs.mic}
video={appConfig?.inputs.camera}
connect={shouldConnect}
onError={(e) => {
setToastMessage({ message: e.message, type: "error" });
console.error(e);
}}
>
<Playground
title={appConfig?.title}
githubLink={appConfig?.github_link}
outputs={outputs}
showQR={appConfig?.show_qr}
description={appConfig?.description}
themeColors={themeColors}
defaultColor={appConfig?.theme_color ?? "cyan"}
onConnect={handleConnect}
metadata={[{ name: "Room Name", value: roomName }]}
/>
<RoomAudioRenderer />
</LiveKitRoom>
) : (
<PlaygroundConnect
accentColor={themeColors[0]}
onConnectClicked={(url, token) => {
handleConnect(true, { url, token });
}}
/>
)}
</main>
</>
);
}

53
src/styles/globals.css Normal file
View File

@@ -0,0 +1,53 @@
@tailwind base;
@tailwind components;
@tailwind utilities;
body {
background: black;
}
#__next {
width: 100%;
height: 100dvh;
}
.repeating-square-background {
background-size: 18px 18px;
background-repeat: repeat;
background-image: url("data:image/svg+xml,%3Csvg width='18' height='18' xmlns='http://www.w3.org/2000/svg'%3E%3Crect x='0' y='0' width='2' height='2' fill='rgba(255, 255, 255, 0.03)' /%3E%3C/svg%3E");
}
.cursor-animation {
animation: fadeIn 0.5s ease-in-out alternate-reverse infinite;
}
::-webkit-scrollbar-track {
background: rgba(255, 255, 255, 0.05);
border-radius: 5px;
}
::-webkit-scrollbar-thumb {
background: rgba(255, 255, 255, 0.1);
border-radius: 5px;
}
::-webkit-scrollbar-thumb:hover {
background: #555; /* Even lighter grey thumb on hover */
}
::-webkit-scrollbar {
width: 10px;
border-radius: 5px;
}
@keyframes fadeIn {
0% {
opacity: 0;
}
100% {
opacity: 1;
}
}

77
tailwind.config.js Normal file
View File

@@ -0,0 +1,77 @@
/** @type {import('tailwindcss').Config} */
const colors = require('tailwindcss/colors')
const shades = ['50', '100', '200', '300', '400', '500', '600', '700', '800', '900', '950'];
const colorList = ['gray', 'green', 'cyan', 'amber', 'violet', 'blue', 'rose', 'pink', 'teal', "red"];
const uiElements = ['bg', 'selection:bg', 'border', 'text', 'hover:bg', 'hover:border', 'hover:text', 'ring', 'focus:ring'];
const customColors = {
cyan: colors.cyan,
green: colors.green,
amber: colors.amber,
violet: colors.violet,
blue: colors.blue,
rose: colors.rose,
pink: colors.pink,
teal: colors.teal,
red: colors.red,
}
let customShadows = {};
let shadowNames = [];
let textShadows = {};
let textShadowNames = [];
for (const [name, color] of Object.entries(customColors)) {
customShadows[`${name}`] = `0px 0px 10px ${color["500"]}`;
customShadows[`lg-${name}`] = `0px 0px 20px ${color["600"]}`;
textShadows[`${name}`] = `0px 0px 4px ${color["700"]}`;
textShadowNames.push(`drop-shadow-${name}`);
shadowNames.push(`shadow-${name}`);
shadowNames.push(`shadow-lg-${name}`);
shadowNames.push(`hover:shadow-${name}`);
}
console.log(customShadows, textShadows);
const safelist = [
'bg-black',
'bg-white',
'transparent',
...shadowNames,
...textShadowNames,
...shades.flatMap(shade => [
...colorList.flatMap(color => [
...uiElements.flatMap(element => [
`${element}-${color}-${shade}`,
]),
]),
]),
];
console.log("Safe list", safelist);
module.exports = {
content: [
"./src/**/*.{js,ts,jsx,tsx,mdx}",
],
theme: {
colors: {
transparent: 'transparent',
current: 'currentColor',
black: colors.black,
white: colors.white,
gray: colors.neutral,
...customColors
},
extend: {
dropShadow: {
...textShadows,
},
boxShadow: {
...customShadows,
}
}
},
plugins: [],
safelist,
};

23
tsconfig.json Normal file
View File

@@ -0,0 +1,23 @@
{
"compilerOptions": {
"target": "es5",
"lib": ["dom", "dom.iterable", "esnext"],
"allowJs": true,
"skipLibCheck": true,
"strict": true,
"noEmit": true,
"esModuleInterop": true,
"module": "esnext",
"moduleResolution": "bundler",
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": "preserve",
"incremental": true,
"baseUrl": ".",
"paths": {
"@/*": ["./src/*"]
}
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"],
"exclude": ["node_modules"]
}