Add attributes inspector and explicit dispatch support (#144)

This commit is contained in:
Ben Cherry
2025-05-29 15:26:04 -07:00
committed by GitHub
parent 5eddfa935c
commit 0218a5a002
30 changed files with 1208 additions and 692 deletions

View File

@@ -2,4 +2,4 @@ blank_issues_enabled: false
contact_links: contact_links:
- name: Slack Community Chat - name: Slack Community Chat
url: https://livekit.io/join-slack url: https://livekit.io/join-slack
about: Ask questions and discuss with other LiveKit users in real time. about: Ask questions and discuss with other LiveKit users in real time.

View File

@@ -41,4 +41,4 @@ body:
id: additional-context id: additional-context
attributes: attributes:
label: Additional Information label: Additional Information
description: Add any other context or screenshots about the feature request here. description: Add any other context or screenshots about the feature request here.

View File

@@ -11,8 +11,10 @@
# LiveKit Agents Playground # LiveKit Agents Playground
<!--BEGIN_DESCRIPTION--> <!--BEGIN_DESCRIPTION-->
The Agents Playground is designed for quickly prototyping with server side agents built with [LiveKit Agents Framework](https://github.com/livekit/agents). Easily tap into LiveKit WebRTC sessions and process or generate audio, video, and data streams. The Agents Playground is designed for quickly prototyping with server side agents built with [LiveKit Agents Framework](https://github.com/livekit/agents). Easily tap into LiveKit WebRTC sessions and process or generate audio, video, and data streams.
The playground includes components to fully interact with any LiveKit agent, through video, audio and chat. The playground includes components to fully interact with any LiveKit agent, through video, audio and chat.
<!--END_DESCRIPTION--> <!--END_DESCRIPTION-->
## Docs and references ## Docs and references
@@ -69,7 +71,9 @@ NEXT_PUBLIC_LIVEKIT_URL=wss://<Your Cloud URL>
- Mobile device sizes not supported currently - Mobile device sizes not supported currently
<!--BEGIN_REPO_NAV--> <!--BEGIN_REPO_NAV-->
<br/><table> <br/><table>
<thead><tr><th colspan="2">LiveKit Ecosystem</th></tr></thead> <thead><tr><th colspan="2">LiveKit Ecosystem</th></tr></thead>
<tbody> <tbody>
<tr><td>LiveKit SDKs</td><td><a href="https://github.com/livekit/client-sdk-js">Browser</a> · <a href="https://github.com/livekit/client-sdk-swift">iOS/macOS/visionOS</a> · <a href="https://github.com/livekit/client-sdk-android">Android</a> · <a href="https://github.com/livekit/client-sdk-flutter">Flutter</a> · <a href="https://github.com/livekit/client-sdk-react-native">React Native</a> · <a href="https://github.com/livekit/rust-sdks">Rust</a> · <a href="https://github.com/livekit/node-sdks">Node.js</a> · <a href="https://github.com/livekit/python-sdks">Python</a> · <a href="https://github.com/livekit/client-sdk-unity">Unity</a> · <a href="https://github.com/livekit/client-sdk-unity-web">Unity (WebGL)</a></td></tr><tr></tr> <tr><td>LiveKit SDKs</td><td><a href="https://github.com/livekit/client-sdk-js">Browser</a> · <a href="https://github.com/livekit/client-sdk-swift">iOS/macOS/visionOS</a> · <a href="https://github.com/livekit/client-sdk-android">Android</a> · <a href="https://github.com/livekit/client-sdk-flutter">Flutter</a> · <a href="https://github.com/livekit/client-sdk-react-native">React Native</a> · <a href="https://github.com/livekit/rust-sdks">Rust</a> · <a href="https://github.com/livekit/node-sdks">Node.js</a> · <a href="https://github.com/livekit/python-sdks">Python</a> · <a href="https://github.com/livekit/client-sdk-unity">Unity</a> · <a href="https://github.com/livekit/client-sdk-unity-web">Unity (WebGL)</a></td></tr><tr></tr>

623
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -6,35 +6,37 @@
"dev": "next dev", "dev": "next dev",
"build": "next build", "build": "next build",
"start": "next start", "start": "next start",
"lint": "next lint" "lint": "next lint",
"format": "prettier --write ."
}, },
"dependencies": { "dependencies": {
"@livekit/components-react": "^2.9.3", "@livekit/components-react": "^2.9.3",
"@livekit/components-styles": "^1.1.1", "@livekit/components-styles": "^1.1.5",
"@radix-ui/react-dropdown-menu": "^2.0.6", "@radix-ui/react-dropdown-menu": "^2.1.2",
"cookies-next": "^4.1.1", "cookies-next": "^4.3.0",
"framer-motion": "^10.16.16", "framer-motion": "^10.18.0",
"js-yaml": "^4.1.0", "js-yaml": "^4.1.0",
"livekit-client": "^2.5.1", "livekit-client": "^2.9.5",
"livekit-server-sdk": "^2.6.1", "livekit-server-sdk": "^2.13.0",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"next": "^14.0.4", "next": "^14.2.20",
"next-plugin-preval": "^1.2.6", "next-plugin-preval": "^1.2.6",
"qrcode.react": "^4.0.0", "qrcode.react": "^4.1.0",
"react": "^18", "react": "^18.3.1",
"react-dom": "^18" "react-dom": "^18.3.1"
}, },
"devDependencies": { "devDependencies": {
"@types/js-yaml": "^4.0.9", "@types/js-yaml": "^4.0.9",
"@types/lodash": "^4.17.0", "@types/lodash": "^4.17.13",
"@types/node": "^20.10.4", "@types/node": "^20.17.9",
"@types/react": "^18.2.43", "@types/react": "^18.3.14",
"@types/react-dom": "^18", "@types/react-dom": "^18.3.3",
"autoprefixer": "^10.4.16", "autoprefixer": "^10.4.20",
"eslint": "^8", "eslint": "^8.57.1",
"eslint-config-next": "14.2.26", "eslint-config-next": "14.2.26",
"postcss": "^8.4.31", "postcss": "^8.4.49",
"tailwindcss": "^3.3.5", "tailwindcss": "^3.4.16",
"typescript": "^5.3.3" "typescript": "^5.7.2",
"prettier": "^3.4.2"
} }
} }

View File

@@ -3,4 +3,4 @@ module.exports = {
tailwindcss: {}, tailwindcss: {},
autoprefixer: {}, autoprefixer: {},
}, },
} };

Binary file not shown.

Before

Width:  |  Height:  |  Size: 25 KiB

After

Width:  |  Height:  |  Size: 15 KiB

View File

@@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>

Before

Width:  |  Height:  |  Size: 1.3 KiB

View File

@@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 283 64"><path fill="black" d="M141 16c-11 0-19 7-19 18s9 18 20 18c7 0 13-3 16-7l-7-5c-2 3-6 4-9 4-5 0-9-3-10-7h28v-3c0-11-8-18-19-18zm-9 15c1-4 4-7 9-7s8 3 9 7h-18zm117-15c-11 0-19 7-19 18s9 18 20 18c6 0 12-3 16-7l-8-5c-2 3-5 4-8 4-5 0-9-3-11-7h28l1-3c0-11-8-18-19-18zm-10 15c2-4 5-7 10-7s8 3 9 7h-19zm-39 3c0 6 4 10 10 10 4 0 7-2 9-5l8 5c-3 5-9 8-17 8-11 0-19-7-19-18s8-18 19-18c8 0 14 3 17 8l-8 5c-2-3-5-5-9-5-6 0-10 4-10 10zm83-29v46h-9V5h9zM37 0l37 64H0L37 0zm92 5-27 48L74 5h10l18 30 17-30h10zm59 12v10l-3-1c-6 0-10 4-10 10v15h-9V17h9v9c0-5 6-9 13-9z"/></svg>

Before

Width:  |  Height:  |  Size: 629 B

View File

@@ -1 +1 @@
Files in this `cloud/` directory can be ignored. They are mocks which we override in our private, hosted version of the agents-playground that supports LiveKit Cloud authentication. Files in this `cloud/` directory can be ignored. They are mocks which we override in our private, hosted version of the agents-playground that supports LiveKit Cloud authentication.

View File

@@ -9,4 +9,4 @@ export function useCloud() {
const wsUrl = ""; const wsUrl = "";
return { generateToken, wsUrl }; return { generateToken, wsUrl };
} }

View File

@@ -0,0 +1,60 @@
import React from "react";
import { AttributeItem } from "@/lib/types";
interface AttributeRowProps {
attribute: AttributeItem;
onKeyChange: (id: string, newKey: string) => void;
onValueChange: (id: string, newValue: string) => void;
onRemove?: (id: string) => void;
disabled?: boolean;
}
export const AttributeRow: React.FC<AttributeRowProps> = ({
attribute,
onKeyChange,
onValueChange,
onRemove,
disabled = false,
}) => {
return (
<div className="flex items-center gap-2 mb-2">
<input
value={attribute.key}
onChange={(e) => onKeyChange(attribute.id, e.target.value)}
className="flex-1 min-w-0 text-gray-400 text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-1 font-mono"
placeholder="Name"
disabled={disabled}
/>
<input
value={attribute.value}
onChange={(e) => onValueChange(attribute.id, e.target.value)}
className="flex-1 min-w-0 text-gray-400 text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-1 font-mono"
placeholder="Value"
disabled={disabled}
/>
{onRemove && (
<button
onClick={() => onRemove(attribute.id)}
className="flex-shrink-0 w-6 h-6 flex items-center justify-center text-gray-400 hover:text-white"
disabled={disabled}
style={{ display: disabled ? "none" : "flex" }}
>
<svg
xmlns="http://www.w3.org/2000/svg"
className="h-4 w-4"
fill="none"
viewBox="0 0 24 24"
stroke="currentColor"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M6 18L18 6M6 6l12 12"
/>
</svg>
</button>
)}
</div>
);
};

View File

@@ -1,93 +0,0 @@
import React from 'react';
import { ConnectionState } from 'livekit-client';
import { AttributeItem } from '@/lib/types';
interface AttributesEditorProps {
attributes: AttributeItem[];
onAttributesChange: (attributes: AttributeItem[]) => void;
themeColor: string;
disabled?: boolean;
connectionState?: ConnectionState;
}
export const AttributesEditor: React.FC<AttributesEditorProps> = ({
attributes,
onAttributesChange,
themeColor,
disabled = false,
connectionState
}) => {
const handleKeyChange = (id: string, newKey: string) => {
const updatedAttributes = attributes.map(attr =>
attr.id === id ? { ...attr, key: newKey } : attr
);
onAttributesChange(updatedAttributes);
};
const handleValueChange = (id: string, newValue: string) => {
const updatedAttributes = attributes.map(attr =>
attr.id === id ? { ...attr, value: newValue } : attr
);
onAttributesChange(updatedAttributes);
};
const handleRemoveAttribute = (id: string) => {
const updatedAttributes = attributes.filter(attr => attr.id !== id);
onAttributesChange(updatedAttributes);
};
const handleAddAttribute = () => {
const newId = `attr_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
const updatedAttributes = [...attributes, { id: newId, key: "", value: "" }];
onAttributesChange(updatedAttributes);
};
return (
<div className="border border-gray-800 rounded-sm p-3 bg-gray-900/30">
{attributes.map((attribute) => (
<div key={attribute.id} className="flex items-center gap-2 mb-2">
<input
value={attribute.key}
onChange={(e) => handleKeyChange(attribute.id, e.target.value)}
className="flex-1 min-w-0 text-white text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-1"
placeholder="Key"
disabled={disabled}
/>
<input
value={attribute.value}
onChange={(e) => handleValueChange(attribute.id, e.target.value)}
className="flex-1 min-w-0 text-white text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-1"
placeholder="Value"
disabled={disabled}
/>
<button
onClick={() => handleRemoveAttribute(attribute.id)}
className="flex-shrink-0 w-6 h-6 flex items-center justify-center text-gray-400 hover:text-white"
disabled={disabled}
style={{ display: disabled ? 'none' : 'flex' }}
>
<svg xmlns="http://www.w3.org/2000/svg" className="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M6 18L18 6M6 6l12 12" />
</svg>
</button>
</div>
))}
<button
onClick={handleAddAttribute}
className={`text-xs py-1 px-2 rounded-sm flex items-center gap-1 ${
disabled
? "bg-gray-700 text-gray-500 cursor-not-allowed"
: `bg-${themeColor}-500 hover:bg-${themeColor}-600 text-white`
}`}
disabled={disabled}
>
<svg xmlns="http://www.w3.org/2000/svg" className="h-3 w-3" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 4v16m8-8H4" />
</svg>
Add Attribute
</button>
</div>
);
};

View File

@@ -0,0 +1,268 @@
import React, { useState, useCallback, useEffect, useRef } from "react";
import { ConnectionState } from "livekit-client";
import { AttributeItem } from "@/lib/types";
import { Button } from "@/components/button/Button";
import { useLocalParticipant } from "@livekit/components-react";
import { AttributeRow } from "./AttributeRow";
interface AttributesInspectorProps {
attributes: AttributeItem[];
onAttributesChange: (attributes: AttributeItem[]) => void;
themeColor: string;
disabled?: boolean;
connectionState?: ConnectionState;
metadata?: string;
onMetadataChange?: (metadata: string) => void;
}
export const AttributesInspector: React.FC<AttributesInspectorProps> = ({
attributes,
onAttributesChange,
themeColor,
disabled = false,
connectionState,
metadata,
onMetadataChange,
}) => {
const [isExpanded, setIsExpanded] = useState(false);
const [isMetadataExpanded, setIsMetadataExpanded] = useState(false);
const [localAttributes, setLocalAttributes] =
useState<AttributeItem[]>(attributes);
const [hasUnsavedChanges, setHasUnsavedChanges] = useState(false);
const [showSyncFlash, setShowSyncFlash] = useState(false);
const { localParticipant } = useLocalParticipant();
const timeoutRef = useRef<NodeJS.Timeout>();
const syncFlashTimeoutRef = useRef<NodeJS.Timeout>();
// Update local attributes when props change
useEffect(() => {
setLocalAttributes(attributes);
}, [attributes]);
const syncAttributesWithRoom = useCallback(() => {
if (!localParticipant || connectionState !== ConnectionState.Connected)
return;
const attributesMap = localAttributes.reduce(
(acc, attr) => {
if (attr.key && attr.key.trim() !== "") {
acc[attr.key] = attr.value;
}
return acc;
},
{} as Record<string, string>,
);
localParticipant.setAttributes(attributesMap);
setHasUnsavedChanges(false);
setShowSyncFlash(true);
if (syncFlashTimeoutRef.current) {
clearTimeout(syncFlashTimeoutRef.current);
}
syncFlashTimeoutRef.current = setTimeout(
() => setShowSyncFlash(false),
1000,
);
}, [localAttributes, localParticipant, connectionState]);
// Handle debounced sync
useEffect(() => {
if (!hasUnsavedChanges) return;
if (timeoutRef.current) {
clearTimeout(timeoutRef.current);
}
timeoutRef.current = setTimeout(() => {
if (connectionState === ConnectionState.Connected && localParticipant) {
syncAttributesWithRoom();
}
}, 2000);
return () => {
if (timeoutRef.current) {
clearTimeout(timeoutRef.current);
}
};
}, [
hasUnsavedChanges,
syncAttributesWithRoom,
connectionState,
localParticipant,
]);
const handleKeyChange = (id: string, newKey: string) => {
const updatedAttributes = localAttributes.map((attr) =>
attr.id === id ? { ...attr, key: newKey } : attr,
);
setLocalAttributes(updatedAttributes);
onAttributesChange(updatedAttributes);
if (connectionState === ConnectionState.Connected && newKey.trim() !== "") {
setHasUnsavedChanges(true);
}
};
const handleValueChange = (id: string, newValue: string) => {
const updatedAttributes = localAttributes.map((attr) =>
attr.id === id ? { ...attr, value: newValue } : attr,
);
setLocalAttributes(updatedAttributes);
onAttributesChange(updatedAttributes);
if (connectionState === ConnectionState.Connected) {
setHasUnsavedChanges(true);
}
};
const handleRemoveAttribute = (id: string) => {
const updatedAttributes = localAttributes.filter((attr) => attr.id !== id);
setLocalAttributes(updatedAttributes);
onAttributesChange(updatedAttributes);
if (connectionState === ConnectionState.Connected) {
setHasUnsavedChanges(true);
}
};
const handleAddAttribute = () => {
const newId = `attr_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
const updatedAttributes = [
...localAttributes,
{ id: newId, key: "", value: "" },
];
setLocalAttributes(updatedAttributes);
onAttributesChange(updatedAttributes);
if (connectionState === ConnectionState.Connected) {
setHasUnsavedChanges(true);
}
};
return (
<div>
<div
className="flex items-center justify-between mb-2 cursor-pointer"
onClick={() => setIsExpanded(!isExpanded)}
>
<div className="text-sm text-gray-500">Attributes</div>
<svg
xmlns="http://www.w3.org/2000/svg"
className={`h-4 w-4 text-gray-500 transition-transform ${isExpanded ? "rotate-180" : ""}`}
fill="none"
viewBox="0 0 24 24"
stroke="currentColor"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M19 9l-7 7-7-7"
/>
</svg>
</div>
{isExpanded && (
<div className="border border-gray-800 rounded-sm bg-gray-900/30 p-3 mb-2">
{disabled ? (
localAttributes.length === 0 ? (
<div className="text-sm text-gray-400 font-sans">
No attributes set
</div>
) : (
localAttributes.map((attribute) => (
<AttributeRow
key={attribute.id}
attribute={attribute}
onKeyChange={handleKeyChange}
onValueChange={handleValueChange}
disabled={true}
/>
))
)
) : (
<>
{localAttributes.map((attribute) => (
<AttributeRow
key={attribute.id}
attribute={attribute}
onKeyChange={handleKeyChange}
onValueChange={handleValueChange}
onRemove={handleRemoveAttribute}
disabled={disabled}
/>
))}
<div className="flex justify-between items-center">
<Button
accentColor={themeColor}
onClick={handleAddAttribute}
className="text-xs flex items-center gap-1"
>
<svg
xmlns="http://www.w3.org/2000/svg"
className="h-3 w-3"
fill="none"
viewBox="0 0 24 24"
stroke="currentColor"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M12 4v16m8-8H4"
/>
</svg>
Attribute
</Button>
{showSyncFlash && (
<div className="text-xs text-gray-400 animate-fade-in-out">
Changes saved
</div>
)}
</div>
</>
)}
</div>
)}
<>
<div
className="flex items-center justify-between mb-2 cursor-pointer"
onClick={() => setIsMetadataExpanded(!isMetadataExpanded)}
>
<div className="text-sm text-gray-500">Metadata</div>
<svg
xmlns="http://www.w3.org/2000/svg"
className={`h-4 w-4 text-gray-500 transition-transform ${isMetadataExpanded ? "rotate-180" : ""}`}
fill="none"
viewBox="0 0 24 24"
stroke="currentColor"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M19 9l-7 7-7-7"
/>
</svg>
</div>
{isMetadataExpanded &&
(disabled || connectionState === ConnectionState.Connected ? (
<div className="border border-gray-800 rounded-sm bg-gray-900/30 px-3 py-2 mb-4 min-h-[40px] flex items-center">
{metadata ? (
<pre className="w-full text-gray-400 text-xs bg-transparent font-mono whitespace-pre-wrap break-words m-0 p-0 border-0">
{metadata}
</pre>
) : (
<div className="text-sm text-gray-400 font-sans w-full text-left">
No metadata set
</div>
)}
</div>
) : (
<textarea
value={metadata}
onChange={(e) => onMetadataChange?.(e.target.value)}
className="w-full text-gray-400 text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-2 font-mono mb-4"
placeholder="Enter metadata..."
rows={3}
/>
))}
</>
</div>
);
};

View File

@@ -1,4 +1,4 @@
import { ReactNode } from "react"; import { ReactNode, useState } from "react";
import { PlaygroundDeviceSelector } from "@/components/playground/PlaygroundDeviceSelector"; import { PlaygroundDeviceSelector } from "@/components/playground/PlaygroundDeviceSelector";
import { TrackToggle } from "@livekit/components-react"; import { TrackToggle } from "@livekit/components-react";
import { Track } from "livekit-client"; import { Track } from "livekit-client";
@@ -7,35 +7,65 @@ type ConfigurationPanelItemProps = {
title: string; title: string;
children?: ReactNode; children?: ReactNode;
source?: Track.Source; source?: Track.Source;
collapsible?: boolean;
defaultCollapsed?: boolean;
}; };
export const ConfigurationPanelItem: React.FC<ConfigurationPanelItemProps> = ({ export const ConfigurationPanelItem: React.FC<ConfigurationPanelItemProps> = ({
children, children,
title, title,
source, source,
collapsible = false,
defaultCollapsed = false,
}) => { }) => {
const [isCollapsed, setIsCollapsed] = useState(defaultCollapsed);
return ( return (
<div className="w-full text-gray-300 py-4 border-b border-b-gray-800 relative"> <div className="w-full text-gray-300 py-4 border-b border-b-gray-800 relative">
<div className="flex flex-row justify-between items-center px-4 text-xs uppercase tracking-wider"> <div className="flex flex-row justify-between items-center px-4 text-xs uppercase tracking-wider">
<h3>{title}</h3> <h3>{title}</h3>
{source && ( <div className="flex items-center gap-2">
<span className="flex flex-row gap-2"> {source && (
<TrackToggle <span className="flex flex-row gap-2">
className="px-2 py-1 bg-gray-900 text-gray-300 border border-gray-800 rounded-sm hover:bg-gray-800" <TrackToggle
source={source} className="px-2 py-1 bg-gray-900 text-gray-300 border border-gray-800 rounded-sm hover:bg-gray-800"
/> source={source}
{source === Track.Source.Camera && ( />
<PlaygroundDeviceSelector kind="videoinput" /> {source === Track.Source.Camera && (
)} <PlaygroundDeviceSelector kind="videoinput" />
{source === Track.Source.Microphone && ( )}
<PlaygroundDeviceSelector kind="audioinput" /> {source === Track.Source.Microphone && (
)} <PlaygroundDeviceSelector kind="audioinput" />
</span> )}
)} </span>
</div> )}
<div className="px-4 py-2 text-xs text-gray-500 leading-normal"> {collapsible && (
{children} <button
onClick={() => setIsCollapsed(!isCollapsed)}
className="text-gray-400 hover:text-gray-300 transition-colors"
>
<svg
className={`w-4 h-4 transform transition-transform ${!isCollapsed ? "rotate-180" : ""}`}
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M19 9l-7 7-7-7"
/>
</svg>
</button>
)}
</div>
</div> </div>
{!isCollapsed && (
<div className="px-4 py-2 text-xs text-gray-500 leading-normal">
{children}
</div>
)}
</div> </div>
); );
}; };

View File

@@ -52,11 +52,5 @@ export const EditableNameValueRow: React.FC<EditableNameValueRowProps> = ({
</div> </div>
); );
} }
return ( return <NameValueRow name={name} value={value} valueColor={valueColor} />;
<NameValueRow
name={name}
value={value}
valueColor={valueColor}
/>
);
}; };

View File

@@ -24,13 +24,15 @@ import {
useTracks, useTracks,
useVoiceAssistant, useVoiceAssistant,
useRoomContext, useRoomContext,
useParticipantAttributes,
} from "@livekit/components-react"; } from "@livekit/components-react";
import { ConnectionState, LocalParticipant, Track } from "livekit-client"; import { ConnectionState, LocalParticipant, Track } from "livekit-client";
import { QRCodeSVG } from "qrcode.react"; import { QRCodeSVG } from "qrcode.react";
import { ReactNode, useCallback, useEffect, useMemo, useState } from "react"; import { ReactNode, useCallback, useEffect, useMemo, useState } from "react";
import tailwindTheme from "../../lib/tailwindTheme.preval"; import tailwindTheme from "../../lib/tailwindTheme.preval";
import { EditableNameValueRow } from "@/components/config/NameValueRow"; import { EditableNameValueRow } from "@/components/config/NameValueRow";
import { AttributesEditor } from "@/components/config/AttributesEditor"; import { AttributesInspector } from "@/components/config/AttributesInspector";
import { RpcPanel } from "./RpcPanel";
export interface PlaygroundMeta { export interface PlaygroundMeta {
name: string; name: string;
@@ -63,6 +65,7 @@ export default function Playground({
const [rpcMethod, setRpcMethod] = useState(""); const [rpcMethod, setRpcMethod] = useState("");
const [rpcPayload, setRpcPayload] = useState(""); const [rpcPayload, setRpcPayload] = useState("");
const [showRpc, setShowRpc] = useState(false);
useEffect(() => { useEffect(() => {
if (roomState === ConnectionState.Connected) { if (roomState === ConnectionState.Connected) {
@@ -74,27 +77,27 @@ export default function Playground({
const agentVideoTrack = tracks.find( const agentVideoTrack = tracks.find(
(trackRef) => (trackRef) =>
trackRef.publication.kind === Track.Kind.Video && trackRef.publication.kind === Track.Kind.Video &&
trackRef.participant.isAgent trackRef.participant.isAgent,
); );
const localTracks = tracks.filter( const localTracks = tracks.filter(
({ participant }) => participant instanceof LocalParticipant ({ participant }) => participant instanceof LocalParticipant,
); );
const localCameraTrack = localTracks.find( const localCameraTrack = localTracks.find(
({ source }) => source === Track.Source.Camera ({ source }) => source === Track.Source.Camera,
); );
const localScreenTrack = localTracks.find( const localScreenTrack = localTracks.find(
({ source }) => source === Track.Source.ScreenShare ({ source }) => source === Track.Source.ScreenShare,
); );
const localMicTrack = localTracks.find( const localMicTrack = localTracks.find(
({ source }) => source === Track.Source.Microphone ({ source }) => source === Track.Source.Microphone,
); );
const onDataReceived = useCallback( const onDataReceived = useCallback(
(msg: any) => { (msg: any) => {
if (msg.topic === "transcription") { if (msg.topic === "transcription") {
const decoded = JSON.parse( const decoded = JSON.parse(
new TextDecoder("utf-8").decode(msg.payload) new TextDecoder("utf-8").decode(msg.payload),
); );
let timestamp = new Date().getTime(); let timestamp = new Date().getTime();
if ("timestamp" in decoded && decoded.timestamp > 0) { if ("timestamp" in decoded && decoded.timestamp > 0) {
@@ -111,7 +114,7 @@ export default function Playground({
]); ]);
} }
}, },
[transcripts] [transcripts],
); );
useDataChannel(onDataReceived); useDataChannel(onDataReceived);
@@ -121,14 +124,14 @@ export default function Playground({
const disconnectedContent = ( const disconnectedContent = (
<div className="flex items-center justify-center text-gray-700 text-center w-full h-full"> <div className="flex items-center justify-center text-gray-700 text-center w-full h-full">
No video track. Connect to get started. No agent video track. Connect to get started.
</div> </div>
); );
const loadingContent = ( const loadingContent = (
<div className="flex flex-col items-center justify-center gap-2 text-gray-700 text-center h-full w-full"> <div className="flex flex-col items-center justify-center gap-2 text-gray-700 text-center h-full w-full">
<LoadingSVG /> <LoadingSVG />
Waiting for video track Waiting for agent video track
</div> </div>
); );
@@ -159,25 +162,25 @@ export default function Playground({
document.body.style.setProperty( document.body.style.setProperty(
"--lk-theme-color", "--lk-theme-color",
// @ts-ignore // @ts-ignore
tailwindTheme.colors[config.settings.theme_color]["500"] tailwindTheme.colors[config.settings.theme_color]["500"],
); );
document.body.style.setProperty( document.body.style.setProperty(
"--lk-drop-shadow", "--lk-drop-shadow",
`var(--lk-theme-color) 0px 0px 18px` `var(--lk-theme-color) 0px 0px 18px`,
); );
}, [config.settings.theme_color]); }, [config.settings.theme_color]);
const audioTileContent = useMemo(() => { const audioTileContent = useMemo(() => {
const disconnectedContent = ( const disconnectedContent = (
<div className="flex flex-col items-center justify-center gap-2 text-gray-700 text-center w-full"> <div className="flex flex-col items-center justify-center gap-2 text-gray-700 text-center w-full">
No audio track. Connect to get started. No agent audio track. Connect to get started.
</div> </div>
); );
const waitingContent = ( const waitingContent = (
<div className="flex flex-col items-center gap-2 text-gray-700 text-center w-full"> <div className="flex flex-col items-center gap-2 text-gray-700 text-center w-full">
<LoadingSVG /> <LoadingSVG />
Waiting for audio track Waiting for agent audio track
</div> </div>
); );
@@ -220,145 +223,63 @@ export default function Playground({
); );
} }
return <></>; return <></>;
}, [config.settings.theme_color, voiceAssistant.audioTrack, voiceAssistant.agent]); }, [
config.settings.theme_color,
voiceAssistant.audioTrack,
voiceAssistant.agent,
]);
const handleRpcCall = useCallback(async () => { const handleRpcCall = useCallback(async () => {
if (!voiceAssistant.agent || !room) return; if (!voiceAssistant.agent || !room) {
throw new Error("No agent or room available");
try {
const response = await room.localParticipant.performRpc({
destinationIdentity: voiceAssistant.agent.identity,
method: rpcMethod,
payload: rpcPayload,
});
console.log('RPC response:', response);
} catch (e) {
console.error('RPC call failed:', e);
} }
const response = await room.localParticipant.performRpc({
destinationIdentity: voiceAssistant.agent.identity,
method: rpcMethod,
payload: rpcPayload,
});
return response;
}, [room, rpcMethod, rpcPayload, voiceAssistant.agent]); }, [room, rpcMethod, rpcPayload, voiceAssistant.agent]);
const agentAttributes = useParticipantAttributes({
participant: voiceAssistant.agent,
});
const settingsTileContent = useMemo(() => { const settingsTileContent = useMemo(() => {
return ( return (
<div className="flex flex-col gap-4 h-full w-full items-start overflow-y-auto"> <div className="flex flex-col h-full w-full items-start overflow-y-auto">
{config.description && ( {config.description && (
<ConfigurationPanelItem title="Description"> <ConfigurationPanelItem title="Description">
{config.description} {config.description}
</ConfigurationPanelItem> </ConfigurationPanelItem>
)} )}
<ConfigurationPanelItem title="Settings"> <ConfigurationPanelItem title="Room">
<div className="flex flex-col gap-4"> <div className="flex flex-col gap-2">
<EditableNameValueRow <EditableNameValueRow
name="Room" name="Room name"
value={roomState === ConnectionState.Connected ? name : config.settings.room_name} value={
roomState === ConnectionState.Connected
? name
: config.settings.room_name
}
valueColor={`${config.settings.theme_color}-500`} valueColor={`${config.settings.theme_color}-500`}
onValueChange={(value) => { onValueChange={(value) => {
const newSettings = { ...config.settings }; const newSettings = { ...config.settings };
newSettings.room_name = value; newSettings.room_name = value;
setUserSettings(newSettings); setUserSettings(newSettings);
}} }}
placeholder="Enter room name" placeholder="Auto"
editable={roomState !== ConnectionState.Connected} editable={roomState !== ConnectionState.Connected}
/> />
<EditableNameValueRow
name="Participant ID"
value={roomState === ConnectionState.Connected ?
(localParticipant?.identity || '') :
(config.settings.participant_id || '')}
valueColor={`${config.settings.theme_color}-500`}
onValueChange={(value) => {
const newSettings = { ...config.settings };
newSettings.participant_id = value;
setUserSettings(newSettings);
}}
placeholder="Enter participant id"
editable={roomState !== ConnectionState.Connected}
/>
<EditableNameValueRow
name="Participant Name"
value={roomState === ConnectionState.Connected ?
(localParticipant?.name || '') :
(config.settings.participant_name || '')}
valueColor={`${config.settings.theme_color}-500`}
onValueChange={(value) => {
const newSettings = { ...config.settings };
newSettings.participant_name = value;
setUserSettings(newSettings);
}}
placeholder="Enter participant name"
editable={roomState !== ConnectionState.Connected}
/>
</div>
<div className="flex flex-col gap-2 mt-4">
<div className="text-xs text-gray-500 mt-2">Metadata</div>
<textarea
value={config.settings.metadata || ""}
onChange={(e) => {
const newSettings = { ...config.settings };
newSettings.metadata = e.target.value;
setUserSettings(newSettings);
}}
className="w-full text-white text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-2"
placeholder="Custom metadata..."
rows={2}
disabled={roomState === ConnectionState.Connected}
/>
<div className="text-xs text-gray-500 mt-2">Attributes</div>
<AttributesEditor
attributes={config.settings.attributes || []}
onAttributesChange={(newAttributes) => {
const newSettings = { ...config.settings };
newSettings.attributes = newAttributes;
setUserSettings(newSettings);
}}
themeColor={config.settings.theme_color}
disabled={roomState === ConnectionState.Connected}
/>
</div>
<div className="flex flex-col gap-2 mt-4">
<div className="text-xs text-gray-500 mt-2">RPC Method</div>
<input
type="text"
value={rpcMethod}
onChange={(e) => setRpcMethod(e.target.value)}
className="w-full text-white text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-2"
placeholder="RPC method name"
/>
<div className="text-xs text-gray-500 mt-2">RPC Payload</div>
<textarea
value={rpcPayload}
onChange={(e) => setRpcPayload(e.target.value)}
className="w-full text-white text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-2"
placeholder="RPC payload"
rows={2}
/>
<button
onClick={handleRpcCall}
disabled={!voiceAssistant.agent || !rpcMethod}
className={`mt-2 px-2 py-1 rounded-sm text-xs
${voiceAssistant.agent && rpcMethod
? `bg-${config.settings.theme_color}-500 hover:bg-${config.settings.theme_color}-600`
: 'bg-gray-700 cursor-not-allowed'
} text-white`}
>
Perform RPC Call
</button>
</div>
</ConfigurationPanelItem>
<ConfigurationPanelItem title="Status">
<div className="flex flex-col gap-2">
<NameValueRow <NameValueRow
name="Room connected" name="Status"
value={ value={
roomState === ConnectionState.Connecting ? ( roomState === ConnectionState.Connecting ? (
<LoadingSVG diameter={16} strokeWidth={2} /> <LoadingSVG diameter={16} strokeWidth={2} />
) : ( ) : (
roomState.toUpperCase() roomState.charAt(0).toUpperCase() + roomState.slice(1)
) )
} }
valueColor={ valueColor={
@@ -367,15 +288,36 @@ export default function Playground({
: "gray-500" : "gray-500"
} }
/> />
</div>
</ConfigurationPanelItem>
<ConfigurationPanelItem title="Agent">
<div className="flex flex-col gap-2">
<EditableNameValueRow
name="Agent name"
value={
roomState === ConnectionState.Connected
? config.settings.agent_name || "None"
: config.settings.agent_name || ""
}
valueColor={`${config.settings.theme_color}-500`}
onValueChange={(value) => {
const newSettings = { ...config.settings };
newSettings.agent_name = value;
setUserSettings(newSettings);
}}
placeholder="None"
editable={roomState !== ConnectionState.Connected}
/>
<NameValueRow <NameValueRow
name="Agent connected" name="Identity"
value={ value={
voiceAssistant.agent ? ( voiceAssistant.agent ? (
"TRUE" voiceAssistant.agent.identity
) : roomState === ConnectionState.Connected ? ( ) : roomState === ConnectionState.Connected ? (
<LoadingSVG diameter={12} strokeWidth={2} /> <LoadingSVG diameter={12} strokeWidth={2} />
) : ( ) : (
"FALSE" "No agent connected"
) )
} }
valueColor={ valueColor={
@@ -384,32 +326,122 @@ export default function Playground({
: "gray-500" : "gray-500"
} }
/> />
{roomState === ConnectionState.Connected &&
voiceAssistant.agent && (
<AttributesInspector
attributes={Object.entries(
agentAttributes.attributes || {},
).map(([key, value], index) => ({
id: `agent-attr-${index}`,
key,
value: String(value),
}))}
onAttributesChange={() => {}}
themeColor={config.settings.theme_color}
disabled={true}
/>
)}
<p className="text-xs text-gray-500 text-right">
Set an agent name to use{" "}
<a
href="https://docs.livekit.io/agents/worker/dispatch#explicit"
target="_blank"
rel="noopener noreferrer"
className="text-gray-500 hover:text-gray-300 underline"
>
explicit dispatch
</a>
.
</p>
</div> </div>
</ConfigurationPanelItem> </ConfigurationPanelItem>
{roomState === ConnectionState.Connected && config.settings.inputs.screen && (
<ConfigurationPanelItem <ConfigurationPanelItem title="User">
title="Screen" <div className="flex flex-col gap-2">
source={Track.Source.ScreenShare} <EditableNameValueRow
> name="Name"
{localScreenTrack ? ( value={
<div className="relative"> roomState === ConnectionState.Connected
<VideoTrack ? localParticipant?.name || ""
className="rounded-sm border border-gray-800 opacity-70 w-full" : config.settings.participant_name || ""
trackRef={localScreenTrack} }
/> valueColor={`${config.settings.theme_color}-500`}
</div> onValueChange={(value) => {
) : ( const newSettings = { ...config.settings };
<div className="flex items-center justify-center text-gray-700 text-center w-full h-full"> newSettings.participant_name = value;
Press the button above to share your screen. setUserSettings(newSettings);
</div> }}
)} placeholder="Auto"
</ConfigurationPanelItem> editable={roomState !== ConnectionState.Connected}
/>
<EditableNameValueRow
name="Identity"
value={
roomState === ConnectionState.Connected
? localParticipant?.identity || ""
: config.settings.participant_id || ""
}
valueColor={`${config.settings.theme_color}-500`}
onValueChange={(value) => {
const newSettings = { ...config.settings };
newSettings.participant_id = value;
setUserSettings(newSettings);
}}
placeholder="Auto"
editable={roomState !== ConnectionState.Connected}
/>
<AttributesInspector
attributes={config.settings.attributes || []}
onAttributesChange={(newAttributes) => {
const newSettings = { ...config.settings };
newSettings.attributes = newAttributes;
setUserSettings(newSettings);
}}
metadata={config.settings.metadata}
onMetadataChange={(metadata) => {
const newSettings = { ...config.settings };
newSettings.metadata = metadata;
setUserSettings(newSettings);
}}
themeColor={config.settings.theme_color}
disabled={false}
connectionState={roomState}
/>
</div>
</ConfigurationPanelItem>
{roomState === ConnectionState.Connected &&
config.settings.inputs.screen && (
<ConfigurationPanelItem
title="Screen"
source={Track.Source.ScreenShare}
>
{localScreenTrack ? (
<div className="relative">
<VideoTrack
className="rounded-sm border border-gray-800 opacity-70 w-full"
trackRef={localScreenTrack}
/>
</div>
) : (
<div className="flex items-center justify-center text-gray-700 text-center w-full h-full">
Press the button above to share your screen.
</div>
)}
</ConfigurationPanelItem>
)}
{roomState === ConnectionState.Connected && voiceAssistant.agent && (
<RpcPanel
config={config}
rpcMethod={rpcMethod}
rpcPayload={rpcPayload}
setRpcMethod={setRpcMethod}
setRpcPayload={setRpcPayload}
handleRpcCall={handleRpcCall}
/>
)} )}
{localCameraTrack && ( {localCameraTrack && (
<ConfigurationPanelItem <ConfigurationPanelItem title="Camera" source={Track.Source.Camera}>
title="Camera"
source={Track.Source.Camera}
>
<div className="relative"> <div className="relative">
<VideoTrack <VideoTrack
className="rounded-sm border border-gray-800 opacity-70 w-full" className="rounded-sm border border-gray-800 opacity-70 w-full"
@@ -464,6 +496,8 @@ export default function Playground({
rpcMethod, rpcMethod,
rpcPayload, rpcPayload,
handleRpcCall, handleRpcCall,
showRpc,
setShowRpc,
]); ]);
let mobileTabs: PlaygroundTab[] = []; let mobileTabs: PlaygroundTab[] = [];
@@ -549,7 +583,7 @@ export default function Playground({
> >
{config.settings.outputs.video && ( {config.settings.outputs.video && (
<PlaygroundTile <PlaygroundTile
title="Video" title="Agent Video"
className="w-full h-full grow" className="w-full h-full grow"
childrenClassName="justify-center" childrenClassName="justify-center"
> >
@@ -558,7 +592,7 @@ export default function Playground({
)} )}
{config.settings.outputs.audio && ( {config.settings.outputs.audio && (
<PlaygroundTile <PlaygroundTile
title="Audio" title="Agent Audio"
className="w-full h-full grow" className="w-full h-full grow"
childrenClassName="justify-center" childrenClassName="justify-center"
> >

View File

@@ -66,7 +66,7 @@ export const PlaygroundTabbedTile: React.FC<PlaygroundTabbedTileProps> = ({
}) => { }) => {
const contentPadding = 4; const contentPadding = 4;
const [activeTab, setActiveTab] = useState(initialTab); const [activeTab, setActiveTab] = useState(initialTab);
if(activeTab >= tabs.length) { if (activeTab >= tabs.length) {
return null; return null;
} }
return ( return (

View File

@@ -0,0 +1,107 @@
import { ConfigurationPanelItem } from "@/components/config/ConfigurationPanelItem";
import { useState } from "react";
import { LoadingSVG } from "@/components/button/LoadingSVG";
import { Button } from "@/components/button/Button";
interface RpcPanelProps {
config: any;
rpcMethod: string;
rpcPayload: string;
setRpcMethod: (method: string) => void;
setRpcPayload: (payload: string) => void;
handleRpcCall: () => Promise<any>;
}
export function RpcPanel({
config,
rpcMethod,
rpcPayload,
setRpcMethod,
setRpcPayload,
handleRpcCall,
}: RpcPanelProps) {
const [rpcResult, setRpcResult] = useState<{
success: boolean;
data: any;
} | null>(null);
const [isLoading, setIsLoading] = useState(false);
const handleCall = async () => {
setIsLoading(true);
setRpcResult(null);
try {
const result = await handleRpcCall();
setRpcResult({ success: true, data: result });
} catch (error) {
setRpcResult({
success: false,
data: error instanceof Error ? error.message : String(error),
});
} finally {
setIsLoading(false);
}
};
return (
<ConfigurationPanelItem
title="RPC"
collapsible={true}
defaultCollapsed={true}
>
<div className="flex flex-col gap-2">
<div className="text-xs text-gray-500 mt-2">Method Name</div>
<input
type="text"
value={rpcMethod}
onChange={(e) => setRpcMethod(e.target.value)}
className="w-full text-white text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-2"
placeholder="my_method"
/>
<div className="text-xs text-gray-500 mt-2">Payload</div>
<textarea
value={rpcPayload}
onChange={(e) => setRpcPayload(e.target.value)}
className="w-full text-white text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-2"
placeholder='{"my": "payload"}'
rows={2}
/>
<Button
accentColor={config.settings.theme_color}
onClick={handleCall}
disabled={!rpcMethod || isLoading}
className="mt-2 text-xs flex items-center justify-center gap-2"
>
{isLoading ? (
<>
<LoadingSVG diameter={12} strokeWidth={2} />
Performing RPC...
</>
) : (
"Perform RPC"
)}
</Button>
{rpcResult && (
<>
<div className="text-xs text-gray-500 mt-2">
{rpcResult.success ? "Result" : "Error"}
</div>
<div
className={`w-full text-sm bg-transparent border rounded-sm px-3 py-2 ${
rpcResult.success
? "border-green-500 text-green-500"
: "border-red-500 text-red-500"
}`}
>
{typeof rpcResult.data === "object"
? JSON.stringify(rpcResult.data, null, 2)
: String(rpcResult.data)}
</div>
</>
)}
</div>
</ConfigurationPanelItem>
);
}

View File

@@ -2,7 +2,7 @@ import * as DropdownMenu from "@radix-ui/react-dropdown-menu";
import { CheckIcon, ChevronIcon } from "./icons"; import { CheckIcon, ChevronIcon } from "./icons";
import { useConfig } from "@/hooks/useConfig"; import { useConfig } from "@/hooks/useConfig";
type SettingType = "inputs" | "outputs" | "chat" | "theme_color" type SettingType = "inputs" | "outputs" | "chat" | "theme_color";
type SettingValue = { type SettingValue = {
title: string; title: string;
@@ -55,18 +55,19 @@ const settingsDropdown: SettingValue[] = [
]; ];
export const SettingsDropdown = () => { export const SettingsDropdown = () => {
const {config, setUserSettings} = useConfig(); const { config, setUserSettings } = useConfig();
const isEnabled = (setting: SettingValue) => { const isEnabled = (setting: SettingValue) => {
if (setting.type === "separator" || setting.type === "theme_color") return false; if (setting.type === "separator" || setting.type === "theme_color")
return false;
if (setting.type === "chat") { if (setting.type === "chat") {
return config.settings[setting.type]; return config.settings[setting.type];
} }
if(setting.type === "inputs") { if (setting.type === "inputs") {
const key = setting.key as "camera" | "mic" | "screen"; const key = setting.key as "camera" | "mic" | "screen";
return config.settings.inputs[key]; return config.settings.inputs[key];
} else if(setting.type === "outputs") { } else if (setting.type === "outputs") {
const key = setting.key as "video" | "audio"; const key = setting.key as "video" | "audio";
return config.settings.outputs[key]; return config.settings.outputs[key];
} }
@@ -77,13 +78,13 @@ export const SettingsDropdown = () => {
const toggleSetting = (setting: SettingValue) => { const toggleSetting = (setting: SettingValue) => {
if (setting.type === "separator" || setting.type === "theme_color") return; if (setting.type === "separator" || setting.type === "theme_color") return;
const newValue = !isEnabled(setting); const newValue = !isEnabled(setting);
const newSettings = {...config.settings} const newSettings = { ...config.settings };
if(setting.type === "chat") { if (setting.type === "chat") {
newSettings.chat = newValue; newSettings.chat = newValue;
} else if(setting.type === "inputs") { } else if (setting.type === "inputs") {
newSettings.inputs[setting.key as "camera" | "mic" | "screen"] = newValue; newSettings.inputs[setting.key as "camera" | "mic" | "screen"] = newValue;
} else if(setting.type === "outputs") { } else if (setting.type === "outputs") {
newSettings.outputs[setting.key as "video" | "audio"] = newValue; newSettings.outputs[setting.key as "video" | "audio"] = newValue;
} }
setUserSettings(newSettings); setUserSettings(newSettings);
@@ -91,11 +92,9 @@ export const SettingsDropdown = () => {
return ( return (
<DropdownMenu.Root modal={false}> <DropdownMenu.Root modal={false}>
<DropdownMenu.Trigger className="group inline-flex max-h-12 items-center gap-1 rounded-md hover:bg-gray-800 bg-gray-900 border-gray-800 p-1 pr-2 text-gray-100"> <DropdownMenu.Trigger className="group inline-flex max-h-12 items-center gap-1 rounded-md hover:bg-gray-800 bg-gray-900 border-gray-800 p-1 pr-2 text-gray-100 my-auto text-sm flex gap-1 pl-2 py-1 h-full items-center">
<button className="my-auto text-sm flex gap-1 pl-2 py-1 h-full items-center"> Settings
Settings <ChevronIcon />
<ChevronIcon />
</button>
</DropdownMenu.Trigger> </DropdownMenu.Trigger>
<DropdownMenu.Portal> <DropdownMenu.Portal>
<DropdownMenu.Content <DropdownMenu.Content
@@ -130,4 +129,4 @@ export const SettingsDropdown = () => {
</DropdownMenu.Portal> </DropdownMenu.Portal>
</DropdownMenu.Root> </DropdownMenu.Root>
); );
}; };

View File

@@ -13,8 +13,8 @@ export const PlaygroundToast = () => {
toastMessage?.type === "error" toastMessage?.type === "error"
? "red" ? "red"
: toastMessage?.type === "success" : toastMessage?.type === "success"
? "green" ? "green"
: "amber"; : "amber";
return ( return (
<div <div

View File

@@ -1,29 +1,28 @@
"use client" "use client";
import React, { createContext, useState } from "react"; import React, { createContext, useState } from "react";
import { ToastType } from "./PlaygroundToast"; import { ToastType } from "./PlaygroundToast";
type ToastProviderData = { type ToastProviderData = {
setToastMessage: ( setToastMessage: (
message: { message: string; type: ToastType } | null message: { message: string; type: ToastType } | null,
) => void; ) => void;
toastMessage: { message: string; type: ToastType } | null; toastMessage: { message: string; type: ToastType } | null;
}; };
const ToastContext = createContext<ToastProviderData | undefined>(undefined); const ToastContext = createContext<ToastProviderData | undefined>(undefined);
export const ToastProvider = ({ export const ToastProvider = ({ children }: { children: React.ReactNode }) => {
children, const [toastMessage, setToastMessage] = useState<{
}: { message: string;
children: React.ReactNode; type: ToastType;
}) => { } | null>(null);
const [toastMessage, setToastMessage] = useState<{message: string, type: ToastType} | null>(null);
return ( return (
<ToastContext.Provider <ToastContext.Provider
value={{ value={{
toastMessage, toastMessage,
setToastMessage setToastMessage,
}} }}
> >
{children} {children}
@@ -37,4 +36,4 @@ export const useToast = () => {
throw new Error("useToast must be used within a ToastProvider"); throw new Error("useToast must be used within a ToastProvider");
} }
return context; return context;
} };

View File

@@ -39,6 +39,7 @@ export type UserSettings = {
room_name: string; room_name: string;
participant_id: string; participant_id: string;
participant_name: string; participant_name: string;
agent_name?: string;
metadata?: string; metadata?: string;
attributes?: AttributeItem[]; attributes?: AttributeItem[];
}; };
@@ -77,7 +78,7 @@ const useAppConfig = (): AppConfig => {
if (process.env.NEXT_PUBLIC_APP_CONFIG) { if (process.env.NEXT_PUBLIC_APP_CONFIG) {
try { try {
const parsedConfig = jsYaml.load( const parsedConfig = jsYaml.load(
process.env.NEXT_PUBLIC_APP_CONFIG process.env.NEXT_PUBLIC_APP_CONFIG,
) as AppConfig; ) as AppConfig;
if (parsedConfig.settings === undefined) { if (parsedConfig.settings === undefined) {
parsedConfig.settings = defaultConfig.settings; parsedConfig.settings = defaultConfig.settings;
@@ -105,7 +106,7 @@ export const ConfigProvider = ({ children }: { children: React.ReactNode }) => {
const appConfig = useAppConfig(); const appConfig = useAppConfig();
const router = useRouter(); const router = useRouter();
const [localColorOverride, setLocalColorOverride] = useState<string | null>( const [localColorOverride, setLocalColorOverride] = useState<string | null>(
null null,
); );
const getSettingsFromUrl = useCallback(() => { const getSettingsFromUrl = useCallback(() => {
@@ -168,7 +169,7 @@ export const ConfigProvider = ({ children }: { children: React.ReactNode }) => {
// Note: We don't set ws_url and token to the URL on purpose // Note: We don't set ws_url and token to the URL on purpose
router.replace("/#" + obj.toString()); router.replace("/#" + obj.toString());
}, },
[router] [router],
); );
const setCookieSettings = useCallback((us: UserSettings) => { const setCookieSettings = useCallback((us: UserSettings) => {
@@ -228,7 +229,7 @@ export const ConfigProvider = ({ children }: { children: React.ReactNode }) => {
}; };
}); });
}, },
[appConfig, setCookieSettings, setUrlSettings] [appConfig, setCookieSettings, setUrlSettings],
); );
const [config, _setConfig] = useState<AppConfig>(getConfig()); const [config, _setConfig] = useState<AppConfig>(getConfig());

View File

@@ -1,4 +1,4 @@
"use client" "use client";
import { useCloud } from "@/cloud/useCloud"; import { useCloud } from "@/cloud/useCloud";
import React, { createContext, useState } from "react"; import React, { createContext, useState } from "react";
@@ -6,7 +6,7 @@ import { useCallback } from "react";
import { useConfig } from "./useConfig"; import { useConfig } from "./useConfig";
import { useToast } from "@/components/toast/ToasterProvider"; import { useToast } from "@/components/toast/ToasterProvider";
export type ConnectionMode = "cloud" | "manual" | "env" export type ConnectionMode = "cloud" | "manual" | "env";
type TokenGeneratorData = { type TokenGeneratorData = {
shouldConnect: boolean; shouldConnect: boolean;
@@ -17,7 +17,9 @@ type TokenGeneratorData = {
connect: (mode: ConnectionMode) => Promise<void>; connect: (mode: ConnectionMode) => Promise<void>;
}; };
const ConnectionContext = createContext<TokenGeneratorData | undefined>(undefined); const ConnectionContext = createContext<TokenGeneratorData | undefined>(
undefined,
);
export const ConnectionProvider = ({ export const ConnectionProvider = ({
children, children,
@@ -64,17 +66,25 @@ export const ConnectionProvider = ({
if (config.settings.participant_name) { if (config.settings.participant_name) {
body.participantName = config.settings.participant_name; body.participantName = config.settings.participant_name;
} }
if (config.settings.agent_name) {
body.agentName = config.settings.agent_name;
}
if (config.settings.metadata) { if (config.settings.metadata) {
body.metadata = config.settings.metadata; body.metadata = config.settings.metadata;
} }
const attributesArray = Array.isArray(config.settings.attributes) ? config.settings.attributes : []; const attributesArray = Array.isArray(config.settings.attributes)
? config.settings.attributes
: [];
if (attributesArray?.length) { if (attributesArray?.length) {
const attributes = attributesArray.reduce((acc, attr) => { const attributes = attributesArray.reduce(
if (attr.key){ (acc, attr) => {
acc[attr.key] = attr.value; if (attr.key) {
} acc[attr.key] = attr.value;
return acc; }
}, {} as Record<string, string>); return acc;
},
{} as Record<string, string>,
);
body.attributes = attributes; body.attributes = attributes;
} }
const { accessToken } = await fetch(`/api/token`, { const { accessToken } = await fetch(`/api/token`, {
@@ -83,9 +93,7 @@ export const ConnectionProvider = ({
"Content-Type": "application/json", "Content-Type": "application/json",
}, },
body: JSON.stringify(body), body: JSON.stringify(body),
}).then((res) => }).then((res) => res.json());
res.json()
);
token = accessToken; token = accessToken;
} else { } else {
token = config.settings.token; token = config.settings.token;
@@ -98,12 +106,14 @@ export const ConnectionProvider = ({
config.settings.token, config.settings.token,
config.settings.ws_url, config.settings.ws_url,
config.settings.room_name, config.settings.room_name,
config.settings.participant_name,
config.settings.agent_name,
config.settings.participant_id, config.settings.participant_id,
config.settings.metadata, config.settings.metadata,
config.settings.attributes, config.settings.attributes,
generateToken, generateToken,
setToastMessage, setToastMessage,
] ],
); );
const disconnect = useCallback(async () => { const disconnect = useCallback(async () => {
@@ -132,4 +142,4 @@ export const useConnection = () => {
throw new Error("useConnection must be used within a ConnectionProvider"); throw new Error("useConnection must be used within a ConnectionProvider");
} }
return context; return context;
} };

View File

@@ -62,7 +62,7 @@ export const useMultibandTrackVolume = (
track?: Track, track?: Track,
bands: number = 5, bands: number = 5,
loPass: number = 100, loPass: number = 100,
hiPass: number = 600 hiPass: number = 600,
) => { ) => {
const [frequencyBands, setFrequencyBands] = useState<Float32Array[]>([]); const [frequencyBands, setFrequencyBands] = useState<Float32Array[]>([]);
@@ -93,7 +93,7 @@ export const useMultibandTrackVolume = (
const chunks: Float32Array[] = []; const chunks: Float32Array[] = [];
for (let i = 0; i < bands; i++) { for (let i = 0; i < bands; i++) {
chunks.push( chunks.push(
normalizedFrequencies.slice(i * chunkSize, (i + 1) * chunkSize) normalizedFrequencies.slice(i * chunkSize, (i + 1) * chunkSize),
); );
} }

View File

@@ -19,4 +19,4 @@ export interface AttributeItem {
id: string; id: string;
key: string; key: string;
value: string; value: string;
} }

View File

@@ -2,21 +2,36 @@ import { NextApiRequest, NextApiResponse } from "next";
import { generateRandomAlphanumeric } from "@/lib/util"; import { generateRandomAlphanumeric } from "@/lib/util";
import { AccessToken } from "livekit-server-sdk"; import { AccessToken } from "livekit-server-sdk";
import { RoomAgentDispatch, RoomConfiguration } from "@livekit/protocol";
import type { AccessTokenOptions, VideoGrant } from "livekit-server-sdk"; import type { AccessTokenOptions, VideoGrant } from "livekit-server-sdk";
import { TokenResult } from "../../lib/types"; import { TokenResult } from "../../lib/types";
const apiKey = process.env.LIVEKIT_API_KEY; const apiKey = process.env.LIVEKIT_API_KEY;
const apiSecret = process.env.LIVEKIT_API_SECRET; const apiSecret = process.env.LIVEKIT_API_SECRET;
const createToken = (userInfo: AccessTokenOptions, grant: VideoGrant) => { const createToken = (
userInfo: AccessTokenOptions,
grant: VideoGrant,
agentName?: string,
) => {
const at = new AccessToken(apiKey, apiSecret, userInfo); const at = new AccessToken(apiKey, apiSecret, userInfo);
at.addGrant(grant); at.addGrant(grant);
if (agentName) {
at.roomConfig = new RoomConfiguration({
agents: [
new RoomAgentDispatch({
agentName: agentName,
metadata: '{"user_id": "12345"}',
}),
],
});
}
return at.toJwt(); return at.toJwt();
}; };
export default async function handleToken( export default async function handleToken(
req: NextApiRequest, req: NextApiRequest,
res: NextApiResponse res: NextApiResponse,
) { ) {
try { try {
if (req.method !== "POST") { if (req.method !== "POST") {
@@ -32,34 +47,47 @@ export default async function handleToken(
const { const {
roomName: roomNameFromBody, roomName: roomNameFromBody,
participantName, participantName: participantNameFromBody,
participantId: participantIdFromBody, participantId: participantIdFromBody,
metadata: metadataFromBody, metadata: metadataFromBody,
attributes: attributesFromBody, attributes: attributesFromBody,
agentName: agentNameFromBody,
} = req.body; } = req.body;
// Get room name from query params or generate random one // Get room name from query params or generate random one
const roomName = roomNameFromBody as string || const roomName =
(roomNameFromBody as string) ||
`room-${generateRandomAlphanumeric(4)}-${generateRandomAlphanumeric(4)}`; `room-${generateRandomAlphanumeric(4)}-${generateRandomAlphanumeric(4)}`;
// Get participant name from query params or generate random one // Get participant name from query params or generate random one
const identity = participantIdFromBody as string || const identity =
(participantIdFromBody as string) ||
`identity-${generateRandomAlphanumeric(4)}`; `identity-${generateRandomAlphanumeric(4)}`;
// Get agent name from query params or use none (automatic dispatch)
const agentName = (agentNameFromBody as string) || undefined;
// Get metadata and attributes from query params // Get metadata and attributes from query params
const metadata = metadataFromBody as string | undefined; const metadata = metadataFromBody as string | undefined;
const attributesStr = attributesFromBody as string | undefined; const attributesStr = attributesFromBody as string | undefined;
const attributes = attributesStr || {}; const attributes = attributesStr || {};
const participantName = participantNameFromBody || identity;
const grant: VideoGrant = { const grant: VideoGrant = {
room: roomName, room: roomName,
roomJoin: true, roomJoin: true,
canPublish: true, canPublish: true,
canPublishData: true, canPublishData: true,
canSubscribe: true, canSubscribe: true,
canUpdateOwnMetadata: true,
}; };
const token = await createToken({ identity, metadata, attributes, name: participantName }, grant); const token = await createToken(
{ identity, metadata, attributes, name: participantName },
grant,
agentName,
);
const result: TokenResult = { const result: TokenResult = {
identity, identity,
accessToken: token, accessToken: token,
@@ -70,4 +98,4 @@ export default async function handleToken(
res.statusMessage = (e as Error).message; res.statusMessage = (e as Error).message;
res.status(500).end(); res.status(500).end();
} }
} }

View File

@@ -12,7 +12,11 @@ import { PlaygroundConnect } from "@/components/PlaygroundConnect";
import Playground from "@/components/playground/Playground"; import Playground from "@/components/playground/Playground";
import { PlaygroundToast, ToastType } from "@/components/toast/PlaygroundToast"; import { PlaygroundToast, ToastType } from "@/components/toast/PlaygroundToast";
import { ConfigProvider, useConfig } from "@/hooks/useConfig"; import { ConfigProvider, useConfig } from "@/hooks/useConfig";
import { ConnectionMode, ConnectionProvider, useConnection } from "@/hooks/useConnection"; import {
ConnectionMode,
ConnectionProvider,
useConnection,
} from "@/hooks/useConnection";
import { useMemo } from "react"; import { useMemo } from "react";
import { ToastProvider, useToast } from "@/components/toast/ToasterProvider"; import { ToastProvider, useToast } from "@/components/toast/ToasterProvider";
@@ -44,26 +48,26 @@ export default function Home() {
export function HomeInner() { export function HomeInner() {
const { shouldConnect, wsUrl, token, mode, connect, disconnect } = const { shouldConnect, wsUrl, token, mode, connect, disconnect } =
useConnection(); useConnection();
const {config} = useConfig(); const { config } = useConfig();
const { toastMessage, setToastMessage } = useToast(); const { toastMessage, setToastMessage } = useToast();
const handleConnect = useCallback( const handleConnect = useCallback(
async (c: boolean, mode: ConnectionMode) => { async (c: boolean, mode: ConnectionMode) => {
c ? connect(mode) : disconnect(); c ? connect(mode) : disconnect();
}, },
[connect, disconnect] [connect, disconnect],
); );
const showPG = useMemo(() => { const showPG = useMemo(() => {
if (process.env.NEXT_PUBLIC_LIVEKIT_URL) { if (process.env.NEXT_PUBLIC_LIVEKIT_URL) {
return true; return true;
} }
if(wsUrl) { if (wsUrl) {
return true; return true;
} }
return false; return false;
}, [wsUrl]) }, [wsUrl]);
return ( return (
<> <>
@@ -129,4 +133,4 @@ export function HomeInner() {
</main> </main>
</> </>
); );
} }

View File

@@ -29,7 +29,7 @@ export function TranscriptionTile({
}); });
const [transcripts, setTranscripts] = useState<Map<string, ChatMessageType>>( const [transcripts, setTranscripts] = useState<Map<string, ChatMessageType>>(
new Map() new Map(),
); );
const [messages, setMessages] = useState<ChatMessageType[]>([]); const [messages, setMessages] = useState<ChatMessageType[]>([]);
const { chatMessages, send: sendChat } = useChat(); const { chatMessages, send: sendChat } = useChat();
@@ -43,21 +43,21 @@ export function TranscriptionTile({
segmentToChatMessage( segmentToChatMessage(
s, s,
transcripts.get(s.id), transcripts.get(s.id),
agentAudioTrack.participant agentAudioTrack.participant,
) ),
) ),
); );
} }
localMessages.segments.forEach((s) => localMessages.segments.forEach((s) =>
transcripts.set( transcripts.set(
s.id, s.id,
segmentToChatMessage( segmentToChatMessage(
s, s,
transcripts.get(s.id), transcripts.get(s.id),
localParticipant.localParticipant localParticipant.localParticipant,
) ),
) ),
); );
const allMessages = Array.from(transcripts.values()); const allMessages = Array.from(transcripts.values());
@@ -104,7 +104,7 @@ export function TranscriptionTile({
function segmentToChatMessage( function segmentToChatMessage(
s: TranscriptionSegment, s: TranscriptionSegment,
existingMessage: ChatMessageType | undefined, existingMessage: ChatMessageType | undefined,
participant: Participant participant: Participant,
): ChatMessageType { ): ChatMessageType {
const msg: ChatMessageType = { const msg: ChatMessageType = {
message: s.final ? s.text : `${s.text} ...`, message: s.final ? s.text : `${s.text} ...`,

View File

@@ -1,9 +1,42 @@
/** @type {import('tailwindcss').Config} */ /** @type {import('tailwindcss').Config} */
const colors = require('tailwindcss/colors') const colors = require("tailwindcss/colors");
const shades = ['50', '100', '200', '300', '400', '500', '600', '700', '800', '900', '950']; const shades = [
const colorList = ['gray', 'green', 'cyan', 'amber', 'violet', 'blue', 'rose', 'pink', 'teal', "red"]; "50",
const uiElements = ['bg', 'selection:bg', 'border', 'text', 'hover:bg', 'hover:border', 'hover:text', 'ring', 'focus:ring']; "100",
"200",
"300",
"400",
"500",
"600",
"700",
"800",
"900",
"950",
];
const colorList = [
"gray",
"green",
"cyan",
"amber",
"violet",
"blue",
"rose",
"pink",
"teal",
"red",
];
const uiElements = [
"bg",
"selection:bg",
"border",
"text",
"hover:bg",
"hover:border",
"hover:text",
"ring",
"focus:ring",
];
const customColors = { const customColors = {
cyan: colors.cyan, cyan: colors.cyan,
green: colors.green, green: colors.green,
@@ -32,44 +65,51 @@ for (const [name, color] of Object.entries(customColors)) {
} }
const safelist = [ const safelist = [
'bg-black', "bg-black",
'bg-white', "bg-white",
'transparent', "transparent",
'object-cover', "object-cover",
'object-contain', "object-contain",
...shadowNames, ...shadowNames,
...textShadowNames, ...textShadowNames,
...shades.flatMap(shade => [ ...shades.flatMap((shade) => [
...colorList.flatMap(color => [ ...colorList.flatMap((color) => [
...uiElements.flatMap(element => [ ...uiElements.flatMap((element) => [`${element}-${color}-${shade}`]),
`${element}-${color}-${shade}`,
]),
]), ]),
]), ]),
]; ];
module.exports = { module.exports = {
content: [ content: ["./src/**/*.{js,ts,jsx,tsx,mdx}"],
"./src/**/*.{js,ts,jsx,tsx,mdx}",
],
theme: { theme: {
colors: { colors: {
transparent: 'transparent', transparent: "transparent",
current: 'currentColor', current: "currentColor",
black: colors.black, black: colors.black,
white: colors.white, white: colors.white,
gray: colors.neutral, gray: colors.neutral,
...customColors ...customColors,
}, },
extend: { extend: {
dropShadow: { dropShadow: {
...textShadows, ...textShadows,
}, },
boxShadow: { boxShadow: {
...customShadows, ...customShadows,
} },
} keyframes: {
"fade-in-out": {
"0%": { opacity: "0" },
"20%": { opacity: "1" },
"80%": { opacity: "1" },
"100%": { opacity: "0" },
},
},
animation: {
"fade-in-out": "fade-in-out 1s ease-in-out",
},
},
}, },
plugins: [], plugins: [],
safelist, safelist,
}; };