Add attributes inspector and explicit dispatch support (#144)

This commit is contained in:
Ben Cherry
2025-05-29 15:26:04 -07:00
committed by GitHub
parent 5eddfa935c
commit 0218a5a002
30 changed files with 1208 additions and 692 deletions

View File

@@ -1 +1 @@
Files in this `cloud/` directory can be ignored. They are mocks which we override in our private, hosted version of the agents-playground that supports LiveKit Cloud authentication.
Files in this `cloud/` directory can be ignored. They are mocks which we override in our private, hosted version of the agents-playground that supports LiveKit Cloud authentication.

View File

@@ -9,4 +9,4 @@ export function useCloud() {
const wsUrl = "";
return { generateToken, wsUrl };
}
}

View File

@@ -0,0 +1,60 @@
import React from "react";
import { AttributeItem } from "@/lib/types";
interface AttributeRowProps {
attribute: AttributeItem;
onKeyChange: (id: string, newKey: string) => void;
onValueChange: (id: string, newValue: string) => void;
onRemove?: (id: string) => void;
disabled?: boolean;
}
export const AttributeRow: React.FC<AttributeRowProps> = ({
attribute,
onKeyChange,
onValueChange,
onRemove,
disabled = false,
}) => {
return (
<div className="flex items-center gap-2 mb-2">
<input
value={attribute.key}
onChange={(e) => onKeyChange(attribute.id, e.target.value)}
className="flex-1 min-w-0 text-gray-400 text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-1 font-mono"
placeholder="Name"
disabled={disabled}
/>
<input
value={attribute.value}
onChange={(e) => onValueChange(attribute.id, e.target.value)}
className="flex-1 min-w-0 text-gray-400 text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-1 font-mono"
placeholder="Value"
disabled={disabled}
/>
{onRemove && (
<button
onClick={() => onRemove(attribute.id)}
className="flex-shrink-0 w-6 h-6 flex items-center justify-center text-gray-400 hover:text-white"
disabled={disabled}
style={{ display: disabled ? "none" : "flex" }}
>
<svg
xmlns="http://www.w3.org/2000/svg"
className="h-4 w-4"
fill="none"
viewBox="0 0 24 24"
stroke="currentColor"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M6 18L18 6M6 6l12 12"
/>
</svg>
</button>
)}
</div>
);
};

View File

@@ -1,93 +0,0 @@
import React from 'react';
import { ConnectionState } from 'livekit-client';
import { AttributeItem } from '@/lib/types';
interface AttributesEditorProps {
attributes: AttributeItem[];
onAttributesChange: (attributes: AttributeItem[]) => void;
themeColor: string;
disabled?: boolean;
connectionState?: ConnectionState;
}
export const AttributesEditor: React.FC<AttributesEditorProps> = ({
attributes,
onAttributesChange,
themeColor,
disabled = false,
connectionState
}) => {
const handleKeyChange = (id: string, newKey: string) => {
const updatedAttributes = attributes.map(attr =>
attr.id === id ? { ...attr, key: newKey } : attr
);
onAttributesChange(updatedAttributes);
};
const handleValueChange = (id: string, newValue: string) => {
const updatedAttributes = attributes.map(attr =>
attr.id === id ? { ...attr, value: newValue } : attr
);
onAttributesChange(updatedAttributes);
};
const handleRemoveAttribute = (id: string) => {
const updatedAttributes = attributes.filter(attr => attr.id !== id);
onAttributesChange(updatedAttributes);
};
const handleAddAttribute = () => {
const newId = `attr_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
const updatedAttributes = [...attributes, { id: newId, key: "", value: "" }];
onAttributesChange(updatedAttributes);
};
return (
<div className="border border-gray-800 rounded-sm p-3 bg-gray-900/30">
{attributes.map((attribute) => (
<div key={attribute.id} className="flex items-center gap-2 mb-2">
<input
value={attribute.key}
onChange={(e) => handleKeyChange(attribute.id, e.target.value)}
className="flex-1 min-w-0 text-white text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-1"
placeholder="Key"
disabled={disabled}
/>
<input
value={attribute.value}
onChange={(e) => handleValueChange(attribute.id, e.target.value)}
className="flex-1 min-w-0 text-white text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-1"
placeholder="Value"
disabled={disabled}
/>
<button
onClick={() => handleRemoveAttribute(attribute.id)}
className="flex-shrink-0 w-6 h-6 flex items-center justify-center text-gray-400 hover:text-white"
disabled={disabled}
style={{ display: disabled ? 'none' : 'flex' }}
>
<svg xmlns="http://www.w3.org/2000/svg" className="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M6 18L18 6M6 6l12 12" />
</svg>
</button>
</div>
))}
<button
onClick={handleAddAttribute}
className={`text-xs py-1 px-2 rounded-sm flex items-center gap-1 ${
disabled
? "bg-gray-700 text-gray-500 cursor-not-allowed"
: `bg-${themeColor}-500 hover:bg-${themeColor}-600 text-white`
}`}
disabled={disabled}
>
<svg xmlns="http://www.w3.org/2000/svg" className="h-3 w-3" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 4v16m8-8H4" />
</svg>
Add Attribute
</button>
</div>
);
};

View File

@@ -0,0 +1,268 @@
import React, { useState, useCallback, useEffect, useRef } from "react";
import { ConnectionState } from "livekit-client";
import { AttributeItem } from "@/lib/types";
import { Button } from "@/components/button/Button";
import { useLocalParticipant } from "@livekit/components-react";
import { AttributeRow } from "./AttributeRow";
interface AttributesInspectorProps {
attributes: AttributeItem[];
onAttributesChange: (attributes: AttributeItem[]) => void;
themeColor: string;
disabled?: boolean;
connectionState?: ConnectionState;
metadata?: string;
onMetadataChange?: (metadata: string) => void;
}
export const AttributesInspector: React.FC<AttributesInspectorProps> = ({
attributes,
onAttributesChange,
themeColor,
disabled = false,
connectionState,
metadata,
onMetadataChange,
}) => {
const [isExpanded, setIsExpanded] = useState(false);
const [isMetadataExpanded, setIsMetadataExpanded] = useState(false);
const [localAttributes, setLocalAttributes] =
useState<AttributeItem[]>(attributes);
const [hasUnsavedChanges, setHasUnsavedChanges] = useState(false);
const [showSyncFlash, setShowSyncFlash] = useState(false);
const { localParticipant } = useLocalParticipant();
const timeoutRef = useRef<NodeJS.Timeout>();
const syncFlashTimeoutRef = useRef<NodeJS.Timeout>();
// Update local attributes when props change
useEffect(() => {
setLocalAttributes(attributes);
}, [attributes]);
const syncAttributesWithRoom = useCallback(() => {
if (!localParticipant || connectionState !== ConnectionState.Connected)
return;
const attributesMap = localAttributes.reduce(
(acc, attr) => {
if (attr.key && attr.key.trim() !== "") {
acc[attr.key] = attr.value;
}
return acc;
},
{} as Record<string, string>,
);
localParticipant.setAttributes(attributesMap);
setHasUnsavedChanges(false);
setShowSyncFlash(true);
if (syncFlashTimeoutRef.current) {
clearTimeout(syncFlashTimeoutRef.current);
}
syncFlashTimeoutRef.current = setTimeout(
() => setShowSyncFlash(false),
1000,
);
}, [localAttributes, localParticipant, connectionState]);
// Handle debounced sync
useEffect(() => {
if (!hasUnsavedChanges) return;
if (timeoutRef.current) {
clearTimeout(timeoutRef.current);
}
timeoutRef.current = setTimeout(() => {
if (connectionState === ConnectionState.Connected && localParticipant) {
syncAttributesWithRoom();
}
}, 2000);
return () => {
if (timeoutRef.current) {
clearTimeout(timeoutRef.current);
}
};
}, [
hasUnsavedChanges,
syncAttributesWithRoom,
connectionState,
localParticipant,
]);
const handleKeyChange = (id: string, newKey: string) => {
const updatedAttributes = localAttributes.map((attr) =>
attr.id === id ? { ...attr, key: newKey } : attr,
);
setLocalAttributes(updatedAttributes);
onAttributesChange(updatedAttributes);
if (connectionState === ConnectionState.Connected && newKey.trim() !== "") {
setHasUnsavedChanges(true);
}
};
const handleValueChange = (id: string, newValue: string) => {
const updatedAttributes = localAttributes.map((attr) =>
attr.id === id ? { ...attr, value: newValue } : attr,
);
setLocalAttributes(updatedAttributes);
onAttributesChange(updatedAttributes);
if (connectionState === ConnectionState.Connected) {
setHasUnsavedChanges(true);
}
};
const handleRemoveAttribute = (id: string) => {
const updatedAttributes = localAttributes.filter((attr) => attr.id !== id);
setLocalAttributes(updatedAttributes);
onAttributesChange(updatedAttributes);
if (connectionState === ConnectionState.Connected) {
setHasUnsavedChanges(true);
}
};
const handleAddAttribute = () => {
const newId = `attr_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
const updatedAttributes = [
...localAttributes,
{ id: newId, key: "", value: "" },
];
setLocalAttributes(updatedAttributes);
onAttributesChange(updatedAttributes);
if (connectionState === ConnectionState.Connected) {
setHasUnsavedChanges(true);
}
};
return (
<div>
<div
className="flex items-center justify-between mb-2 cursor-pointer"
onClick={() => setIsExpanded(!isExpanded)}
>
<div className="text-sm text-gray-500">Attributes</div>
<svg
xmlns="http://www.w3.org/2000/svg"
className={`h-4 w-4 text-gray-500 transition-transform ${isExpanded ? "rotate-180" : ""}`}
fill="none"
viewBox="0 0 24 24"
stroke="currentColor"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M19 9l-7 7-7-7"
/>
</svg>
</div>
{isExpanded && (
<div className="border border-gray-800 rounded-sm bg-gray-900/30 p-3 mb-2">
{disabled ? (
localAttributes.length === 0 ? (
<div className="text-sm text-gray-400 font-sans">
No attributes set
</div>
) : (
localAttributes.map((attribute) => (
<AttributeRow
key={attribute.id}
attribute={attribute}
onKeyChange={handleKeyChange}
onValueChange={handleValueChange}
disabled={true}
/>
))
)
) : (
<>
{localAttributes.map((attribute) => (
<AttributeRow
key={attribute.id}
attribute={attribute}
onKeyChange={handleKeyChange}
onValueChange={handleValueChange}
onRemove={handleRemoveAttribute}
disabled={disabled}
/>
))}
<div className="flex justify-between items-center">
<Button
accentColor={themeColor}
onClick={handleAddAttribute}
className="text-xs flex items-center gap-1"
>
<svg
xmlns="http://www.w3.org/2000/svg"
className="h-3 w-3"
fill="none"
viewBox="0 0 24 24"
stroke="currentColor"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M12 4v16m8-8H4"
/>
</svg>
Attribute
</Button>
{showSyncFlash && (
<div className="text-xs text-gray-400 animate-fade-in-out">
Changes saved
</div>
)}
</div>
</>
)}
</div>
)}
<>
<div
className="flex items-center justify-between mb-2 cursor-pointer"
onClick={() => setIsMetadataExpanded(!isMetadataExpanded)}
>
<div className="text-sm text-gray-500">Metadata</div>
<svg
xmlns="http://www.w3.org/2000/svg"
className={`h-4 w-4 text-gray-500 transition-transform ${isMetadataExpanded ? "rotate-180" : ""}`}
fill="none"
viewBox="0 0 24 24"
stroke="currentColor"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M19 9l-7 7-7-7"
/>
</svg>
</div>
{isMetadataExpanded &&
(disabled || connectionState === ConnectionState.Connected ? (
<div className="border border-gray-800 rounded-sm bg-gray-900/30 px-3 py-2 mb-4 min-h-[40px] flex items-center">
{metadata ? (
<pre className="w-full text-gray-400 text-xs bg-transparent font-mono whitespace-pre-wrap break-words m-0 p-0 border-0">
{metadata}
</pre>
) : (
<div className="text-sm text-gray-400 font-sans w-full text-left">
No metadata set
</div>
)}
</div>
) : (
<textarea
value={metadata}
onChange={(e) => onMetadataChange?.(e.target.value)}
className="w-full text-gray-400 text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-2 font-mono mb-4"
placeholder="Enter metadata..."
rows={3}
/>
))}
</>
</div>
);
};

View File

@@ -1,4 +1,4 @@
import { ReactNode } from "react";
import { ReactNode, useState } from "react";
import { PlaygroundDeviceSelector } from "@/components/playground/PlaygroundDeviceSelector";
import { TrackToggle } from "@livekit/components-react";
import { Track } from "livekit-client";
@@ -7,35 +7,65 @@ type ConfigurationPanelItemProps = {
title: string;
children?: ReactNode;
source?: Track.Source;
collapsible?: boolean;
defaultCollapsed?: boolean;
};
export const ConfigurationPanelItem: React.FC<ConfigurationPanelItemProps> = ({
children,
title,
source,
collapsible = false,
defaultCollapsed = false,
}) => {
const [isCollapsed, setIsCollapsed] = useState(defaultCollapsed);
return (
<div className="w-full text-gray-300 py-4 border-b border-b-gray-800 relative">
<div className="flex flex-row justify-between items-center px-4 text-xs uppercase tracking-wider">
<h3>{title}</h3>
{source && (
<span className="flex flex-row gap-2">
<TrackToggle
className="px-2 py-1 bg-gray-900 text-gray-300 border border-gray-800 rounded-sm hover:bg-gray-800"
source={source}
/>
{source === Track.Source.Camera && (
<PlaygroundDeviceSelector kind="videoinput" />
)}
{source === Track.Source.Microphone && (
<PlaygroundDeviceSelector kind="audioinput" />
)}
</span>
)}
</div>
<div className="px-4 py-2 text-xs text-gray-500 leading-normal">
{children}
<div className="flex items-center gap-2">
{source && (
<span className="flex flex-row gap-2">
<TrackToggle
className="px-2 py-1 bg-gray-900 text-gray-300 border border-gray-800 rounded-sm hover:bg-gray-800"
source={source}
/>
{source === Track.Source.Camera && (
<PlaygroundDeviceSelector kind="videoinput" />
)}
{source === Track.Source.Microphone && (
<PlaygroundDeviceSelector kind="audioinput" />
)}
</span>
)}
{collapsible && (
<button
onClick={() => setIsCollapsed(!isCollapsed)}
className="text-gray-400 hover:text-gray-300 transition-colors"
>
<svg
className={`w-4 h-4 transform transition-transform ${!isCollapsed ? "rotate-180" : ""}`}
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M19 9l-7 7-7-7"
/>
</svg>
</button>
)}
</div>
</div>
{!isCollapsed && (
<div className="px-4 py-2 text-xs text-gray-500 leading-normal">
{children}
</div>
)}
</div>
);
};

View File

@@ -52,11 +52,5 @@ export const EditableNameValueRow: React.FC<EditableNameValueRowProps> = ({
</div>
);
}
return (
<NameValueRow
name={name}
value={value}
valueColor={valueColor}
/>
);
return <NameValueRow name={name} value={value} valueColor={valueColor} />;
};

View File

@@ -24,13 +24,15 @@ import {
useTracks,
useVoiceAssistant,
useRoomContext,
useParticipantAttributes,
} from "@livekit/components-react";
import { ConnectionState, LocalParticipant, Track } from "livekit-client";
import { QRCodeSVG } from "qrcode.react";
import { ReactNode, useCallback, useEffect, useMemo, useState } from "react";
import tailwindTheme from "../../lib/tailwindTheme.preval";
import { EditableNameValueRow } from "@/components/config/NameValueRow";
import { AttributesEditor } from "@/components/config/AttributesEditor";
import { AttributesInspector } from "@/components/config/AttributesInspector";
import { RpcPanel } from "./RpcPanel";
export interface PlaygroundMeta {
name: string;
@@ -63,6 +65,7 @@ export default function Playground({
const [rpcMethod, setRpcMethod] = useState("");
const [rpcPayload, setRpcPayload] = useState("");
const [showRpc, setShowRpc] = useState(false);
useEffect(() => {
if (roomState === ConnectionState.Connected) {
@@ -74,27 +77,27 @@ export default function Playground({
const agentVideoTrack = tracks.find(
(trackRef) =>
trackRef.publication.kind === Track.Kind.Video &&
trackRef.participant.isAgent
trackRef.participant.isAgent,
);
const localTracks = tracks.filter(
({ participant }) => participant instanceof LocalParticipant
({ participant }) => participant instanceof LocalParticipant,
);
const localCameraTrack = localTracks.find(
({ source }) => source === Track.Source.Camera
({ source }) => source === Track.Source.Camera,
);
const localScreenTrack = localTracks.find(
({ source }) => source === Track.Source.ScreenShare
({ source }) => source === Track.Source.ScreenShare,
);
const localMicTrack = localTracks.find(
({ source }) => source === Track.Source.Microphone
({ source }) => source === Track.Source.Microphone,
);
const onDataReceived = useCallback(
(msg: any) => {
if (msg.topic === "transcription") {
const decoded = JSON.parse(
new TextDecoder("utf-8").decode(msg.payload)
new TextDecoder("utf-8").decode(msg.payload),
);
let timestamp = new Date().getTime();
if ("timestamp" in decoded && decoded.timestamp > 0) {
@@ -111,7 +114,7 @@ export default function Playground({
]);
}
},
[transcripts]
[transcripts],
);
useDataChannel(onDataReceived);
@@ -121,14 +124,14 @@ export default function Playground({
const disconnectedContent = (
<div className="flex items-center justify-center text-gray-700 text-center w-full h-full">
No video track. Connect to get started.
No agent video track. Connect to get started.
</div>
);
const loadingContent = (
<div className="flex flex-col items-center justify-center gap-2 text-gray-700 text-center h-full w-full">
<LoadingSVG />
Waiting for video track
Waiting for agent video track
</div>
);
@@ -159,25 +162,25 @@ export default function Playground({
document.body.style.setProperty(
"--lk-theme-color",
// @ts-ignore
tailwindTheme.colors[config.settings.theme_color]["500"]
tailwindTheme.colors[config.settings.theme_color]["500"],
);
document.body.style.setProperty(
"--lk-drop-shadow",
`var(--lk-theme-color) 0px 0px 18px`
`var(--lk-theme-color) 0px 0px 18px`,
);
}, [config.settings.theme_color]);
const audioTileContent = useMemo(() => {
const disconnectedContent = (
<div className="flex flex-col items-center justify-center gap-2 text-gray-700 text-center w-full">
No audio track. Connect to get started.
No agent audio track. Connect to get started.
</div>
);
const waitingContent = (
<div className="flex flex-col items-center gap-2 text-gray-700 text-center w-full">
<LoadingSVG />
Waiting for audio track
Waiting for agent audio track
</div>
);
@@ -220,145 +223,63 @@ export default function Playground({
);
}
return <></>;
}, [config.settings.theme_color, voiceAssistant.audioTrack, voiceAssistant.agent]);
}, [
config.settings.theme_color,
voiceAssistant.audioTrack,
voiceAssistant.agent,
]);
const handleRpcCall = useCallback(async () => {
if (!voiceAssistant.agent || !room) return;
try {
const response = await room.localParticipant.performRpc({
destinationIdentity: voiceAssistant.agent.identity,
method: rpcMethod,
payload: rpcPayload,
});
console.log('RPC response:', response);
} catch (e) {
console.error('RPC call failed:', e);
if (!voiceAssistant.agent || !room) {
throw new Error("No agent or room available");
}
const response = await room.localParticipant.performRpc({
destinationIdentity: voiceAssistant.agent.identity,
method: rpcMethod,
payload: rpcPayload,
});
return response;
}, [room, rpcMethod, rpcPayload, voiceAssistant.agent]);
const agentAttributes = useParticipantAttributes({
participant: voiceAssistant.agent,
});
const settingsTileContent = useMemo(() => {
return (
<div className="flex flex-col gap-4 h-full w-full items-start overflow-y-auto">
<div className="flex flex-col h-full w-full items-start overflow-y-auto">
{config.description && (
<ConfigurationPanelItem title="Description">
{config.description}
</ConfigurationPanelItem>
)}
<ConfigurationPanelItem title="Settings">
<div className="flex flex-col gap-4">
<ConfigurationPanelItem title="Room">
<div className="flex flex-col gap-2">
<EditableNameValueRow
name="Room"
value={roomState === ConnectionState.Connected ? name : config.settings.room_name}
name="Room name"
value={
roomState === ConnectionState.Connected
? name
: config.settings.room_name
}
valueColor={`${config.settings.theme_color}-500`}
onValueChange={(value) => {
const newSettings = { ...config.settings };
newSettings.room_name = value;
setUserSettings(newSettings);
}}
placeholder="Enter room name"
placeholder="Auto"
editable={roomState !== ConnectionState.Connected}
/>
<EditableNameValueRow
name="Participant ID"
value={roomState === ConnectionState.Connected ?
(localParticipant?.identity || '') :
(config.settings.participant_id || '')}
valueColor={`${config.settings.theme_color}-500`}
onValueChange={(value) => {
const newSettings = { ...config.settings };
newSettings.participant_id = value;
setUserSettings(newSettings);
}}
placeholder="Enter participant id"
editable={roomState !== ConnectionState.Connected}
/>
<EditableNameValueRow
name="Participant Name"
value={roomState === ConnectionState.Connected ?
(localParticipant?.name || '') :
(config.settings.participant_name || '')}
valueColor={`${config.settings.theme_color}-500`}
onValueChange={(value) => {
const newSettings = { ...config.settings };
newSettings.participant_name = value;
setUserSettings(newSettings);
}}
placeholder="Enter participant name"
editable={roomState !== ConnectionState.Connected}
/>
</div>
<div className="flex flex-col gap-2 mt-4">
<div className="text-xs text-gray-500 mt-2">Metadata</div>
<textarea
value={config.settings.metadata || ""}
onChange={(e) => {
const newSettings = { ...config.settings };
newSettings.metadata = e.target.value;
setUserSettings(newSettings);
}}
className="w-full text-white text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-2"
placeholder="Custom metadata..."
rows={2}
disabled={roomState === ConnectionState.Connected}
/>
<div className="text-xs text-gray-500 mt-2">Attributes</div>
<AttributesEditor
attributes={config.settings.attributes || []}
onAttributesChange={(newAttributes) => {
const newSettings = { ...config.settings };
newSettings.attributes = newAttributes;
setUserSettings(newSettings);
}}
themeColor={config.settings.theme_color}
disabled={roomState === ConnectionState.Connected}
/>
</div>
<div className="flex flex-col gap-2 mt-4">
<div className="text-xs text-gray-500 mt-2">RPC Method</div>
<input
type="text"
value={rpcMethod}
onChange={(e) => setRpcMethod(e.target.value)}
className="w-full text-white text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-2"
placeholder="RPC method name"
/>
<div className="text-xs text-gray-500 mt-2">RPC Payload</div>
<textarea
value={rpcPayload}
onChange={(e) => setRpcPayload(e.target.value)}
className="w-full text-white text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-2"
placeholder="RPC payload"
rows={2}
/>
<button
onClick={handleRpcCall}
disabled={!voiceAssistant.agent || !rpcMethod}
className={`mt-2 px-2 py-1 rounded-sm text-xs
${voiceAssistant.agent && rpcMethod
? `bg-${config.settings.theme_color}-500 hover:bg-${config.settings.theme_color}-600`
: 'bg-gray-700 cursor-not-allowed'
} text-white`}
>
Perform RPC Call
</button>
</div>
</ConfigurationPanelItem>
<ConfigurationPanelItem title="Status">
<div className="flex flex-col gap-2">
<NameValueRow
name="Room connected"
name="Status"
value={
roomState === ConnectionState.Connecting ? (
<LoadingSVG diameter={16} strokeWidth={2} />
) : (
roomState.toUpperCase()
roomState.charAt(0).toUpperCase() + roomState.slice(1)
)
}
valueColor={
@@ -367,15 +288,36 @@ export default function Playground({
: "gray-500"
}
/>
</div>
</ConfigurationPanelItem>
<ConfigurationPanelItem title="Agent">
<div className="flex flex-col gap-2">
<EditableNameValueRow
name="Agent name"
value={
roomState === ConnectionState.Connected
? config.settings.agent_name || "None"
: config.settings.agent_name || ""
}
valueColor={`${config.settings.theme_color}-500`}
onValueChange={(value) => {
const newSettings = { ...config.settings };
newSettings.agent_name = value;
setUserSettings(newSettings);
}}
placeholder="None"
editable={roomState !== ConnectionState.Connected}
/>
<NameValueRow
name="Agent connected"
name="Identity"
value={
voiceAssistant.agent ? (
"TRUE"
voiceAssistant.agent.identity
) : roomState === ConnectionState.Connected ? (
<LoadingSVG diameter={12} strokeWidth={2} />
) : (
"FALSE"
"No agent connected"
)
}
valueColor={
@@ -384,32 +326,122 @@ export default function Playground({
: "gray-500"
}
/>
{roomState === ConnectionState.Connected &&
voiceAssistant.agent && (
<AttributesInspector
attributes={Object.entries(
agentAttributes.attributes || {},
).map(([key, value], index) => ({
id: `agent-attr-${index}`,
key,
value: String(value),
}))}
onAttributesChange={() => {}}
themeColor={config.settings.theme_color}
disabled={true}
/>
)}
<p className="text-xs text-gray-500 text-right">
Set an agent name to use{" "}
<a
href="https://docs.livekit.io/agents/worker/dispatch#explicit"
target="_blank"
rel="noopener noreferrer"
className="text-gray-500 hover:text-gray-300 underline"
>
explicit dispatch
</a>
.
</p>
</div>
</ConfigurationPanelItem>
{roomState === ConnectionState.Connected && config.settings.inputs.screen && (
<ConfigurationPanelItem
title="Screen"
source={Track.Source.ScreenShare}
>
{localScreenTrack ? (
<div className="relative">
<VideoTrack
className="rounded-sm border border-gray-800 opacity-70 w-full"
trackRef={localScreenTrack}
/>
</div>
) : (
<div className="flex items-center justify-center text-gray-700 text-center w-full h-full">
Press the button above to share your screen.
</div>
)}
</ConfigurationPanelItem>
<ConfigurationPanelItem title="User">
<div className="flex flex-col gap-2">
<EditableNameValueRow
name="Name"
value={
roomState === ConnectionState.Connected
? localParticipant?.name || ""
: config.settings.participant_name || ""
}
valueColor={`${config.settings.theme_color}-500`}
onValueChange={(value) => {
const newSettings = { ...config.settings };
newSettings.participant_name = value;
setUserSettings(newSettings);
}}
placeholder="Auto"
editable={roomState !== ConnectionState.Connected}
/>
<EditableNameValueRow
name="Identity"
value={
roomState === ConnectionState.Connected
? localParticipant?.identity || ""
: config.settings.participant_id || ""
}
valueColor={`${config.settings.theme_color}-500`}
onValueChange={(value) => {
const newSettings = { ...config.settings };
newSettings.participant_id = value;
setUserSettings(newSettings);
}}
placeholder="Auto"
editable={roomState !== ConnectionState.Connected}
/>
<AttributesInspector
attributes={config.settings.attributes || []}
onAttributesChange={(newAttributes) => {
const newSettings = { ...config.settings };
newSettings.attributes = newAttributes;
setUserSettings(newSettings);
}}
metadata={config.settings.metadata}
onMetadataChange={(metadata) => {
const newSettings = { ...config.settings };
newSettings.metadata = metadata;
setUserSettings(newSettings);
}}
themeColor={config.settings.theme_color}
disabled={false}
connectionState={roomState}
/>
</div>
</ConfigurationPanelItem>
{roomState === ConnectionState.Connected &&
config.settings.inputs.screen && (
<ConfigurationPanelItem
title="Screen"
source={Track.Source.ScreenShare}
>
{localScreenTrack ? (
<div className="relative">
<VideoTrack
className="rounded-sm border border-gray-800 opacity-70 w-full"
trackRef={localScreenTrack}
/>
</div>
) : (
<div className="flex items-center justify-center text-gray-700 text-center w-full h-full">
Press the button above to share your screen.
</div>
)}
</ConfigurationPanelItem>
)}
{roomState === ConnectionState.Connected && voiceAssistant.agent && (
<RpcPanel
config={config}
rpcMethod={rpcMethod}
rpcPayload={rpcPayload}
setRpcMethod={setRpcMethod}
setRpcPayload={setRpcPayload}
handleRpcCall={handleRpcCall}
/>
)}
{localCameraTrack && (
<ConfigurationPanelItem
title="Camera"
source={Track.Source.Camera}
>
<ConfigurationPanelItem title="Camera" source={Track.Source.Camera}>
<div className="relative">
<VideoTrack
className="rounded-sm border border-gray-800 opacity-70 w-full"
@@ -464,6 +496,8 @@ export default function Playground({
rpcMethod,
rpcPayload,
handleRpcCall,
showRpc,
setShowRpc,
]);
let mobileTabs: PlaygroundTab[] = [];
@@ -549,7 +583,7 @@ export default function Playground({
>
{config.settings.outputs.video && (
<PlaygroundTile
title="Video"
title="Agent Video"
className="w-full h-full grow"
childrenClassName="justify-center"
>
@@ -558,7 +592,7 @@ export default function Playground({
)}
{config.settings.outputs.audio && (
<PlaygroundTile
title="Audio"
title="Agent Audio"
className="w-full h-full grow"
childrenClassName="justify-center"
>

View File

@@ -66,7 +66,7 @@ export const PlaygroundTabbedTile: React.FC<PlaygroundTabbedTileProps> = ({
}) => {
const contentPadding = 4;
const [activeTab, setActiveTab] = useState(initialTab);
if(activeTab >= tabs.length) {
if (activeTab >= tabs.length) {
return null;
}
return (

View File

@@ -0,0 +1,107 @@
import { ConfigurationPanelItem } from "@/components/config/ConfigurationPanelItem";
import { useState } from "react";
import { LoadingSVG } from "@/components/button/LoadingSVG";
import { Button } from "@/components/button/Button";
interface RpcPanelProps {
config: any;
rpcMethod: string;
rpcPayload: string;
setRpcMethod: (method: string) => void;
setRpcPayload: (payload: string) => void;
handleRpcCall: () => Promise<any>;
}
export function RpcPanel({
config,
rpcMethod,
rpcPayload,
setRpcMethod,
setRpcPayload,
handleRpcCall,
}: RpcPanelProps) {
const [rpcResult, setRpcResult] = useState<{
success: boolean;
data: any;
} | null>(null);
const [isLoading, setIsLoading] = useState(false);
const handleCall = async () => {
setIsLoading(true);
setRpcResult(null);
try {
const result = await handleRpcCall();
setRpcResult({ success: true, data: result });
} catch (error) {
setRpcResult({
success: false,
data: error instanceof Error ? error.message : String(error),
});
} finally {
setIsLoading(false);
}
};
return (
<ConfigurationPanelItem
title="RPC"
collapsible={true}
defaultCollapsed={true}
>
<div className="flex flex-col gap-2">
<div className="text-xs text-gray-500 mt-2">Method Name</div>
<input
type="text"
value={rpcMethod}
onChange={(e) => setRpcMethod(e.target.value)}
className="w-full text-white text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-2"
placeholder="my_method"
/>
<div className="text-xs text-gray-500 mt-2">Payload</div>
<textarea
value={rpcPayload}
onChange={(e) => setRpcPayload(e.target.value)}
className="w-full text-white text-sm bg-transparent border border-gray-800 rounded-sm px-3 py-2"
placeholder='{"my": "payload"}'
rows={2}
/>
<Button
accentColor={config.settings.theme_color}
onClick={handleCall}
disabled={!rpcMethod || isLoading}
className="mt-2 text-xs flex items-center justify-center gap-2"
>
{isLoading ? (
<>
<LoadingSVG diameter={12} strokeWidth={2} />
Performing RPC...
</>
) : (
"Perform RPC"
)}
</Button>
{rpcResult && (
<>
<div className="text-xs text-gray-500 mt-2">
{rpcResult.success ? "Result" : "Error"}
</div>
<div
className={`w-full text-sm bg-transparent border rounded-sm px-3 py-2 ${
rpcResult.success
? "border-green-500 text-green-500"
: "border-red-500 text-red-500"
}`}
>
{typeof rpcResult.data === "object"
? JSON.stringify(rpcResult.data, null, 2)
: String(rpcResult.data)}
</div>
</>
)}
</div>
</ConfigurationPanelItem>
);
}

View File

@@ -2,7 +2,7 @@ import * as DropdownMenu from "@radix-ui/react-dropdown-menu";
import { CheckIcon, ChevronIcon } from "./icons";
import { useConfig } from "@/hooks/useConfig";
type SettingType = "inputs" | "outputs" | "chat" | "theme_color"
type SettingType = "inputs" | "outputs" | "chat" | "theme_color";
type SettingValue = {
title: string;
@@ -55,18 +55,19 @@ const settingsDropdown: SettingValue[] = [
];
export const SettingsDropdown = () => {
const {config, setUserSettings} = useConfig();
const { config, setUserSettings } = useConfig();
const isEnabled = (setting: SettingValue) => {
if (setting.type === "separator" || setting.type === "theme_color") return false;
if (setting.type === "separator" || setting.type === "theme_color")
return false;
if (setting.type === "chat") {
return config.settings[setting.type];
}
if(setting.type === "inputs") {
if (setting.type === "inputs") {
const key = setting.key as "camera" | "mic" | "screen";
return config.settings.inputs[key];
} else if(setting.type === "outputs") {
} else if (setting.type === "outputs") {
const key = setting.key as "video" | "audio";
return config.settings.outputs[key];
}
@@ -77,13 +78,13 @@ export const SettingsDropdown = () => {
const toggleSetting = (setting: SettingValue) => {
if (setting.type === "separator" || setting.type === "theme_color") return;
const newValue = !isEnabled(setting);
const newSettings = {...config.settings}
const newSettings = { ...config.settings };
if(setting.type === "chat") {
if (setting.type === "chat") {
newSettings.chat = newValue;
} else if(setting.type === "inputs") {
} else if (setting.type === "inputs") {
newSettings.inputs[setting.key as "camera" | "mic" | "screen"] = newValue;
} else if(setting.type === "outputs") {
} else if (setting.type === "outputs") {
newSettings.outputs[setting.key as "video" | "audio"] = newValue;
}
setUserSettings(newSettings);
@@ -91,11 +92,9 @@ export const SettingsDropdown = () => {
return (
<DropdownMenu.Root modal={false}>
<DropdownMenu.Trigger className="group inline-flex max-h-12 items-center gap-1 rounded-md hover:bg-gray-800 bg-gray-900 border-gray-800 p-1 pr-2 text-gray-100">
<button className="my-auto text-sm flex gap-1 pl-2 py-1 h-full items-center">
Settings
<ChevronIcon />
</button>
<DropdownMenu.Trigger className="group inline-flex max-h-12 items-center gap-1 rounded-md hover:bg-gray-800 bg-gray-900 border-gray-800 p-1 pr-2 text-gray-100 my-auto text-sm flex gap-1 pl-2 py-1 h-full items-center">
Settings
<ChevronIcon />
</DropdownMenu.Trigger>
<DropdownMenu.Portal>
<DropdownMenu.Content
@@ -130,4 +129,4 @@ export const SettingsDropdown = () => {
</DropdownMenu.Portal>
</DropdownMenu.Root>
);
};
};

View File

@@ -13,8 +13,8 @@ export const PlaygroundToast = () => {
toastMessage?.type === "error"
? "red"
: toastMessage?.type === "success"
? "green"
: "amber";
? "green"
: "amber";
return (
<div

View File

@@ -1,29 +1,28 @@
"use client"
"use client";
import React, { createContext, useState } from "react";
import { ToastType } from "./PlaygroundToast";
type ToastProviderData = {
setToastMessage: (
message: { message: string; type: ToastType } | null
message: { message: string; type: ToastType } | null,
) => void;
toastMessage: { message: string; type: ToastType } | null;
};
const ToastContext = createContext<ToastProviderData | undefined>(undefined);
export const ToastProvider = ({
children,
}: {
children: React.ReactNode;
}) => {
const [toastMessage, setToastMessage] = useState<{message: string, type: ToastType} | null>(null);
export const ToastProvider = ({ children }: { children: React.ReactNode }) => {
const [toastMessage, setToastMessage] = useState<{
message: string;
type: ToastType;
} | null>(null);
return (
<ToastContext.Provider
value={{
toastMessage,
setToastMessage
setToastMessage,
}}
>
{children}
@@ -37,4 +36,4 @@ export const useToast = () => {
throw new Error("useToast must be used within a ToastProvider");
}
return context;
}
};

View File

@@ -39,6 +39,7 @@ export type UserSettings = {
room_name: string;
participant_id: string;
participant_name: string;
agent_name?: string;
metadata?: string;
attributes?: AttributeItem[];
};
@@ -77,7 +78,7 @@ const useAppConfig = (): AppConfig => {
if (process.env.NEXT_PUBLIC_APP_CONFIG) {
try {
const parsedConfig = jsYaml.load(
process.env.NEXT_PUBLIC_APP_CONFIG
process.env.NEXT_PUBLIC_APP_CONFIG,
) as AppConfig;
if (parsedConfig.settings === undefined) {
parsedConfig.settings = defaultConfig.settings;
@@ -105,7 +106,7 @@ export const ConfigProvider = ({ children }: { children: React.ReactNode }) => {
const appConfig = useAppConfig();
const router = useRouter();
const [localColorOverride, setLocalColorOverride] = useState<string | null>(
null
null,
);
const getSettingsFromUrl = useCallback(() => {
@@ -168,7 +169,7 @@ export const ConfigProvider = ({ children }: { children: React.ReactNode }) => {
// Note: We don't set ws_url and token to the URL on purpose
router.replace("/#" + obj.toString());
},
[router]
[router],
);
const setCookieSettings = useCallback((us: UserSettings) => {
@@ -228,7 +229,7 @@ export const ConfigProvider = ({ children }: { children: React.ReactNode }) => {
};
});
},
[appConfig, setCookieSettings, setUrlSettings]
[appConfig, setCookieSettings, setUrlSettings],
);
const [config, _setConfig] = useState<AppConfig>(getConfig());

View File

@@ -1,4 +1,4 @@
"use client"
"use client";
import { useCloud } from "@/cloud/useCloud";
import React, { createContext, useState } from "react";
@@ -6,7 +6,7 @@ import { useCallback } from "react";
import { useConfig } from "./useConfig";
import { useToast } from "@/components/toast/ToasterProvider";
export type ConnectionMode = "cloud" | "manual" | "env"
export type ConnectionMode = "cloud" | "manual" | "env";
type TokenGeneratorData = {
shouldConnect: boolean;
@@ -17,7 +17,9 @@ type TokenGeneratorData = {
connect: (mode: ConnectionMode) => Promise<void>;
};
const ConnectionContext = createContext<TokenGeneratorData | undefined>(undefined);
const ConnectionContext = createContext<TokenGeneratorData | undefined>(
undefined,
);
export const ConnectionProvider = ({
children,
@@ -64,17 +66,25 @@ export const ConnectionProvider = ({
if (config.settings.participant_name) {
body.participantName = config.settings.participant_name;
}
if (config.settings.agent_name) {
body.agentName = config.settings.agent_name;
}
if (config.settings.metadata) {
body.metadata = config.settings.metadata;
}
const attributesArray = Array.isArray(config.settings.attributes) ? config.settings.attributes : [];
const attributesArray = Array.isArray(config.settings.attributes)
? config.settings.attributes
: [];
if (attributesArray?.length) {
const attributes = attributesArray.reduce((acc, attr) => {
if (attr.key){
acc[attr.key] = attr.value;
}
return acc;
}, {} as Record<string, string>);
const attributes = attributesArray.reduce(
(acc, attr) => {
if (attr.key) {
acc[attr.key] = attr.value;
}
return acc;
},
{} as Record<string, string>,
);
body.attributes = attributes;
}
const { accessToken } = await fetch(`/api/token`, {
@@ -83,9 +93,7 @@ export const ConnectionProvider = ({
"Content-Type": "application/json",
},
body: JSON.stringify(body),
}).then((res) =>
res.json()
);
}).then((res) => res.json());
token = accessToken;
} else {
token = config.settings.token;
@@ -98,12 +106,14 @@ export const ConnectionProvider = ({
config.settings.token,
config.settings.ws_url,
config.settings.room_name,
config.settings.participant_name,
config.settings.agent_name,
config.settings.participant_id,
config.settings.metadata,
config.settings.attributes,
generateToken,
setToastMessage,
]
],
);
const disconnect = useCallback(async () => {
@@ -132,4 +142,4 @@ export const useConnection = () => {
throw new Error("useConnection must be used within a ConnectionProvider");
}
return context;
}
};

View File

@@ -62,7 +62,7 @@ export const useMultibandTrackVolume = (
track?: Track,
bands: number = 5,
loPass: number = 100,
hiPass: number = 600
hiPass: number = 600,
) => {
const [frequencyBands, setFrequencyBands] = useState<Float32Array[]>([]);
@@ -93,7 +93,7 @@ export const useMultibandTrackVolume = (
const chunks: Float32Array[] = [];
for (let i = 0; i < bands; i++) {
chunks.push(
normalizedFrequencies.slice(i * chunkSize, (i + 1) * chunkSize)
normalizedFrequencies.slice(i * chunkSize, (i + 1) * chunkSize),
);
}

View File

@@ -19,4 +19,4 @@ export interface AttributeItem {
id: string;
key: string;
value: string;
}
}

View File

@@ -2,21 +2,36 @@ import { NextApiRequest, NextApiResponse } from "next";
import { generateRandomAlphanumeric } from "@/lib/util";
import { AccessToken } from "livekit-server-sdk";
import { RoomAgentDispatch, RoomConfiguration } from "@livekit/protocol";
import type { AccessTokenOptions, VideoGrant } from "livekit-server-sdk";
import { TokenResult } from "../../lib/types";
const apiKey = process.env.LIVEKIT_API_KEY;
const apiSecret = process.env.LIVEKIT_API_SECRET;
const createToken = (userInfo: AccessTokenOptions, grant: VideoGrant) => {
const createToken = (
userInfo: AccessTokenOptions,
grant: VideoGrant,
agentName?: string,
) => {
const at = new AccessToken(apiKey, apiSecret, userInfo);
at.addGrant(grant);
if (agentName) {
at.roomConfig = new RoomConfiguration({
agents: [
new RoomAgentDispatch({
agentName: agentName,
metadata: '{"user_id": "12345"}',
}),
],
});
}
return at.toJwt();
};
export default async function handleToken(
req: NextApiRequest,
res: NextApiResponse
res: NextApiResponse,
) {
try {
if (req.method !== "POST") {
@@ -32,34 +47,47 @@ export default async function handleToken(
const {
roomName: roomNameFromBody,
participantName,
participantName: participantNameFromBody,
participantId: participantIdFromBody,
metadata: metadataFromBody,
attributes: attributesFromBody,
agentName: agentNameFromBody,
} = req.body;
// Get room name from query params or generate random one
const roomName = roomNameFromBody as string ||
const roomName =
(roomNameFromBody as string) ||
`room-${generateRandomAlphanumeric(4)}-${generateRandomAlphanumeric(4)}`;
// Get participant name from query params or generate random one
const identity = participantIdFromBody as string ||
const identity =
(participantIdFromBody as string) ||
`identity-${generateRandomAlphanumeric(4)}`;
// Get agent name from query params or use none (automatic dispatch)
const agentName = (agentNameFromBody as string) || undefined;
// Get metadata and attributes from query params
const metadata = metadataFromBody as string | undefined;
const attributesStr = attributesFromBody as string | undefined;
const attributes = attributesStr || {};
const participantName = participantNameFromBody || identity;
const grant: VideoGrant = {
room: roomName,
roomJoin: true,
canPublish: true,
canPublishData: true,
canSubscribe: true,
canUpdateOwnMetadata: true,
};
const token = await createToken({ identity, metadata, attributes, name: participantName }, grant);
const token = await createToken(
{ identity, metadata, attributes, name: participantName },
grant,
agentName,
);
const result: TokenResult = {
identity,
accessToken: token,
@@ -70,4 +98,4 @@ export default async function handleToken(
res.statusMessage = (e as Error).message;
res.status(500).end();
}
}
}

View File

@@ -12,7 +12,11 @@ import { PlaygroundConnect } from "@/components/PlaygroundConnect";
import Playground from "@/components/playground/Playground";
import { PlaygroundToast, ToastType } from "@/components/toast/PlaygroundToast";
import { ConfigProvider, useConfig } from "@/hooks/useConfig";
import { ConnectionMode, ConnectionProvider, useConnection } from "@/hooks/useConnection";
import {
ConnectionMode,
ConnectionProvider,
useConnection,
} from "@/hooks/useConnection";
import { useMemo } from "react";
import { ToastProvider, useToast } from "@/components/toast/ToasterProvider";
@@ -44,26 +48,26 @@ export default function Home() {
export function HomeInner() {
const { shouldConnect, wsUrl, token, mode, connect, disconnect } =
useConnection();
const {config} = useConfig();
const { config } = useConfig();
const { toastMessage, setToastMessage } = useToast();
const handleConnect = useCallback(
async (c: boolean, mode: ConnectionMode) => {
c ? connect(mode) : disconnect();
},
[connect, disconnect]
[connect, disconnect],
);
const showPG = useMemo(() => {
if (process.env.NEXT_PUBLIC_LIVEKIT_URL) {
return true;
}
if(wsUrl) {
if (wsUrl) {
return true;
}
return false;
}, [wsUrl])
}, [wsUrl]);
return (
<>
@@ -129,4 +133,4 @@ export function HomeInner() {
</main>
</>
);
}
}

View File

@@ -29,7 +29,7 @@ export function TranscriptionTile({
});
const [transcripts, setTranscripts] = useState<Map<string, ChatMessageType>>(
new Map()
new Map(),
);
const [messages, setMessages] = useState<ChatMessageType[]>([]);
const { chatMessages, send: sendChat } = useChat();
@@ -43,21 +43,21 @@ export function TranscriptionTile({
segmentToChatMessage(
s,
transcripts.get(s.id),
agentAudioTrack.participant
)
)
agentAudioTrack.participant,
),
),
);
}
localMessages.segments.forEach((s) =>
transcripts.set(
s.id,
segmentToChatMessage(
s,
transcripts.get(s.id),
localParticipant.localParticipant
)
)
localParticipant.localParticipant,
),
),
);
const allMessages = Array.from(transcripts.values());
@@ -104,7 +104,7 @@ export function TranscriptionTile({
function segmentToChatMessage(
s: TranscriptionSegment,
existingMessage: ChatMessageType | undefined,
participant: Participant
participant: Participant,
): ChatMessageType {
const msg: ChatMessageType = {
message: s.final ? s.text : `${s.text} ...`,