537 lines
15 KiB
TypeScript
537 lines
15 KiB
TypeScript
"use client";
|
|
|
|
import { LoadingSVG } from "@/components/button/LoadingSVG";
|
|
import { ChatMessageType, ChatTile } from "@/components/chat/ChatTile";
|
|
import { ColorPicker } from "@/components/colorPicker/ColorPicker";
|
|
import { AudioInputTile } from "@/components/config/AudioInputTile";
|
|
import { ConfigurationPanelItem } from "@/components/config/ConfigurationPanelItem";
|
|
import { NameValueRow } from "@/components/config/NameValueRow";
|
|
import { PlaygroundHeader } from "@/components/playground/PlaygroundHeader";
|
|
import {
|
|
PlaygroundTab,
|
|
PlaygroundTabbedTile,
|
|
PlaygroundTile,
|
|
} from "@/components/playground/PlaygroundTile";
|
|
import { AgentMultibandAudioVisualizer } from "@/components/visualization/AgentMultibandAudioVisualizer";
|
|
import { useConfig } from "@/hooks/useConfig";
|
|
import { useMultibandTrackVolume } from "@/hooks/useTrackVolume";
|
|
import { AgentState } from "@/lib/types";
|
|
import {
|
|
VideoTrack,
|
|
useChat,
|
|
useConnectionState,
|
|
useDataChannel,
|
|
useLocalParticipant,
|
|
useRemoteParticipants,
|
|
useRoomContext,
|
|
useRoomInfo,
|
|
useTracks,
|
|
} from "@livekit/components-react";
|
|
import {
|
|
ConnectionState,
|
|
LocalParticipant,
|
|
RoomEvent,
|
|
Track,
|
|
} from "livekit-client";
|
|
import { QRCodeSVG } from "qrcode.react";
|
|
import { ReactNode, useCallback, useEffect, useMemo, useState } from "react";
|
|
|
|
export interface PlaygroundMeta {
|
|
name: string;
|
|
value: string;
|
|
}
|
|
|
|
export interface PlaygroundProps {
|
|
logo?: ReactNode;
|
|
themeColors: string[];
|
|
onConnect: (connect: boolean, opts?: { token: string; url: string }) => void;
|
|
}
|
|
|
|
const headerHeight = 56;
|
|
|
|
export default function Playground({
|
|
logo,
|
|
themeColors,
|
|
onConnect,
|
|
}: PlaygroundProps) {
|
|
const { config, setUserSettings } = useConfig();
|
|
const { name } = useRoomInfo();
|
|
const [agentState, setAgentState] = useState<AgentState>("offline");
|
|
const [messages, setMessages] = useState<ChatMessageType[]>([]);
|
|
const [transcripts, setTranscripts] = useState<ChatMessageType[]>([]);
|
|
const { localParticipant } = useLocalParticipant();
|
|
|
|
const participants = useRemoteParticipants({
|
|
updateOnlyOn: [RoomEvent.ParticipantMetadataChanged],
|
|
});
|
|
const agentParticipant = participants.find((p) => p.isAgent);
|
|
|
|
const { send: sendChat, chatMessages } = useChat();
|
|
const roomState = useConnectionState();
|
|
const tracks = useTracks();
|
|
|
|
useEffect(() => {
|
|
if (roomState === ConnectionState.Connected) {
|
|
localParticipant.setCameraEnabled(config.settings.inputs.camera);
|
|
localParticipant.setMicrophoneEnabled(config.settings.inputs.mic);
|
|
}
|
|
}, [config, localParticipant, roomState]);
|
|
|
|
const agentAudioTrack = tracks.find(
|
|
(trackRef) =>
|
|
trackRef.publication.kind === Track.Kind.Audio &&
|
|
trackRef.participant.isAgent
|
|
);
|
|
|
|
const agentVideoTrack = tracks.find(
|
|
(trackRef) =>
|
|
trackRef.publication.kind === Track.Kind.Video &&
|
|
trackRef.participant.isAgent
|
|
);
|
|
|
|
const subscribedVolumes = useMultibandTrackVolume(
|
|
agentAudioTrack?.publication.track,
|
|
5
|
|
);
|
|
|
|
const localTracks = tracks.filter(
|
|
({ participant }) => participant instanceof LocalParticipant
|
|
);
|
|
const localVideoTrack = localTracks.find(
|
|
({ source }) => source === Track.Source.Camera
|
|
);
|
|
const localMicTrack = localTracks.find(
|
|
({ source }) => source === Track.Source.Microphone
|
|
);
|
|
|
|
const localMultibandVolume = useMultibandTrackVolume(
|
|
localMicTrack?.publication.track,
|
|
20
|
|
);
|
|
|
|
useEffect(() => {
|
|
if (!agentParticipant) {
|
|
setAgentState("offline");
|
|
return;
|
|
}
|
|
let agentMd: any = {};
|
|
if (agentParticipant.metadata) {
|
|
agentMd = JSON.parse(agentParticipant.metadata);
|
|
}
|
|
if (agentMd.agent_state) {
|
|
setAgentState(agentMd.agent_state);
|
|
} else {
|
|
setAgentState("starting");
|
|
}
|
|
}, [agentParticipant, agentParticipant?.metadata]);
|
|
|
|
const isAgentConnected = agentState !== "offline";
|
|
|
|
const onDataReceived = useCallback(
|
|
(msg: any) => {
|
|
if (msg.topic === "transcription") {
|
|
const decoded = JSON.parse(
|
|
new TextDecoder("utf-8").decode(msg.payload)
|
|
);
|
|
let timestamp = new Date().getTime();
|
|
if ("timestamp" in decoded && decoded.timestamp > 0) {
|
|
timestamp = decoded.timestamp;
|
|
}
|
|
setTranscripts([
|
|
...transcripts,
|
|
{
|
|
name: "You",
|
|
message: decoded.text,
|
|
timestamp: timestamp,
|
|
isSelf: true,
|
|
},
|
|
]);
|
|
}
|
|
},
|
|
[transcripts]
|
|
);
|
|
|
|
// combine transcripts and chat together
|
|
useEffect(() => {
|
|
const allMessages = [...transcripts];
|
|
for (const msg of chatMessages) {
|
|
const isAgent = msg.from?.identity === agentParticipant?.identity;
|
|
const isSelf = msg.from?.identity === localParticipant?.identity;
|
|
let name = msg.from?.name;
|
|
if (!name) {
|
|
if (isAgent) {
|
|
name = "Agent";
|
|
} else if (isSelf) {
|
|
name = "You";
|
|
} else {
|
|
name = "Unknown";
|
|
}
|
|
}
|
|
allMessages.push({
|
|
name,
|
|
message: msg.message,
|
|
timestamp: msg?.timestamp,
|
|
isSelf: isSelf,
|
|
});
|
|
}
|
|
allMessages.sort((a, b) => a.timestamp - b.timestamp);
|
|
setMessages(allMessages);
|
|
}, [transcripts, chatMessages, localParticipant, agentParticipant]);
|
|
|
|
useDataChannel(onDataReceived);
|
|
|
|
const videoTileContent = useMemo(() => {
|
|
const videoFitClassName = `object-${config.video_fit || "cover"}`;
|
|
|
|
const disconnectedContent = (
|
|
<div className="flex items-center justify-center text-gray-700 text-center w-full h-full">
|
|
No video track. Connect to get started.
|
|
</div>
|
|
);
|
|
|
|
const loadingContent = (
|
|
<div className="flex flex-col items-center justify-center gap-2 text-gray-700 text-center h-full w-full">
|
|
<LoadingSVG />
|
|
Waiting for video track
|
|
</div>
|
|
);
|
|
|
|
const videoContent = (
|
|
<VideoTrack
|
|
trackRef={agentVideoTrack}
|
|
className={`absolute top-1/2 -translate-y-1/2 ${videoFitClassName} object-position-center w-full h-full`}
|
|
/>
|
|
);
|
|
|
|
let content = null;
|
|
if (roomState === ConnectionState.Disconnected) {
|
|
content = disconnectedContent;
|
|
} else if (agentVideoTrack) {
|
|
content = videoContent;
|
|
} else {
|
|
content = loadingContent;
|
|
}
|
|
|
|
return (
|
|
<div className="flex flex-col w-full grow text-gray-950 bg-black rounded-sm border border-gray-800 relative">
|
|
{content}
|
|
</div>
|
|
);
|
|
}, [agentVideoTrack, config, roomState]);
|
|
|
|
const audioTileContent = useMemo(() => {
|
|
const disconnectedContent = (
|
|
<div className="flex flex-col items-center justify-center gap-2 text-gray-700 text-center w-full">
|
|
No audio track. Connect to get started.
|
|
</div>
|
|
);
|
|
|
|
const waitingContent = (
|
|
<div className="flex flex-col items-center gap-2 text-gray-700 text-center w-full">
|
|
<LoadingSVG />
|
|
Waiting for audio track
|
|
</div>
|
|
);
|
|
|
|
const visualizerContent = (
|
|
<div className="flex items-center justify-center w-full">
|
|
<AgentMultibandAudioVisualizer
|
|
state={agentState}
|
|
barWidth={30}
|
|
minBarHeight={30}
|
|
maxBarHeight={150}
|
|
accentColor={config.settings.theme_color}
|
|
accentShade={500}
|
|
frequencies={subscribedVolumes}
|
|
borderRadius={12}
|
|
gap={16}
|
|
/>
|
|
</div>
|
|
);
|
|
|
|
if (roomState === ConnectionState.Disconnected) {
|
|
return disconnectedContent;
|
|
}
|
|
|
|
if (!agentAudioTrack) {
|
|
return waitingContent;
|
|
}
|
|
|
|
return visualizerContent;
|
|
}, [
|
|
agentAudioTrack,
|
|
agentState,
|
|
config.settings.theme_color,
|
|
subscribedVolumes,
|
|
roomState,
|
|
]);
|
|
|
|
const chatTileContent = useMemo(() => {
|
|
return (
|
|
<ChatTile
|
|
messages={messages}
|
|
accentColor={config.settings.theme_color}
|
|
onSend={sendChat}
|
|
/>
|
|
);
|
|
}, [config.settings.theme_color, messages, sendChat]);
|
|
|
|
const settingsTileContent = useMemo(() => {
|
|
return (
|
|
<div className="flex flex-col gap-4 h-full w-full items-start overflow-y-auto">
|
|
{config.description && (
|
|
<ConfigurationPanelItem title="Description">
|
|
{config.description}
|
|
</ConfigurationPanelItem>
|
|
)}
|
|
|
|
<ConfigurationPanelItem title="Settings">
|
|
{localParticipant && (
|
|
<div className="flex flex-col gap-2">
|
|
<NameValueRow
|
|
name="Room"
|
|
value={name}
|
|
valueColor={`${config.settings.theme_color}-500`}
|
|
/>
|
|
<NameValueRow
|
|
name="Participant"
|
|
value={localParticipant.identity}
|
|
/>
|
|
</div>
|
|
)}
|
|
</ConfigurationPanelItem>
|
|
<ConfigurationPanelItem title="Status">
|
|
<div className="flex flex-col gap-2">
|
|
<NameValueRow
|
|
name="Room connected"
|
|
value={
|
|
roomState === ConnectionState.Connecting ? (
|
|
<LoadingSVG diameter={16} strokeWidth={2} />
|
|
) : (
|
|
roomState
|
|
)
|
|
}
|
|
valueColor={
|
|
roomState === ConnectionState.Connected
|
|
? `${config.settings.theme_color}-500`
|
|
: "gray-500"
|
|
}
|
|
/>
|
|
<NameValueRow
|
|
name="Agent connected"
|
|
value={
|
|
isAgentConnected ? (
|
|
"true"
|
|
) : roomState === ConnectionState.Connected ? (
|
|
<LoadingSVG diameter={12} strokeWidth={2} />
|
|
) : (
|
|
"false"
|
|
)
|
|
}
|
|
valueColor={
|
|
isAgentConnected
|
|
? `${config.settings.theme_color}-500`
|
|
: "gray-500"
|
|
}
|
|
/>
|
|
<NameValueRow
|
|
name="Agent status"
|
|
value={
|
|
agentState !== "offline" && agentState !== "speaking" ? (
|
|
<div className="flex gap-2 items-center">
|
|
<LoadingSVG diameter={12} strokeWidth={2} />
|
|
{agentState}
|
|
</div>
|
|
) : (
|
|
agentState
|
|
)
|
|
}
|
|
valueColor={
|
|
agentState === "speaking"
|
|
? `${config.settings.theme_color}-500`
|
|
: "gray-500"
|
|
}
|
|
/>
|
|
</div>
|
|
</ConfigurationPanelItem>
|
|
{localVideoTrack && (
|
|
<ConfigurationPanelItem
|
|
title="Camera"
|
|
deviceSelectorKind="videoinput"
|
|
>
|
|
<div className="relative">
|
|
<VideoTrack
|
|
className="rounded-sm border border-gray-800 opacity-70 w-full"
|
|
trackRef={localVideoTrack}
|
|
/>
|
|
</div>
|
|
</ConfigurationPanelItem>
|
|
)}
|
|
{localMicTrack && (
|
|
<ConfigurationPanelItem
|
|
title="Microphone"
|
|
deviceSelectorKind="audioinput"
|
|
>
|
|
<AudioInputTile frequencies={localMultibandVolume} />
|
|
</ConfigurationPanelItem>
|
|
)}
|
|
<div className="w-full">
|
|
<ConfigurationPanelItem title="Color">
|
|
<ColorPicker
|
|
colors={themeColors}
|
|
selectedColor={config.settings.theme_color}
|
|
onSelect={(color) => {
|
|
const userSettings = { ...config.settings };
|
|
userSettings.theme_color = color;
|
|
setUserSettings(userSettings);
|
|
}}
|
|
/>
|
|
</ConfigurationPanelItem>
|
|
</div>
|
|
{config.show_qr && (
|
|
<div className="w-full">
|
|
<ConfigurationPanelItem title="QR Code">
|
|
<QRCodeSVG value={window.location.href} width="128" />
|
|
</ConfigurationPanelItem>
|
|
</div>
|
|
)}
|
|
</div>
|
|
);
|
|
}, [
|
|
config.description,
|
|
config.settings,
|
|
config.show_qr,
|
|
localParticipant,
|
|
name,
|
|
roomState,
|
|
isAgentConnected,
|
|
agentState,
|
|
localVideoTrack,
|
|
localMicTrack,
|
|
localMultibandVolume,
|
|
themeColors,
|
|
setUserSettings,
|
|
]);
|
|
|
|
let mobileTabs: PlaygroundTab[] = [];
|
|
if (config.settings.outputs.video) {
|
|
mobileTabs.push({
|
|
title: "Video",
|
|
content: (
|
|
<PlaygroundTile
|
|
className="w-full h-full grow"
|
|
childrenClassName="justify-center"
|
|
>
|
|
{videoTileContent}
|
|
</PlaygroundTile>
|
|
),
|
|
});
|
|
}
|
|
|
|
if (config.settings.outputs.audio) {
|
|
mobileTabs.push({
|
|
title: "Audio",
|
|
content: (
|
|
<PlaygroundTile
|
|
className="w-full h-full grow"
|
|
childrenClassName="justify-center"
|
|
>
|
|
{audioTileContent}
|
|
</PlaygroundTile>
|
|
),
|
|
});
|
|
}
|
|
|
|
if (config.settings.chat) {
|
|
mobileTabs.push({
|
|
title: "Chat",
|
|
content: chatTileContent,
|
|
});
|
|
}
|
|
|
|
mobileTabs.push({
|
|
title: "Settings",
|
|
content: (
|
|
<PlaygroundTile
|
|
padding={false}
|
|
backgroundColor="gray-950"
|
|
className="h-full w-full basis-1/4 items-start overflow-y-auto flex"
|
|
childrenClassName="h-full grow items-start"
|
|
>
|
|
{settingsTileContent}
|
|
</PlaygroundTile>
|
|
),
|
|
});
|
|
|
|
return (
|
|
<>
|
|
<PlaygroundHeader
|
|
title={config.title}
|
|
logo={logo}
|
|
githubLink={config.github_link}
|
|
height={headerHeight}
|
|
accentColor={config.settings.theme_color}
|
|
connectionState={roomState}
|
|
onConnectClicked={() =>
|
|
onConnect(roomState === ConnectionState.Disconnected)
|
|
}
|
|
/>
|
|
<div
|
|
className={`flex gap-4 py-4 grow w-full selection:bg-${config.settings.theme_color}-900`}
|
|
style={{ height: `calc(100% - ${headerHeight}px)` }}
|
|
>
|
|
<div className="flex flex-col grow basis-1/2 gap-4 h-full lg:hidden">
|
|
<PlaygroundTabbedTile
|
|
className="h-full"
|
|
tabs={mobileTabs}
|
|
initialTab={mobileTabs.length - 1}
|
|
/>
|
|
</div>
|
|
<div
|
|
className={`flex-col grow basis-1/2 gap-4 h-full hidden lg:${
|
|
!config.settings.outputs.audio && !config.settings.outputs.video
|
|
? "hidden"
|
|
: "flex"
|
|
}`}
|
|
>
|
|
{config.settings.outputs.video && (
|
|
<PlaygroundTile
|
|
title="Video"
|
|
className="w-full h-full grow"
|
|
childrenClassName="justify-center"
|
|
>
|
|
{videoTileContent}
|
|
</PlaygroundTile>
|
|
)}
|
|
{config.settings.outputs.audio && (
|
|
<PlaygroundTile
|
|
title="Audio"
|
|
className="w-full h-full grow"
|
|
childrenClassName="justify-center"
|
|
>
|
|
{audioTileContent}
|
|
</PlaygroundTile>
|
|
)}
|
|
</div>
|
|
|
|
{config.settings.chat && (
|
|
<PlaygroundTile
|
|
title="Chat"
|
|
className="h-full grow basis-1/4 hidden lg:flex"
|
|
>
|
|
{chatTileContent}
|
|
</PlaygroundTile>
|
|
)}
|
|
<PlaygroundTile
|
|
padding={false}
|
|
backgroundColor="gray-950"
|
|
className="h-full w-full basis-1/4 items-start overflow-y-auto hidden max-w-[480px] lg:flex"
|
|
childrenClassName="h-full grow items-start"
|
|
>
|
|
{settingsTileContent}
|
|
</PlaygroundTile>
|
|
</div>
|
|
</>
|
|
);
|
|
}
|