"use client"; import { LoadingSVG } from "@/components/button/LoadingSVG"; import { ChatMessageType, ChatTile } from "@/components/chat/ChatTile"; import { ColorPicker } from "@/components/colorPicker/ColorPicker"; import { AudioInputTile } from "@/components/config/AudioInputTile"; import { ConfigurationPanelItem } from "@/components/config/ConfigurationPanelItem"; import { NameValueRow } from "@/components/config/NameValueRow"; import { PlaygroundHeader } from "@/components/playground/PlaygroundHeader"; import { PlaygroundTab, PlaygroundTabbedTile, PlaygroundTile, } from "@/components/playground/PlaygroundTile"; import { AgentMultibandAudioVisualizer } from "@/components/visualization/AgentMultibandAudioVisualizer"; import { useConfig } from "@/hooks/useConfig"; import { useMultibandTrackVolume } from "@/hooks/useTrackVolume"; import { AgentState } from "@/lib/types"; import { VideoTrack, useChat, useConnectionState, useDataChannel, useLocalParticipant, useRemoteParticipants, useRoomContext, useRoomInfo, useTracks, } from "@livekit/components-react"; import { ConnectionState, LocalParticipant, RoomEvent, Track, } from "livekit-client"; import { QRCodeSVG } from "qrcode.react"; import { ReactNode, useCallback, useEffect, useMemo, useState } from "react"; export interface PlaygroundMeta { name: string; value: string; } export interface PlaygroundProps { logo?: ReactNode; themeColors: string[]; onConnect: (connect: boolean, opts?: { token: string; url: string }) => void; } const headerHeight = 56; export default function Playground({ logo, themeColors, onConnect, }: PlaygroundProps) { const { config, setUserSettings } = useConfig(); const { name } = useRoomInfo(); const [agentState, setAgentState] = useState("offline"); const [messages, setMessages] = useState([]); const [transcripts, setTranscripts] = useState([]); const { localParticipant } = useLocalParticipant(); const participants = useRemoteParticipants({ updateOnlyOn: [RoomEvent.ParticipantMetadataChanged], }); const agentParticipant = participants.find((p) => p.isAgent); const { send: sendChat, chatMessages } = useChat(); const roomState = useConnectionState(); const tracks = useTracks(); useEffect(() => { if (roomState === ConnectionState.Connected) { localParticipant.setCameraEnabled(config.settings.inputs.camera); localParticipant.setMicrophoneEnabled(config.settings.inputs.mic); } }, [config, localParticipant, roomState]); const agentAudioTrack = tracks.find( (trackRef) => trackRef.publication.kind === Track.Kind.Audio && trackRef.participant.isAgent ); const agentVideoTrack = tracks.find( (trackRef) => trackRef.publication.kind === Track.Kind.Video && trackRef.participant.isAgent ); const subscribedVolumes = useMultibandTrackVolume( agentAudioTrack?.publication.track, 5 ); const localTracks = tracks.filter( ({ participant }) => participant instanceof LocalParticipant ); const localVideoTrack = localTracks.find( ({ source }) => source === Track.Source.Camera ); const localMicTrack = localTracks.find( ({ source }) => source === Track.Source.Microphone ); const localMultibandVolume = useMultibandTrackVolume( localMicTrack?.publication.track, 20 ); useEffect(() => { if (!agentParticipant) { setAgentState("offline"); return; } let agentMd: any = {}; if (agentParticipant.metadata) { agentMd = JSON.parse(agentParticipant.metadata); } if (agentMd.agent_state) { setAgentState(agentMd.agent_state); } else { setAgentState("starting"); } }, [agentParticipant, agentParticipant?.metadata]); const isAgentConnected = agentState !== "offline"; const onDataReceived = useCallback( (msg: any) => { if (msg.topic === "transcription") { const decoded = JSON.parse( new TextDecoder("utf-8").decode(msg.payload) ); let timestamp = new Date().getTime(); if ("timestamp" in decoded && decoded.timestamp > 0) { timestamp = decoded.timestamp; } setTranscripts([ ...transcripts, { name: "You", message: decoded.text, timestamp: timestamp, isSelf: true, }, ]); } }, [transcripts] ); // combine transcripts and chat together useEffect(() => { const allMessages = [...transcripts]; for (const msg of chatMessages) { const isAgent = msg.from?.identity === agentParticipant?.identity; const isSelf = msg.from?.identity === localParticipant?.identity; let name = msg.from?.name; if (!name) { if (isAgent) { name = "Agent"; } else if (isSelf) { name = "You"; } else { name = "Unknown"; } } allMessages.push({ name, message: msg.message, timestamp: msg?.timestamp, isSelf: isSelf, }); } allMessages.sort((a, b) => a.timestamp - b.timestamp); setMessages(allMessages); }, [transcripts, chatMessages, localParticipant, agentParticipant]); useDataChannel(onDataReceived); const videoTileContent = useMemo(() => { const videoFitClassName = `object-${config.video_fit || "cover"}`; const disconnectedContent = (
No video track. Connect to get started.
); const loadingContent = (
Waiting for video track
); const videoContent = ( ); let content = null; if (roomState === ConnectionState.Disconnected) { content = disconnectedContent; } else if (agentVideoTrack) { content = videoContent; } else { content = loadingContent; } return (
{content}
); }, [agentVideoTrack, config, roomState]); const audioTileContent = useMemo(() => { const disconnectedContent = (
No audio track. Connect to get started.
); const waitingContent = (
Waiting for audio track
); const visualizerContent = (
); if (roomState === ConnectionState.Disconnected) { return disconnectedContent; } if (!agentAudioTrack) { return waitingContent; } return visualizerContent; }, [ agentAudioTrack, agentState, config.settings.theme_color, subscribedVolumes, roomState, ]); const chatTileContent = useMemo(() => { return ( ); }, [config.settings.theme_color, messages, sendChat]); const settingsTileContent = useMemo(() => { return (
{config.description && ( {config.description} )} {localParticipant && (
)}
) : ( roomState ) } valueColor={ roomState === ConnectionState.Connected ? `${config.settings.theme_color}-500` : "gray-500" } /> ) : ( "false" ) } valueColor={ isAgentConnected ? `${config.settings.theme_color}-500` : "gray-500" } /> {agentState}
) : ( agentState ) } valueColor={ agentState === "speaking" ? `${config.settings.theme_color}-500` : "gray-500" } />
{localVideoTrack && (
)} {localMicTrack && ( )}
{ const userSettings = { ...config.settings }; userSettings.theme_color = color; setUserSettings(userSettings); }} />
{config.show_qr && (
)} ); }, [ config.description, config.settings, config.show_qr, localParticipant, name, roomState, isAgentConnected, agentState, localVideoTrack, localMicTrack, localMultibandVolume, themeColors, setUserSettings, ]); let mobileTabs: PlaygroundTab[] = []; if (config.settings.outputs.video) { mobileTabs.push({ title: "Video", content: ( {videoTileContent} ), }); } if (config.settings.outputs.audio) { mobileTabs.push({ title: "Audio", content: ( {audioTileContent} ), }); } if (config.settings.chat) { mobileTabs.push({ title: "Chat", content: chatTileContent, }); } mobileTabs.push({ title: "Settings", content: ( {settingsTileContent} ), }); return ( <> onConnect(roomState === ConnectionState.Disconnected) } />
{config.settings.chat && ( {chatTileContent} )} {settingsTileContent}
); }