From e1cc9a08a2a854df904eff8012a1477fa1d63792 Mon Sep 17 00:00:00 2001
From: David Zhao <david@davidzhao.com>
Date: Mon, 15 Jan 2024 13:51:56 -0800
Subject: [PATCH] Use agent status from metadata channels (#10)

---
 src/components/chat/ChatMessageInput.tsx      |  12 +-
 src/components/chat/ChatTile.tsx              |   5 +-
 src/components/playground/Playground.tsx      | 209 ++++++++++--------
 .../AgentMultibandAudioVisualizer.tsx         |   3 +-
 src/lib/types.ts                              |   7 +
 src/pages/index.tsx                           |  45 ++--
 6 files changed, 172 insertions(+), 109 deletions(-)

diff --git a/src/components/chat/ChatMessageInput.tsx b/src/components/chat/ChatMessageInput.tsx
index cf5e8f9..37205e5 100644
--- a/src/components/chat/ChatMessageInput.tsx
+++ b/src/components/chat/ChatMessageInput.tsx
@@ -1,11 +1,11 @@
 import { useWindowResize } from "@/hooks/useWindowResize";
-import { useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
 
 type ChatMessageInput = {
   placeholder: string;
   accentColor: string;
   height: number;
-  onSend: (message: string) => void;
+  onSend?: (message: string) => void;
 };
 
 export const ChatMessageInput = ({
@@ -23,14 +23,17 @@ export const ChatMessageInput = ({
   const [isTyping, setIsTyping] = useState(false);
   const [inputHasFocus, setInputHasFocus] = useState(false);
 
-  const handleSend = () => {
+  const handleSend = useCallback(() => {
+    if (!onSend) {
+      return;
+    }
     if (message === "") {
       return;
     }
 
     onSend(message);
     setMessage("");
-  };
+  }, [onSend, message]);
 
   useEffect(() => {
     setIsTyping(true);
@@ -105,6 +108,7 @@ export const ChatMessageInput = ({
           {message.replaceAll(" ", "\u00a0")}
         </span>
         <button
+          disabled={message.length === 0 || !onSend}
           onClick={handleSend}
           className={`text-xs uppercase text-${accentColor}-500 hover:bg-${accentColor}-950 p-2 rounded-md opacity-${
             message.length > 0 ? 100 : 25
diff --git a/src/components/chat/ChatTile.tsx b/src/components/chat/ChatTile.tsx
index 7f229e4..e608cf0 100644
--- a/src/components/chat/ChatTile.tsx
+++ b/src/components/chat/ChatTile.tsx
@@ -1,6 +1,6 @@
-import { useEffect, useRef, useState } from "react";
 import { ChatMessage } from "@/components/chat/ChatMessage";
 import { ChatMessageInput } from "@/components/chat/ChatMessageInput";
+import { useEffect, useRef } from "react";
 
 const inputHeight = 48;
 
@@ -8,12 +8,13 @@ export type ChatMessageType = {
   name: string;
   message: string;
   isSelf: boolean;
+  timestamp: number;
 };
 
 type ChatTileProps = {
   messages: ChatMessageType[];
   accentColor: string;
-  onSend: (message: string) => void;
+  onSend?: (message: string) => Promise<void>;
 };
 
 export const ChatTile = ({ messages, accentColor, onSend }: ChatTileProps) => {
diff --git a/src/components/playground/Playground.tsx b/src/components/playground/Playground.tsx
index 0e2bda3..7a196eb 100644
--- a/src/components/playground/Playground.tsx
+++ b/src/components/playground/Playground.tsx
@@ -1,23 +1,10 @@
 "use client";
 
-import {
-  VideoTrack,
-  useConnectionState,
-  useDataChannel,
-  useLocalParticipant,
-  useRemoteParticipants,
-  useRoomContext,
-  useTracks,
-} from "@livekit/components-react";
-import {
-  ConnectionState,
-  DataPacket_Kind,
-  LocalParticipant,
-  Track,
-} from "livekit-client";
-import { ColorPicker } from "@/components/colorPicker/ColorPicker";
-import { ConfigurationPanelItem } from "@/components/config/ConfigurationPanelItem";
 import { LoadingSVG } from "@/components/button/LoadingSVG";
+import { ChatMessageType, ChatTile } from "@/components/chat/ChatTile";
+import { ColorPicker } from "@/components/colorPicker/ColorPicker";
+import { AudioInputTile } from "@/components/config/AudioInputTile";
+import { ConfigurationPanelItem } from "@/components/config/ConfigurationPanelItem";
 import { NameValueRow } from "@/components/config/NameValueRow";
 import { PlaygroundHeader } from "@/components/playground/PlaygroundHeader";
 import {
@@ -25,12 +12,29 @@ import {
   PlaygroundTabbedTile,
   PlaygroundTile,
 } from "@/components/playground/PlaygroundTile";
-import { ReactNode, useCallback, useEffect, useMemo, useState } from "react";
-import { useMultibandTrackVolume } from "@/hooks/useTrackVolume";
-import { QRCodeSVG } from "qrcode.react";
-import { AudioInputTile } from "@/components/config/AudioInputTile";
-import { ChatMessageType, ChatTile } from "@/components/chat/ChatTile";
 import { AgentMultibandAudioVisualizer } from "@/components/visualization/AgentMultibandAudioVisualizer";
+import { useMultibandTrackVolume } from "@/hooks/useTrackVolume";
+import { AgentState } from "@/lib/types";
+import {
+  VideoTrack,
+  useChat,
+  useConnectionState,
+  useDataChannel,
+  useEnsureRoom,
+  useLocalParticipant,
+  useRemoteParticipants,
+  useTracks,
+} from "@livekit/components-react";
+import {
+  ConnectionState,
+  LocalParticipant,
+  ParticipantEvent,
+  RemoteParticipant,
+  RoomEvent,
+  Track,
+} from "livekit-client";
+import { QRCodeSVG } from "qrcode.react";
+import { ReactNode, useCallback, useEffect, useMemo, useState } from "react";
 
 export enum PlaygroundOutputs {
   Video,
@@ -38,6 +42,11 @@ export enum PlaygroundOutputs {
   Chat,
 }
 
+export interface PlaygroundMeta {
+  name: string;
+  value: string;
+}
+
 export interface PlaygroundProps {
   logo?: ReactNode;
   title?: string;
@@ -48,7 +57,7 @@ export interface PlaygroundProps {
   outputs?: PlaygroundOutputs[];
   showQR?: boolean;
   onConnect: (connect: boolean, opts?: { token: string; url: string }) => void;
-  metadata?: { name: string; value: string }[];
+  metadata?: PlaygroundMeta[];
 }
 
 const headerHeight = 56;
@@ -65,15 +74,17 @@ export default function Playground({
   onConnect,
   metadata,
 }: PlaygroundProps) {
-  const [agentState, setAgentState] = useState<
-    "listening" | "speaking" | "thinking" | "offline"
-  >("offline");
-
-  const [userState, setUserState] = useState<"silent" | "speaking">("silent");
+  const [agentState, setAgentState] = useState<AgentState>("offline");
   const [themeColor, setThemeColor] = useState(defaultColor);
   const [messages, setMessages] = useState<ChatMessageType[]>([]);
-  const localParticipant = useLocalParticipant();
-  const roomContext = useRoomContext();
+  const [transcripts, setTranscripts] = useState<ChatMessageType[]>([]);
+  const { localParticipant } = useLocalParticipant();
+  const room = useEnsureRoom();
+  const participants = useRemoteParticipants({
+    updateOnlyOn: [RoomEvent.ParticipantMetadataChanged],
+  });
+  const agentParticipant = participants.find((p) => p.isAgent);
+  const { send: sendChat, chatMessages } = useChat();
   const visualizerState = useMemo(() => {
     if (agentState === "thinking") {
       return "thinking";
@@ -118,39 +129,93 @@ export default function Playground({
     20
   );
 
-  const isAgentConnected = !!useRemoteParticipants().find((p) => p.isAgent);
-
   useEffect(() => {
-    if (isAgentConnected && agentState === "offline") {
-      setAgentState("listening");
-    } else if (!isAgentConnected) {
-      setAgentState("offline");
+    if (!agentParticipant || !room) {
+      return;
     }
-  }, [isAgentConnected, agentState]);
+    const metadataChanged = () => {
+      let agentMd: any = {};
+      if (agentParticipant.metadata) {
+        agentMd = JSON.parse(agentParticipant.metadata);
+      }
+      if (agentMd.agent_state) {
+        setAgentState(agentMd.agent_state);
+      } else {
+        setAgentState("starting");
+      }
+    };
+    const disconnected = (p: RemoteParticipant) => {
+      if (agentParticipant.identity === p.identity) {
+        setAgentState("offline");
+      }
+    };
+    agentParticipant.on(
+      ParticipantEvent.ParticipantMetadataChanged,
+      metadataChanged
+    );
+    room.on(RoomEvent.ParticipantDisconnected, disconnected);
+    return () => {
+      agentParticipant.off(
+        ParticipantEvent.ParticipantMetadataChanged,
+        metadataChanged
+      );
+      room.off(RoomEvent.ParticipantDisconnected, disconnected);
+    };
+  }, [agentParticipant, room]);
+
+  const isAgentConnected = agentState !== "offline";
 
   const onDataReceived = useCallback(
     (msg: any) => {
-      const decoded = JSON.parse(new TextDecoder("utf-8").decode(msg.payload));
-      if (decoded.type === "state") {
-        const { agent_state, user_state } = decoded;
-        setAgentState(agent_state);
-        setUserState(user_state);
-      } else if (decoded.type === "transcription") {
-        setMessages([
-          ...messages,
-          { name: "You", message: decoded.text, isSelf: true },
-        ]);
-      } else if (decoded.type === "agent_chat_message") {
-        setMessages([
-          ...messages,
-          { name: "Agent", message: decoded.text, isSelf: false },
+      if (msg.topic === "transcription") {
+        const decoded = JSON.parse(
+          new TextDecoder("utf-8").decode(msg.payload)
+        );
+        let timestamp = new Date().getTime();
+        if ("timestamp" in decoded && decoded.timestamp > 0) {
+          timestamp = decoded.timestamp;
+        }
+        setTranscripts([
+          ...transcripts,
+          {
+            name: "You",
+            message: decoded.text,
+            timestamp: timestamp,
+            isSelf: true,
+          },
         ]);
       }
-      console.log("data received", decoded, msg.from);
     },
-    [messages]
+    [transcripts]
   );
 
+  // combine transcripts and chat together
+  useEffect(() => {
+    const allMessages = [...transcripts];
+    for (const msg of chatMessages) {
+      const isAgent = msg.from?.identity === agentParticipant?.identity;
+      const isSelf = msg.from?.identity === localParticipant?.identity;
+      let name = msg.from?.name;
+      if (!name) {
+        if (isAgent) {
+          name = "Agent";
+        } else if (isSelf) {
+          name = "You";
+        } else {
+          name = "Unknown";
+        }
+      }
+      allMessages.push({
+        name,
+        message: msg.message,
+        timestamp: msg?.timestamp,
+        isSelf: isSelf,
+      });
+    }
+    allMessages.sort((a, b) => a.timestamp - b.timestamp);
+    setMessages(allMessages);
+  }, [transcripts, chatMessages, localParticipant, agentParticipant]);
+
   useDataChannel(onDataReceived);
 
   const videoTileContent = useMemo(() => {
@@ -201,32 +266,10 @@ export default function Playground({
       <ChatTile
         messages={messages}
         accentColor={themeColor}
-        onSend={(message) => {
-          if (roomContext.state === ConnectionState.Disconnected) {
-            return;
-          }
-          setMessages([
-            ...messages,
-            { name: "You", message: message, isSelf: true },
-          ]);
-          const data = {
-            type: "user_chat_message",
-            text: message,
-          };
-          const encoder = new TextEncoder();
-          localParticipant.localParticipant?.publishData(
-            encoder.encode(JSON.stringify(data)),
-            DataPacket_Kind.RELIABLE
-          );
-        }}
+        onSend={sendChat}
       />
     );
-  }, [
-    localParticipant.localParticipant,
-    messages,
-    roomContext.state,
-    themeColor,
-  ]);
+  }, [messages, themeColor, sendChat]);
 
   const settingsTileContent = useMemo(() => {
     return (
@@ -239,10 +282,6 @@ export default function Playground({
 
         <ConfigurationPanelItem title="Settings">
           <div className="flex flex-col gap-2">
-            <NameValueRow
-              name="Agent URL"
-              value={process.env.NEXT_PUBLIC_LIVEKIT_URL}
-            />
             {metadata?.map((data, index) => (
               <NameValueRow
                 key={data.name + index}
@@ -298,13 +337,6 @@ export default function Playground({
                 agentState === "speaking" ? `${themeColor}-500` : "gray-500"
               }
             />
-            <NameValueRow
-              name="User status"
-              value={userState}
-              valueColor={
-                userState === "silent" ? "gray-500" : `${themeColor}-500`
-              }
-            />
           </div>
         </ConfigurationPanelItem>
         {localVideoTrack && (
@@ -359,7 +391,6 @@ export default function Playground({
     roomState,
     themeColor,
     themeColors,
-    userState,
     showQR,
   ]);
 
diff --git a/src/components/visualization/AgentMultibandAudioVisualizer.tsx b/src/components/visualization/AgentMultibandAudioVisualizer.tsx
index bd4324a..186cccd 100644
--- a/src/components/visualization/AgentMultibandAudioVisualizer.tsx
+++ b/src/components/visualization/AgentMultibandAudioVisualizer.tsx
@@ -1,7 +1,8 @@
+import { AgentState } from "@/lib/types";
 import { useEffect, useState } from "react";
 
 type AgentMultibandAudioVisualizerProps = {
-  state: "listening" | "speaking" | "thinking" | "offline";
+  state: AgentState;
   barWidth: number;
   minBarHeight: number;
   maxBarHeight: number;
diff --git a/src/lib/types.ts b/src/lib/types.ts
index 30fbc33..0bc8419 100644
--- a/src/lib/types.ts
+++ b/src/lib/types.ts
@@ -14,3 +14,10 @@ export interface TokenResult {
   identity: string;
   accessToken: string;
 }
+
+export type AgentState =
+  | "listening"
+  | "speaking"
+  | "thinking"
+  | "offline"
+  | "starting";
diff --git a/src/pages/index.tsx b/src/pages/index.tsx
index 3610585..f32dc7a 100644
--- a/src/pages/index.tsx
+++ b/src/pages/index.tsx
@@ -1,21 +1,22 @@
-import Head from "next/head";
-import { useCallback, useEffect, useMemo, useState } from "react";
-import { Inter } from "next/font/google";
 import { generateRandomAlphanumeric } from "@/lib/util";
-import { motion, AnimatePresence } from "framer-motion";
 import {
   LiveKitRoom,
   RoomAudioRenderer,
   StartAudio,
   useToken,
 } from "@livekit/components-react";
+import { AnimatePresence, motion } from "framer-motion";
+import { Inter } from "next/font/google";
+import Head from "next/head";
+import { useCallback, useEffect, useMemo, useState } from "react";
 
+import { PlaygroundConnect } from "@/components/PlaygroundConnect";
 import Playground, {
+  PlaygroundMeta,
   PlaygroundOutputs,
 } from "@/components/playground/Playground";
-import { useAppConfig } from "@/hooks/useAppConfig";
-import { PlaygroundConnect } from "@/components/PlaygroundConnect";
 import { PlaygroundToast, ToastType } from "@/components/toast/PlaygroundToast";
+import { useAppConfig } from "@/hooks/useAppConfig";
 
 const themeColors = [
   "cyan",
@@ -40,10 +41,9 @@ export default function Home() {
     process.env.NEXT_PUBLIC_LIVEKIT_URL
   );
   const [customToken, setCustomToken] = useState<string>();
+  const [metadata, setMetadata] = useState<PlaygroundMeta[]>([]);
 
-  const [roomName, setRoomName] = useState(
-    [generateRandomAlphanumeric(4), generateRandomAlphanumeric(4)].join("-")
-  );
+  const [roomName, setRoomName] = useState(createRoomName());
 
   const tokenOptions = useMemo(() => {
     return {
@@ -54,12 +54,25 @@ export default function Home() {
   // set a new room name each time the user disconnects so that a new token gets fetched behind the scenes for a different room
   useEffect(() => {
     if (shouldConnect === false) {
-      setRoomName(
-        [generateRandomAlphanumeric(4), generateRandomAlphanumeric(4)].join("-")
-      );
+      setRoomName(createRoomName());
     }
   }, [shouldConnect]);
 
+  useEffect(() => {
+    const md: PlaygroundMeta[] = [];
+    if (liveKitUrl && liveKitUrl !== process.env.NEXT_PUBLIC_LIVEKIT_URL) {
+      md.push({ name: "LiveKit URL", value: liveKitUrl });
+    }
+    if (!customToken && tokenOptions.userInfo?.identity) {
+      md.push({ name: "Room Name", value: roomName });
+      md.push({
+        name: "Participant Identity",
+        value: tokenOptions.userInfo.identity,
+      });
+    }
+    setMetadata(md);
+  }, [liveKitUrl, roomName, tokenOptions, customToken]);
+
   const token = useToken("/api/token", roomName, tokenOptions);
   const appConfig = useAppConfig();
   const outputs = [
@@ -133,7 +146,7 @@ export default function Home() {
               themeColors={themeColors}
               defaultColor={appConfig?.theme_color ?? "cyan"}
               onConnect={handleConnect}
-              metadata={[{ name: "Room Name", value: roomName }]}
+              metadata={metadata}
             />
             <RoomAudioRenderer />
             <StartAudio label="Click to enable audio playback" />
@@ -150,3 +163,9 @@ export default function Home() {
     </>
   );
 }
+
+function createRoomName() {
+  return [generateRandomAlphanumeric(4), generateRandomAlphanumeric(4)].join(
+    "-"
+  );
+}