wx44wx/livekit-agents-debate-frontend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use agent status from metadata channels (#10)

Browse Source
This commit is contained in:
David Zhao
2024-01-15 13:51:56 -08:00
committed by GitHub
parent 0d00b73f59
commit e1cc9a08a2
6 changed files with 172 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/components/chat/ChatMessageInput.tsx
View File

@@ -1,11 +1,11 @@
import { useWindowResize } from "@/hooks/useWindowResize";
import { useEffect, useRef, useState } from "react";
import { useCallback, useEffect, useRef, useState } from "react";
type ChatMessageInput = {
placeholder: string;
accentColor: string;
height: number;
onSend: (message: string) => void;
onSend?: (message: string) => void;
};
export const ChatMessageInput = ({
@@ -23,14 +23,17 @@ export const ChatMessageInput = ({
const [isTyping, setIsTyping] = useState(false);
const [inputHasFocus, setInputHasFocus] = useState(false);
const handleSend = () => {
const handleSend = useCallback(() => {
if (!onSend) {
return;
}
if (message === "") {
return;
}
onSend(message);
setMessage("");
};
}, [onSend, message]);
useEffect(() => {
setIsTyping(true);
@@ -105,6 +108,7 @@ export const ChatMessageInput = ({
{message.replaceAll(" ", "\u00a0")}
</span>
<button
disabled={message.length === 0 || !onSend}
onClick={handleSend}
className={`text-xs uppercase text-${accentColor}-500 hover:bg-${accentColor}-950 p-2 rounded-md opacity-${
message.length > 0 ? 100 : 25

5
src/components/chat/ChatTile.tsx
View File

@@ -1,6 +1,6 @@
import { useEffect, useRef, useState } from "react";
import { ChatMessage } from "@/components/chat/ChatMessage";
import { ChatMessageInput } from "@/components/chat/ChatMessageInput";
import { useEffect, useRef } from "react";
const inputHeight = 48;
@@ -8,12 +8,13 @@ export type ChatMessageType = {
name: string;
message: string;
isSelf: boolean;
timestamp: number;
};
type ChatTileProps = {
messages: ChatMessageType[];
accentColor: string;
onSend: (message: string) => void;
onSend?: (message: string) => Promise<void>;
};
export const ChatTile = ({ messages, accentColor, onSend }: ChatTileProps) => {

209
src/components/playground/Playground.tsx
View File

@@ -1,23 +1,10 @@
"use client";
import {
VideoTrack,
useConnectionState,
useDataChannel,
useLocalParticipant,
useRemoteParticipants,
useRoomContext,
useTracks,
} from "@livekit/components-react";
import {
ConnectionState,
DataPacket_Kind,
LocalParticipant,
Track,
} from "livekit-client";
import { ColorPicker } from "@/components/colorPicker/ColorPicker";
import { ConfigurationPanelItem } from "@/components/config/ConfigurationPanelItem";
import { LoadingSVG } from "@/components/button/LoadingSVG";
import { ChatMessageType, ChatTile } from "@/components/chat/ChatTile";
import { ColorPicker } from "@/components/colorPicker/ColorPicker";
import { AudioInputTile } from "@/components/config/AudioInputTile";
import { ConfigurationPanelItem } from "@/components/config/ConfigurationPanelItem";
import { NameValueRow } from "@/components/config/NameValueRow";
import { PlaygroundHeader } from "@/components/playground/PlaygroundHeader";
import {
@@ -25,12 +12,29 @@ import {
PlaygroundTabbedTile,
PlaygroundTile,
} from "@/components/playground/PlaygroundTile";
import { ReactNode, useCallback, useEffect, useMemo, useState } from "react";
import { useMultibandTrackVolume } from "@/hooks/useTrackVolume";
import { QRCodeSVG } from "qrcode.react";
import { AudioInputTile } from "@/components/config/AudioInputTile";
import { ChatMessageType, ChatTile } from "@/components/chat/ChatTile";
import { AgentMultibandAudioVisualizer } from "@/components/visualization/AgentMultibandAudioVisualizer";
import { useMultibandTrackVolume } from "@/hooks/useTrackVolume";
import { AgentState } from "@/lib/types";
import {
VideoTrack,
useChat,
useConnectionState,
useDataChannel,
useEnsureRoom,
useLocalParticipant,
useRemoteParticipants,
useTracks,
} from "@livekit/components-react";
import {
ConnectionState,
LocalParticipant,
ParticipantEvent,
RemoteParticipant,
RoomEvent,
Track,
} from "livekit-client";
import { QRCodeSVG } from "qrcode.react";
import { ReactNode, useCallback, useEffect, useMemo, useState } from "react";
export enum PlaygroundOutputs {
Video,
@@ -38,6 +42,11 @@ export enum PlaygroundOutputs {
Chat,
}
export interface PlaygroundMeta {
name: string;
value: string;
}
export interface PlaygroundProps {
logo?: ReactNode;
title?: string;
@@ -48,7 +57,7 @@ export interface PlaygroundProps {
outputs?: PlaygroundOutputs[];
showQR?: boolean;
onConnect: (connect: boolean, opts?: { token: string; url: string }) => void;
metadata?: { name: string; value: string }[];
metadata?: PlaygroundMeta[];
}
const headerHeight = 56;
@@ -65,15 +74,17 @@ export default function Playground({
onConnect,
metadata,
}: PlaygroundProps) {
const [agentState, setAgentState] = useState<
"listening" | "speaking" | "thinking" | "offline"
>("offline");
const [userState, setUserState] = useState<"silent" | "speaking">("silent");
const [agentState, setAgentState] = useState<AgentState>("offline");
const [themeColor, setThemeColor] = useState(defaultColor);
const [messages, setMessages] = useState<ChatMessageType[]>([]);
const localParticipant = useLocalParticipant();
const roomContext = useRoomContext();
const [transcripts, setTranscripts] = useState<ChatMessageType[]>([]);
const { localParticipant } = useLocalParticipant();
const room = useEnsureRoom();
const participants = useRemoteParticipants({
updateOnlyOn: [RoomEvent.ParticipantMetadataChanged],
});
const agentParticipant = participants.find((p) => p.isAgent);
const { send: sendChat, chatMessages } = useChat();
const visualizerState = useMemo(() => {
if (agentState === "thinking") {
return "thinking";
@@ -118,39 +129,93 @@ export default function Playground({
20
);
const isAgentConnected = !!useRemoteParticipants().find((p) => p.isAgent);
useEffect(() => {
if (isAgentConnected && agentState === "offline") {
setAgentState("listening");
} else if (!isAgentConnected) {
setAgentState("offline");
if (!agentParticipant || !room) {
return;
}
}, [isAgentConnected, agentState]);
const metadataChanged = () => {
let agentMd: any = {};
if (agentParticipant.metadata) {
agentMd = JSON.parse(agentParticipant.metadata);
}
if (agentMd.agent_state) {
setAgentState(agentMd.agent_state);
} else {
setAgentState("starting");
}
};
const disconnected = (p: RemoteParticipant) => {
if (agentParticipant.identity === p.identity) {
setAgentState("offline");
}
};
agentParticipant.on(
ParticipantEvent.ParticipantMetadataChanged,
metadataChanged
);
room.on(RoomEvent.ParticipantDisconnected, disconnected);
return () => {
agentParticipant.off(
ParticipantEvent.ParticipantMetadataChanged,
metadataChanged
);
room.off(RoomEvent.ParticipantDisconnected, disconnected);
};
}, [agentParticipant, room]);
const isAgentConnected = agentState !== "offline";
const onDataReceived = useCallback(
(msg: any) => {
const decoded = JSON.parse(new TextDecoder("utf-8").decode(msg.payload));
if (decoded.type === "state") {
const { agent_state, user_state } = decoded;
setAgentState(agent_state);
setUserState(user_state);
} else if (decoded.type === "transcription") {
setMessages([
...messages,
{ name: "You", message: decoded.text, isSelf: true },
]);
} else if (decoded.type === "agent_chat_message") {
setMessages([
...messages,
{ name: "Agent", message: decoded.text, isSelf: false },
if (msg.topic === "transcription") {
const decoded = JSON.parse(
new TextDecoder("utf-8").decode(msg.payload)
);
let timestamp = new Date().getTime();
if ("timestamp" in decoded && decoded.timestamp > 0) {
timestamp = decoded.timestamp;
}
setTranscripts([
...transcripts,
{
name: "You",
message: decoded.text,
timestamp: timestamp,
isSelf: true,
},
]);
}
console.log("data received", decoded, msg.from);
},
[messages]
[transcripts]
);
// combine transcripts and chat together
useEffect(() => {
const allMessages = [...transcripts];
for (const msg of chatMessages) {
const isAgent = msg.from?.identity === agentParticipant?.identity;
const isSelf = msg.from?.identity === localParticipant?.identity;
let name = msg.from?.name;
if (!name) {
if (isAgent) {
name = "Agent";
} else if (isSelf) {
name = "You";
} else {
name = "Unknown";
}
}
allMessages.push({
name,
message: msg.message,
timestamp: msg?.timestamp,
isSelf: isSelf,
});
}
allMessages.sort((a, b) => a.timestamp - b.timestamp);
setMessages(allMessages);
}, [transcripts, chatMessages, localParticipant, agentParticipant]);
useDataChannel(onDataReceived);
const videoTileContent = useMemo(() => {
@@ -201,32 +266,10 @@ export default function Playground({
<ChatTile
messages={messages}
accentColor={themeColor}
onSend={(message) => {
if (roomContext.state === ConnectionState.Disconnected) {
return;
}
setMessages([
...messages,
{ name: "You", message: message, isSelf: true },
]);
const data = {
type: "user_chat_message",
text: message,
};
const encoder = new TextEncoder();
localParticipant.localParticipant?.publishData(
encoder.encode(JSON.stringify(data)),
DataPacket_Kind.RELIABLE
);
}}
onSend={sendChat}
/>
);
}, [
localParticipant.localParticipant,
messages,
roomContext.state,
themeColor,
]);
}, [messages, themeColor, sendChat]);
const settingsTileContent = useMemo(() => {
return (
@@ -239,10 +282,6 @@ export default function Playground({
<ConfigurationPanelItem title="Settings">
<div className="flex flex-col gap-2">
<NameValueRow
name="Agent URL"
value={process.env.NEXT_PUBLIC_LIVEKIT_URL}
/>
{metadata?.map((data, index) => (
<NameValueRow
key={data.name + index}
@@ -298,13 +337,6 @@ export default function Playground({
agentState === "speaking" ? `${themeColor}-500` : "gray-500"
}
/>
<NameValueRow
name="User status"
value={userState}
valueColor={
userState === "silent" ? "gray-500" : `${themeColor}-500`
}
/>
</div>
</ConfigurationPanelItem>
{localVideoTrack && (
@@ -359,7 +391,6 @@ export default function Playground({
roomState,
themeColor,
themeColors,
userState,
showQR,
]);

3
src/components/visualization/AgentMultibandAudioVisualizer.tsx
View File

@@ -1,7 +1,8 @@
import { AgentState } from "@/lib/types";
import { useEffect, useState } from "react";
type AgentMultibandAudioVisualizerProps = {
state: "listening" | "speaking" | "thinking" | "offline";
state: AgentState;
barWidth: number;
minBarHeight: number;
maxBarHeight: number;

7
src/lib/types.ts
View File

@@ -14,3 +14,10 @@ export interface TokenResult {
identity: string;
accessToken: string;
}
export type AgentState =
| "listening"
| "speaking"
| "thinking"
| "offline"
| "starting";

45
src/pages/index.tsx
View File

@@ -1,21 +1,22 @@
import Head from "next/head";
import { useCallback, useEffect, useMemo, useState } from "react";
import { Inter } from "next/font/google";
import { generateRandomAlphanumeric } from "@/lib/util";
import { motion, AnimatePresence } from "framer-motion";
import {
LiveKitRoom,
RoomAudioRenderer,
StartAudio,
useToken,
} from "@livekit/components-react";
import { AnimatePresence, motion } from "framer-motion";
import { Inter } from "next/font/google";
import Head from "next/head";
import { useCallback, useEffect, useMemo, useState } from "react";
import { PlaygroundConnect } from "@/components/PlaygroundConnect";
import Playground, {
PlaygroundMeta,
PlaygroundOutputs,
} from "@/components/playground/Playground";
import { useAppConfig } from "@/hooks/useAppConfig";
import { PlaygroundConnect } from "@/components/PlaygroundConnect";
import { PlaygroundToast, ToastType } from "@/components/toast/PlaygroundToast";
import { useAppConfig } from "@/hooks/useAppConfig";
const themeColors = [
"cyan",
@@ -40,10 +41,9 @@ export default function Home() {
process.env.NEXT_PUBLIC_LIVEKIT_URL
);
const [customToken, setCustomToken] = useState<string>();
const [metadata, setMetadata] = useState<PlaygroundMeta[]>([]);
const [roomName, setRoomName] = useState(
[generateRandomAlphanumeric(4), generateRandomAlphanumeric(4)].join("-")
);
const [roomName, setRoomName] = useState(createRoomName());
const tokenOptions = useMemo(() => {
return {
@@ -54,12 +54,25 @@ export default function Home() {
// set a new room name each time the user disconnects so that a new token gets fetched behind the scenes for a different room
useEffect(() => {
if (shouldConnect === false) {
setRoomName(
[generateRandomAlphanumeric(4), generateRandomAlphanumeric(4)].join("-")
);
setRoomName(createRoomName());
}
}, [shouldConnect]);
useEffect(() => {
const md: PlaygroundMeta[] = [];
if (liveKitUrl && liveKitUrl !== process.env.NEXT_PUBLIC_LIVEKIT_URL) {
md.push({ name: "LiveKit URL", value: liveKitUrl });
}
if (!customToken && tokenOptions.userInfo?.identity) {
md.push({ name: "Room Name", value: roomName });
md.push({
name: "Participant Identity",
value: tokenOptions.userInfo.identity,
});
}
setMetadata(md);
}, [liveKitUrl, roomName, tokenOptions, customToken]);
const token = useToken("/api/token", roomName, tokenOptions);
const appConfig = useAppConfig();
const outputs = [
@@ -133,7 +146,7 @@ export default function Home() {
themeColors={themeColors}
defaultColor={appConfig?.theme_color ?? "cyan"}
onConnect={handleConnect}
metadata={[{ name: "Room Name", value: roomName }]}
metadata={metadata}
/>
<RoomAudioRenderer />
<StartAudio label="Click to enable audio playback" />
@@ -150,3 +163,9 @@ export default function Home() {
</>
);
}
function createRoomName() {
return [generateRandomAlphanumeric(4), generateRandomAlphanumeric(4)].join(
"-"
);
}