diff --git a/package-lock.json b/package-lock.json index 111d620..9ec425c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,12 +8,12 @@ "name": "agents-playground", "version": "0.1.0", "dependencies": { - "@livekit/components-react": "^2.1.1", + "@livekit/components-react": "^2.3.1", "@radix-ui/react-dropdown-menu": "^2.0.6", "cookies-next": "^4.1.1", "framer-motion": "^10.16.16", "js-yaml": "^4.1.0", - "livekit-client": "^2.1.0", + "livekit-client": "^2.1.5", "livekit-server-sdk": "^2.1.2", "lodash": "^4.17.21", "next": "^14.0.4", @@ -153,9 +153,9 @@ } }, "node_modules/@floating-ui/dom": { - "version": "1.6.3", - "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.3.tgz", - "integrity": "sha512-RnDthu3mzPlQ31Ss/BTwQ1zjzIhr3lk1gZB1OC56h/1vEtaXkESrOqL5fQVMfXpwGtRwX+YsZBdyHtJMQnkArw==", + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.4.tgz", + "integrity": "sha512-0G8R+zOvQsAG1pg2Q99P21jiqxqGBW1iRe/iXHsBRBxnpXKFI8QwbB4x5KmYLggNO5m34IQgOIu9SCRfR/WWiQ==", "dependencies": { "@floating-ui/core": "^1.0.0", "@floating-ui/utils": "^0.2.0" @@ -306,11 +306,11 @@ } }, "node_modules/@livekit/components-core": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/@livekit/components-core/-/components-core-0.10.0.tgz", - "integrity": "sha512-TSsIG2BRLABT5FP+5sueZgkByGYyFhv3UTb8fneWchvQRBHtiU9s4FF8SIoAw9z3znhwp1tKaJyuIyKp7k0Juw==", + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/@livekit/components-core/-/components-core-0.10.2.tgz", + "integrity": "sha512-hB/Qn52c1c/+p9zz+zeUy7A3wkHfDSG6S1o6Pq2nVrnYBTrC8SzpMyQ1pKI2j5O4oJl4QBFN2bvbDV5xWMghXw==", "dependencies": { - "@floating-ui/dom": "1.6.3", + "@floating-ui/dom": "1.6.4", "email-regex": "5.0.0", "loglevel": "1.9.1", "rxjs": "7.8.1" @@ -319,36 +319,36 @@ "node": ">=18" }, "peerDependencies": { - "@livekit/protocol": "^1.12.0", - "livekit-client": "^2.1.0", + "@livekit/protocol": "^1.16.0", + "livekit-client": "^2.1.5", "tslib": "^2.6.2" } }, "node_modules/@livekit/components-react": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@livekit/components-react/-/components-react-2.2.0.tgz", - "integrity": "sha512-TDa2YNBphkdf2dz85pEZs1UBl8wD/LHFeYupNoTqjtlLVlTXpr09Buv3/eegQFJhXoDSK6fAYqKZ4U/oYydv/w==", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@livekit/components-react/-/components-react-2.3.1.tgz", + "integrity": "sha512-AKb8RIPOrxSp/CKw18AkR/qmdOZtgV1nfp2AORqaCBmovtXolKbFOSOD2CC0LlUPZjZiN0DlFztl91dr6UZ9Zw==", "dependencies": { - "@livekit/components-core": "0.10.0", + "@livekit/components-core": "0.10.2", "@react-hook/latest": "1.0.3", - "clsx": "2.1.0", + "clsx": "2.1.1", "usehooks-ts": "2.16.0" }, "engines": { "node": ">=18" }, "peerDependencies": { - "@livekit/protocol": "^1.12.0", - "livekit-client": "^2.1.0", + "@livekit/protocol": "^1.16.0", + "livekit-client": "^2.1.5", "react": ">=18", "react-dom": ">=18", "tslib": "^2.6.2" } }, "node_modules/@livekit/protocol": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/@livekit/protocol/-/protocol-1.13.0.tgz", - "integrity": "sha512-M3U36VgRfb0VutWG6pnozXusL+mkYstbCctTLUyCIyye36Ztv1wA9zYpyYvz7VnsgmCn+g/g0eB2rnAkTVwcnA==", + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/@livekit/protocol/-/protocol-1.16.0.tgz", + "integrity": "sha512-xZZTZVh2FmWmUgNS3n+oGNbA4GcS4XOwhg8CWy75jenYxbgQ89ds7ixfMQ+F+oxktcXfJ1qsph086oRTlg8e5Q==", "dependencies": { "@bufbuild/protobuf": "^1.7.2" } @@ -1899,9 +1899,9 @@ "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==" }, "node_modules/clsx": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.0.tgz", - "integrity": "sha512-m3iNNWpd9rl3jvvcBnu70ylMdrXt8Vlq4HYadnU5fwcOtvkSQWPmj7amUcDT2qYI7risszBjI5AUIUox9D16pg==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", "engines": { "node": ">=6" } @@ -3761,11 +3761,11 @@ "dev": true }, "node_modules/livekit-client": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.1.1.tgz", - "integrity": "sha512-ffnXHQt210GPJ9sR846o7g0lCg/3TJqZxdu55mzQFS1YXGgn9PYKGzcAhKtuOsQ0NEkkn1zKQ0ABHBt7iADiqg==", + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.1.5.tgz", + "integrity": "sha512-8sc1ltfKRjy51Q/V/SaDpjptXBamm9LXhixKBYdXdQFZdew4hgqTGYlHIyee/IM9QSqAXk1W+uCtVfkmxPD1EA==", "dependencies": { - "@livekit/protocol": "1.13.0", + "@livekit/protocol": "1.16.0", "events": "^3.3.0", "loglevel": "^1.8.0", "sdp-transform": "^2.14.1", diff --git a/package.json b/package.json index b2eb123..d36680b 100644 --- a/package.json +++ b/package.json @@ -9,12 +9,12 @@ "lint": "next lint" }, "dependencies": { - "@livekit/components-react": "^2.1.1", + "@livekit/components-react": "^2.3.1", "@radix-ui/react-dropdown-menu": "^2.0.6", "cookies-next": "^4.1.1", "framer-motion": "^10.16.16", "js-yaml": "^4.1.0", - "livekit-client": "^2.1.0", + "livekit-client": "^2.1.5", "livekit-server-sdk": "^2.1.2", "lodash": "^4.17.21", "next": "^14.0.4", diff --git a/src/components/playground/Playground.tsx b/src/components/playground/Playground.tsx index 87270b3..0c3089f 100644 --- a/src/components/playground/Playground.tsx +++ b/src/components/playground/Playground.tsx @@ -15,14 +15,13 @@ import { import { AgentMultibandAudioVisualizer } from "@/components/visualization/AgentMultibandAudioVisualizer"; import { useConfig } from "@/hooks/useConfig"; import { useMultibandTrackVolume } from "@/hooks/useTrackVolume"; +import { TranscriptionTile } from "@/transcriptions/TranscriptionTile"; import { VideoTrack, - useChat, useConnectionState, useDataChannel, useLocalParticipant, useRemoteParticipants, - useRoomContext, useRoomInfo, useTracks, } from "@livekit/components-react"; @@ -65,7 +64,6 @@ export default function Playground({ const agentParticipant = participants.find((p) => p.isAgent); const isAgentConnected = agentParticipant !== undefined; - const { send: sendChat, chatMessages } = useChat(); const roomState = useConnectionState(); const tracks = useTracks(); @@ -132,33 +130,6 @@ export default function Playground({ [transcripts] ); - // combine transcripts and chat together - useEffect(() => { - const allMessages = [...transcripts]; - for (const msg of chatMessages) { - const isAgent = msg.from?.identity === agentParticipant?.identity; - const isSelf = msg.from?.identity === localParticipant?.identity; - let name = msg.from?.name; - if (!name) { - if (isAgent) { - name = "Agent"; - } else if (isSelf) { - name = "You"; - } else { - name = "Unknown"; - } - } - allMessages.push({ - name, - message: msg.message, - timestamp: msg?.timestamp, - isSelf: isSelf, - }); - } - allMessages.sort((a, b) => a.timestamp - b.timestamp); - setMessages(allMessages); - }, [transcripts, chatMessages, localParticipant, agentParticipant]); - useDataChannel(onDataReceived); const videoTileContent = useMemo(() => { @@ -248,14 +219,16 @@ export default function Playground({ ]); const chatTileContent = useMemo(() => { - return ( - - ); - }, [config.settings.theme_color, messages, sendChat]); + if (agentAudioTrack) { + return ( + + ); + } + return <>; + }, [config.settings.theme_color, agentAudioTrack]); const settingsTileContent = useMemo(() => { return ( diff --git a/src/transcriptions/TranscriptionTile.tsx b/src/transcriptions/TranscriptionTile.tsx new file mode 100644 index 0000000..d83e7a6 --- /dev/null +++ b/src/transcriptions/TranscriptionTile.tsx @@ -0,0 +1,113 @@ +import { ChatMessageType, ChatTile } from "@/components/chat/ChatTile"; +import { + Chat, + ChatMessage as ComponentsChatMessage, + TrackReferenceOrPlaceholder, + useChat, + useLocalParticipant, + useTrackTranscription, +} from "@livekit/components-react"; +import { + LocalParticipant, + Participant, + Track, + TranscriptionSegment, +} from "livekit-client"; +import { useEffect, useState } from "react"; + +export function TranscriptionTile({ + agentAudioTrack, + accentColor, +}: { + agentAudioTrack: TrackReferenceOrPlaceholder; + accentColor: string; +}) { + const agentMessages = useTrackTranscription(agentAudioTrack); + const localParticipant = useLocalParticipant(); + const localMessages = useTrackTranscription({ + publication: localParticipant.microphoneTrack, + source: Track.Source.Microphone, + participant: localParticipant.localParticipant, + }); + + const [transcripts, setTranscripts] = useState>( + new Map() + ); + const [messages, setMessages] = useState([]); + const { chatMessages, send: sendChat } = useChat(); + + // store transcripts + useEffect(() => { + agentMessages.segments.forEach((s) => + transcripts.set( + s.id, + segmentToChatMessage( + s, + transcripts.get(s.id), + agentAudioTrack.participant + ) + ) + ); + localMessages.segments.forEach((s) => + transcripts.set( + s.id, + segmentToChatMessage( + s, + transcripts.get(s.id), + localParticipant.localParticipant + ) + ) + ); + + const allMessages = Array.from(transcripts.values()); + for (const msg of chatMessages) { + const isAgent = + msg.from?.identity === agentAudioTrack.participant?.identity; + const isSelf = + msg.from?.identity === localParticipant.localParticipant.identity; + let name = msg.from?.name; + if (!name) { + if (isAgent) { + name = "Agent"; + } else if (isSelf) { + name = "You"; + } else { + name = "Unknown"; + } + } + allMessages.push({ + name, + message: msg.message, + timestamp: msg.timestamp, + isSelf: isSelf, + }); + } + allMessages.sort((a, b) => a.timestamp - b.timestamp); + setMessages(allMessages); + }, [ + transcripts, + chatMessages, + localParticipant.localParticipant, + agentAudioTrack.participant, + agentMessages.segments, + localMessages.segments, + ]); + + return ( + + ); +} + +function segmentToChatMessage( + s: TranscriptionSegment, + existingMessage: ChatMessageType | undefined, + participant: Participant +): ChatMessageType { + const msg: ChatMessageType = { + message: s.final ? s.text : `${s.text} ...`, + name: participant instanceof LocalParticipant ? "You" : "Agent", + isSelf: participant instanceof LocalParticipant, + timestamp: existingMessage?.timestamp ?? Date.now(), + }; + return msg; +}