Render transcriptions using useTrackTranscription hook (#60)

package-lock.json

@@ -8,12 +8,12 @@
       "name": "agents-playground",
       "version": "0.1.0",
       "dependencies": {
-        "@livekit/components-react": "^2.1.1",
+        "@livekit/components-react": "^2.3.1",
         "@radix-ui/react-dropdown-menu": "^2.0.6",
         "cookies-next": "^4.1.1",
         "framer-motion": "^10.16.16",
         "js-yaml": "^4.1.0",
-        "livekit-client": "^2.1.0",
+        "livekit-client": "^2.1.5",
         "livekit-server-sdk": "^2.1.2",
         "lodash": "^4.17.21",
         "next": "^14.0.4",
@@ -153,9 +153,9 @@
       }
     },
     "node_modules/@floating-ui/dom": {
-      "version": "1.6.3",
+      "version": "1.6.4",
-      "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.3.tgz",
+      "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.4.tgz",
-      "integrity": "sha512-RnDthu3mzPlQ31Ss/BTwQ1zjzIhr3lk1gZB1OC56h/1vEtaXkESrOqL5fQVMfXpwGtRwX+YsZBdyHtJMQnkArw==",
+      "integrity": "sha512-0G8R+zOvQsAG1pg2Q99P21jiqxqGBW1iRe/iXHsBRBxnpXKFI8QwbB4x5KmYLggNO5m34IQgOIu9SCRfR/WWiQ==",
       "dependencies": {
         "@floating-ui/core": "^1.0.0",
         "@floating-ui/utils": "^0.2.0"
@@ -306,11 +306,11 @@
       }
     },
     "node_modules/@livekit/components-core": {
-      "version": "0.10.0",
+      "version": "0.10.2",
-      "resolved": "https://registry.npmjs.org/@livekit/components-core/-/components-core-0.10.0.tgz",
+      "resolved": "https://registry.npmjs.org/@livekit/components-core/-/components-core-0.10.2.tgz",
-      "integrity": "sha512-TSsIG2BRLABT5FP+5sueZgkByGYyFhv3UTb8fneWchvQRBHtiU9s4FF8SIoAw9z3znhwp1tKaJyuIyKp7k0Juw==",
+      "integrity": "sha512-hB/Qn52c1c/+p9zz+zeUy7A3wkHfDSG6S1o6Pq2nVrnYBTrC8SzpMyQ1pKI2j5O4oJl4QBFN2bvbDV5xWMghXw==",
       "dependencies": {
-        "@floating-ui/dom": "1.6.3",
+        "@floating-ui/dom": "1.6.4",
         "email-regex": "5.0.0",
         "loglevel": "1.9.1",
         "rxjs": "7.8.1"
@@ -319,36 +319,36 @@
         "node": ">=18"
       },
       "peerDependencies": {
-        "@livekit/protocol": "^1.12.0",
+        "@livekit/protocol": "^1.16.0",
-        "livekit-client": "^2.1.0",
+        "livekit-client": "^2.1.5",
         "tslib": "^2.6.2"
       }
     },
     "node_modules/@livekit/components-react": {
-      "version": "2.2.0",
+      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/@livekit/components-react/-/components-react-2.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/@livekit/components-react/-/components-react-2.3.1.tgz",
-      "integrity": "sha512-TDa2YNBphkdf2dz85pEZs1UBl8wD/LHFeYupNoTqjtlLVlTXpr09Buv3/eegQFJhXoDSK6fAYqKZ4U/oYydv/w==",
+      "integrity": "sha512-AKb8RIPOrxSp/CKw18AkR/qmdOZtgV1nfp2AORqaCBmovtXolKbFOSOD2CC0LlUPZjZiN0DlFztl91dr6UZ9Zw==",
       "dependencies": {
-        "@livekit/components-core": "0.10.0",
+        "@livekit/components-core": "0.10.2",
         "@react-hook/latest": "1.0.3",
-        "clsx": "2.1.0",
+        "clsx": "2.1.1",
         "usehooks-ts": "2.16.0"
       },
       "engines": {
         "node": ">=18"
       },
       "peerDependencies": {
-        "@livekit/protocol": "^1.12.0",
+        "@livekit/protocol": "^1.16.0",
-        "livekit-client": "^2.1.0",
+        "livekit-client": "^2.1.5",
         "react": ">=18",
         "react-dom": ">=18",
         "tslib": "^2.6.2"
       }
     },
     "node_modules/@livekit/protocol": {
-      "version": "1.13.0",
+      "version": "1.16.0",
-      "resolved": "https://registry.npmjs.org/@livekit/protocol/-/protocol-1.13.0.tgz",
+      "resolved": "https://registry.npmjs.org/@livekit/protocol/-/protocol-1.16.0.tgz",
-      "integrity": "sha512-M3U36VgRfb0VutWG6pnozXusL+mkYstbCctTLUyCIyye36Ztv1wA9zYpyYvz7VnsgmCn+g/g0eB2rnAkTVwcnA==",
+      "integrity": "sha512-xZZTZVh2FmWmUgNS3n+oGNbA4GcS4XOwhg8CWy75jenYxbgQ89ds7ixfMQ+F+oxktcXfJ1qsph086oRTlg8e5Q==",
       "dependencies": {
         "@bufbuild/protobuf": "^1.7.2"
       }
@@ -1899,9 +1899,9 @@
       "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA=="
     },
     "node_modules/clsx": {
-      "version": "2.1.0",
+      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
-      "integrity": "sha512-m3iNNWpd9rl3jvvcBnu70ylMdrXt8Vlq4HYadnU5fwcOtvkSQWPmj7amUcDT2qYI7risszBjI5AUIUox9D16pg==",
+      "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==",
       "engines": {
         "node": ">=6"
       }
@@ -3761,11 +3761,11 @@
       "dev": true
     },
     "node_modules/livekit-client": {
-      "version": "2.1.1",
+      "version": "2.1.5",
-      "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.1.5.tgz",
-      "integrity": "sha512-ffnXHQt210GPJ9sR846o7g0lCg/3TJqZxdu55mzQFS1YXGgn9PYKGzcAhKtuOsQ0NEkkn1zKQ0ABHBt7iADiqg==",
+      "integrity": "sha512-8sc1ltfKRjy51Q/V/SaDpjptXBamm9LXhixKBYdXdQFZdew4hgqTGYlHIyee/IM9QSqAXk1W+uCtVfkmxPD1EA==",
       "dependencies": {
-        "@livekit/protocol": "1.13.0",
+        "@livekit/protocol": "1.16.0",
         "events": "^3.3.0",
         "loglevel": "^1.8.0",
         "sdp-transform": "^2.14.1",

package.json

@@ -9,12 +9,12 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@livekit/components-react": "^2.1.1",
+    "@livekit/components-react": "^2.3.1",
     "@radix-ui/react-dropdown-menu": "^2.0.6",
     "cookies-next": "^4.1.1",
     "framer-motion": "^10.16.16",
     "js-yaml": "^4.1.0",
-    "livekit-client": "^2.1.0",
+    "livekit-client": "^2.1.5",
     "livekit-server-sdk": "^2.1.2",
     "lodash": "^4.17.21",
     "next": "^14.0.4",

src/components/playground/Playground.tsx

@@ -15,14 +15,13 @@ import {
 import { AgentMultibandAudioVisualizer } from "@/components/visualization/AgentMultibandAudioVisualizer";
 import { useConfig } from "@/hooks/useConfig";
 import { useMultibandTrackVolume } from "@/hooks/useTrackVolume";
+import { TranscriptionTile } from "@/transcriptions/TranscriptionTile";
 import {
   VideoTrack,
-  useChat,
   useConnectionState,
   useDataChannel,
   useLocalParticipant,
   useRemoteParticipants,
-  useRoomContext,
   useRoomInfo,
   useTracks,
 } from "@livekit/components-react";
@@ -65,7 +64,6 @@ export default function Playground({
   const agentParticipant = participants.find((p) => p.isAgent);
   const isAgentConnected = agentParticipant !== undefined;
 
-  const { send: sendChat, chatMessages } = useChat();
   const roomState = useConnectionState();
   const tracks = useTracks();
 
@@ -132,33 +130,6 @@ export default function Playground({
     [transcripts]
   );
 
-  // combine transcripts and chat together
-  useEffect(() => {
-    const allMessages = [...transcripts];
-    for (const msg of chatMessages) {
-      const isAgent = msg.from?.identity === agentParticipant?.identity;
-      const isSelf = msg.from?.identity === localParticipant?.identity;
-      let name = msg.from?.name;
-      if (!name) {
-        if (isAgent) {
-          name = "Agent";
-        } else if (isSelf) {
-          name = "You";
-        } else {
-          name = "Unknown";
-        }
-      }
-      allMessages.push({
-        name,
-        message: msg.message,
-        timestamp: msg?.timestamp,
-        isSelf: isSelf,
-      });
-    }
-    allMessages.sort((a, b) => a.timestamp - b.timestamp);
-    setMessages(allMessages);
-  }, [transcripts, chatMessages, localParticipant, agentParticipant]);
-
   useDataChannel(onDataReceived);
 
   const videoTileContent = useMemo(() => {
@@ -248,14 +219,16 @@ export default function Playground({
   ]);
 
   const chatTileContent = useMemo(() => {
-    return (
+    if (agentAudioTrack) {
-      <ChatTile
+      return (
-        messages={messages}
+        <TranscriptionTile
-        accentColor={config.settings.theme_color}
+          agentAudioTrack={agentAudioTrack}
-        onSend={sendChat}
+          accentColor={config.settings.theme_color}
-      />
+        />
-    );
+      );
-  }, [config.settings.theme_color, messages, sendChat]);
+    }
+    return <></>;
+  }, [config.settings.theme_color, agentAudioTrack]);
 
   const settingsTileContent = useMemo(() => {
     return (

src/transcriptions/TranscriptionTile.tsx

@@ -0,0 +1,113 @@
+import { ChatMessageType, ChatTile } from "@/components/chat/ChatTile";
+import {
+  Chat,
+  ChatMessage as ComponentsChatMessage,
+  TrackReferenceOrPlaceholder,
+  useChat,
+  useLocalParticipant,
+  useTrackTranscription,
+} from "@livekit/components-react";
+import {
+  LocalParticipant,
+  Participant,
+  Track,
+  TranscriptionSegment,
+} from "livekit-client";
+import { useEffect, useState } from "react";
+
+export function TranscriptionTile({
+  agentAudioTrack,
+  accentColor,
+}: {
+  agentAudioTrack: TrackReferenceOrPlaceholder;
+  accentColor: string;
+}) {
+  const agentMessages = useTrackTranscription(agentAudioTrack);
+  const localParticipant = useLocalParticipant();
+  const localMessages = useTrackTranscription({
+    publication: localParticipant.microphoneTrack,
+    source: Track.Source.Microphone,
+    participant: localParticipant.localParticipant,
+  });
+
+  const [transcripts, setTranscripts] = useState<Map<string, ChatMessageType>>(
+    new Map()
+  );
+  const [messages, setMessages] = useState<ChatMessageType[]>([]);
+  const { chatMessages, send: sendChat } = useChat();
+
+  // store transcripts
+  useEffect(() => {
+    agentMessages.segments.forEach((s) =>
+      transcripts.set(
+        s.id,
+        segmentToChatMessage(
+          s,
+          transcripts.get(s.id),
+          agentAudioTrack.participant
+        )
+      )
+    );
+    localMessages.segments.forEach((s) =>
+      transcripts.set(
+        s.id,
+        segmentToChatMessage(
+          s,
+          transcripts.get(s.id),
+          localParticipant.localParticipant
+        )
+      )
+    );
+
+    const allMessages = Array.from(transcripts.values());
+    for (const msg of chatMessages) {
+      const isAgent =
+        msg.from?.identity === agentAudioTrack.participant?.identity;
+      const isSelf =
+        msg.from?.identity === localParticipant.localParticipant.identity;
+      let name = msg.from?.name;
+      if (!name) {
+        if (isAgent) {
+          name = "Agent";
+        } else if (isSelf) {
+          name = "You";
+        } else {
+          name = "Unknown";
+        }
+      }
+      allMessages.push({
+        name,
+        message: msg.message,
+        timestamp: msg.timestamp,
+        isSelf: isSelf,
+      });
+    }
+    allMessages.sort((a, b) => a.timestamp - b.timestamp);
+    setMessages(allMessages);
+  }, [
+    transcripts,
+    chatMessages,
+    localParticipant.localParticipant,
+    agentAudioTrack.participant,
+    agentMessages.segments,
+    localMessages.segments,
+  ]);
+
+  return (
+    <ChatTile messages={messages} accentColor={accentColor} onSend={sendChat} />
+  );
+}
+
+function segmentToChatMessage(
+  s: TranscriptionSegment,
+  existingMessage: ChatMessageType | undefined,
+  participant: Participant
+): ChatMessageType {
+  const msg: ChatMessageType = {
+    message: s.final ? s.text : `${s.text} ...`,
+    name: participant instanceof LocalParticipant ? "You" : "Agent",
+    isSelf: participant instanceof LocalParticipant,
+    timestamp: existingMessage?.timestamp ?? Date.now(),
+  };
+  return msg;
+}