Render transcriptions using useTrackTranscription hook (#60)

src/components/playground/Playground.tsx

@@ -15,14 +15,13 @@ import {
 import { AgentMultibandAudioVisualizer } from "@/components/visualization/AgentMultibandAudioVisualizer";
 import { useConfig } from "@/hooks/useConfig";
 import { useMultibandTrackVolume } from "@/hooks/useTrackVolume";
+import { TranscriptionTile } from "@/transcriptions/TranscriptionTile";
 import {
   VideoTrack,
-  useChat,
   useConnectionState,
   useDataChannel,
   useLocalParticipant,
   useRemoteParticipants,
-  useRoomContext,
   useRoomInfo,
   useTracks,
 } from "@livekit/components-react";
@@ -65,7 +64,6 @@ export default function Playground({
   const agentParticipant = participants.find((p) => p.isAgent);
   const isAgentConnected = agentParticipant !== undefined;
 
-  const { send: sendChat, chatMessages } = useChat();
   const roomState = useConnectionState();
   const tracks = useTracks();
 
@@ -132,33 +130,6 @@ export default function Playground({
     [transcripts]
   );
 
-  // combine transcripts and chat together
-  useEffect(() => {
-    const allMessages = [...transcripts];
-    for (const msg of chatMessages) {
-      const isAgent = msg.from?.identity === agentParticipant?.identity;
-      const isSelf = msg.from?.identity === localParticipant?.identity;
-      let name = msg.from?.name;
-      if (!name) {
-        if (isAgent) {
-          name = "Agent";
-        } else if (isSelf) {
-          name = "You";
-        } else {
-          name = "Unknown";
-        }
-      }
-      allMessages.push({
-        name,
-        message: msg.message,
-        timestamp: msg?.timestamp,
-        isSelf: isSelf,
-      });
-    }
-    allMessages.sort((a, b) => a.timestamp - b.timestamp);
-    setMessages(allMessages);
-  }, [transcripts, chatMessages, localParticipant, agentParticipant]);
-
   useDataChannel(onDataReceived);
 
   const videoTileContent = useMemo(() => {
@@ -248,14 +219,16 @@ export default function Playground({
   ]);
 
   const chatTileContent = useMemo(() => {
-    return (
-      <ChatTile
-        messages={messages}
-        accentColor={config.settings.theme_color}
-        onSend={sendChat}
-      />
-    );
-  }, [config.settings.theme_color, messages, sendChat]);
+    if (agentAudioTrack) {
+      return (
+        <TranscriptionTile
+          agentAudioTrack={agentAudioTrack}
+          accentColor={config.settings.theme_color}
+        />
+      );
+    }
+    return <></>;
+  }, [config.settings.theme_color, agentAudioTrack]);
 
   const settingsTileContent = useMemo(() => {
     return (

src/transcriptions/TranscriptionTile.tsx

@@ -0,0 +1,113 @@
+import { ChatMessageType, ChatTile } from "@/components/chat/ChatTile";
+import {
+  Chat,
+  ChatMessage as ComponentsChatMessage,
+  TrackReferenceOrPlaceholder,
+  useChat,
+  useLocalParticipant,
+  useTrackTranscription,
+} from "@livekit/components-react";
+import {
+  LocalParticipant,
+  Participant,
+  Track,
+  TranscriptionSegment,
+} from "livekit-client";
+import { useEffect, useState } from "react";
+
+export function TranscriptionTile({
+  agentAudioTrack,
+  accentColor,
+}: {
+  agentAudioTrack: TrackReferenceOrPlaceholder;
+  accentColor: string;
+}) {
+  const agentMessages = useTrackTranscription(agentAudioTrack);
+  const localParticipant = useLocalParticipant();
+  const localMessages = useTrackTranscription({
+    publication: localParticipant.microphoneTrack,
+    source: Track.Source.Microphone,
+    participant: localParticipant.localParticipant,
+  });
+
+  const [transcripts, setTranscripts] = useState<Map<string, ChatMessageType>>(
+    new Map()
+  );
+  const [messages, setMessages] = useState<ChatMessageType[]>([]);
+  const { chatMessages, send: sendChat } = useChat();
+
+  // store transcripts
+  useEffect(() => {
+    agentMessages.segments.forEach((s) =>
+      transcripts.set(
+        s.id,
+        segmentToChatMessage(
+          s,
+          transcripts.get(s.id),
+          agentAudioTrack.participant
+        )
+      )
+    );
+    localMessages.segments.forEach((s) =>
+      transcripts.set(
+        s.id,
+        segmentToChatMessage(
+          s,
+          transcripts.get(s.id),
+          localParticipant.localParticipant
+        )
+      )
+    );
+
+    const allMessages = Array.from(transcripts.values());
+    for (const msg of chatMessages) {
+      const isAgent =
+        msg.from?.identity === agentAudioTrack.participant?.identity;
+      const isSelf =
+        msg.from?.identity === localParticipant.localParticipant.identity;
+      let name = msg.from?.name;
+      if (!name) {
+        if (isAgent) {
+          name = "Agent";
+        } else if (isSelf) {
+          name = "You";
+        } else {
+          name = "Unknown";
+        }
+      }
+      allMessages.push({
+        name,
+        message: msg.message,
+        timestamp: msg.timestamp,
+        isSelf: isSelf,
+      });
+    }
+    allMessages.sort((a, b) => a.timestamp - b.timestamp);
+    setMessages(allMessages);
+  }, [
+    transcripts,
+    chatMessages,
+    localParticipant.localParticipant,
+    agentAudioTrack.participant,
+    agentMessages.segments,
+    localMessages.segments,
+  ]);
+
+  return (
+    <ChatTile messages={messages} accentColor={accentColor} onSend={sendChat} />
+  );
+}
+
+function segmentToChatMessage(
+  s: TranscriptionSegment,
+  existingMessage: ChatMessageType | undefined,
+  participant: Participant
+): ChatMessageType {
+  const msg: ChatMessageType = {
+    message: s.final ? s.text : `${s.text} ...`,
+    name: participant instanceof LocalParticipant ? "You" : "Agent",
+    isSelf: participant instanceof LocalParticipant,
+    timestamp: existingMessage?.timestamp ?? Date.now(),
+  };
+  return msg;
+}