wx44wx/livekit-znjj-demo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Render transcriptions using useTrackTranscription hook (#60)

Browse Source
This commit is contained in:
lukasIO
2024-05-29 10:45:14 +02:00
committed by GitHub
parent 401761dbb3
commit 6e2fd46908
4 changed files with 154 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
package-lock.json generated
View File

@@ -8,12 +8,12 @@
"name": "agents-playground",
"version": "0.1.0",
"dependencies": {
"@livekit/components-react": "^2.1.1",
"@livekit/components-react": "^2.3.1",
"@radix-ui/react-dropdown-menu": "^2.0.6",
"cookies-next": "^4.1.1",
"framer-motion": "^10.16.16",
"js-yaml": "^4.1.0",
"livekit-client": "^2.1.0",
"livekit-client": "^2.1.5",
"livekit-server-sdk": "^2.1.2",
"lodash": "^4.17.21",
"next": "^14.0.4",
@@ -153,9 +153,9 @@
}
},
"node_modules/@floating-ui/dom": {
"version": "1.6.3",
"resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.3.tgz",
"integrity": "sha512-RnDthu3mzPlQ31Ss/BTwQ1zjzIhr3lk1gZB1OC56h/1vEtaXkESrOqL5fQVMfXpwGtRwX+YsZBdyHtJMQnkArw==",
"version": "1.6.4",
"resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.4.tgz",
"integrity": "sha512-0G8R+zOvQsAG1pg2Q99P21jiqxqGBW1iRe/iXHsBRBxnpXKFI8QwbB4x5KmYLggNO5m34IQgOIu9SCRfR/WWiQ==",
"dependencies": {
"@floating-ui/core": "^1.0.0",
"@floating-ui/utils": "^0.2.0"
@@ -306,11 +306,11 @@
}
},
"node_modules/@livekit/components-core": {
"version": "0.10.0",
"resolved": "https://registry.npmjs.org/@livekit/components-core/-/components-core-0.10.0.tgz",
"integrity": "sha512-TSsIG2BRLABT5FP+5sueZgkByGYyFhv3UTb8fneWchvQRBHtiU9s4FF8SIoAw9z3znhwp1tKaJyuIyKp7k0Juw==",
"version": "0.10.2",
"resolved": "https://registry.npmjs.org/@livekit/components-core/-/components-core-0.10.2.tgz",
"integrity": "sha512-hB/Qn52c1c/+p9zz+zeUy7A3wkHfDSG6S1o6Pq2nVrnYBTrC8SzpMyQ1pKI2j5O4oJl4QBFN2bvbDV5xWMghXw==",
"dependencies": {
"@floating-ui/dom": "1.6.3",
"@floating-ui/dom": "1.6.4",
"email-regex": "5.0.0",
"loglevel": "1.9.1",
"rxjs": "7.8.1"
@@ -319,36 +319,36 @@
"node": ">=18"
},
"peerDependencies": {
"@livekit/protocol": "^1.12.0",
"livekit-client": "^2.1.0",
"@livekit/protocol": "^1.16.0",
"livekit-client": "^2.1.5",
"tslib": "^2.6.2"
}
},
"node_modules/@livekit/components-react": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/@livekit/components-react/-/components-react-2.2.0.tgz",
"integrity": "sha512-TDa2YNBphkdf2dz85pEZs1UBl8wD/LHFeYupNoTqjtlLVlTXpr09Buv3/eegQFJhXoDSK6fAYqKZ4U/oYydv/w==",
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/@livekit/components-react/-/components-react-2.3.1.tgz",
"integrity": "sha512-AKb8RIPOrxSp/CKw18AkR/qmdOZtgV1nfp2AORqaCBmovtXolKbFOSOD2CC0LlUPZjZiN0DlFztl91dr6UZ9Zw==",
"dependencies": {
"@livekit/components-core": "0.10.0",
"@livekit/components-core": "0.10.2",
"@react-hook/latest": "1.0.3",
"clsx": "2.1.0",
"clsx": "2.1.1",
"usehooks-ts": "2.16.0"
},
"engines": {
"node": ">=18"
},
"peerDependencies": {
"@livekit/protocol": "^1.12.0",
"livekit-client": "^2.1.0",
"@livekit/protocol": "^1.16.0",
"livekit-client": "^2.1.5",
"react": ">=18",
"react-dom": ">=18",
"tslib": "^2.6.2"
}
},
"node_modules/@livekit/protocol": {
"version": "1.13.0",
"resolved": "https://registry.npmjs.org/@livekit/protocol/-/protocol-1.13.0.tgz",
"integrity": "sha512-M3U36VgRfb0VutWG6pnozXusL+mkYstbCctTLUyCIyye36Ztv1wA9zYpyYvz7VnsgmCn+g/g0eB2rnAkTVwcnA==",
"version": "1.16.0",
"resolved": "https://registry.npmjs.org/@livekit/protocol/-/protocol-1.16.0.tgz",
"integrity": "sha512-xZZTZVh2FmWmUgNS3n+oGNbA4GcS4XOwhg8CWy75jenYxbgQ89ds7ixfMQ+F+oxktcXfJ1qsph086oRTlg8e5Q==",
"dependencies": {
"@bufbuild/protobuf": "^1.7.2"
}
@@ -1899,9 +1899,9 @@
"integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA=="
},
"node_modules/clsx": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.0.tgz",
"integrity": "sha512-m3iNNWpd9rl3jvvcBnu70ylMdrXt8Vlq4HYadnU5fwcOtvkSQWPmj7amUcDT2qYI7risszBjI5AUIUox9D16pg==",
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
"integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==",
"engines": {
"node": ">=6"
}
@@ -3761,11 +3761,11 @@
"dev": true
},
"node_modules/livekit-client": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.1.1.tgz",
"integrity": "sha512-ffnXHQt210GPJ9sR846o7g0lCg/3TJqZxdu55mzQFS1YXGgn9PYKGzcAhKtuOsQ0NEkkn1zKQ0ABHBt7iADiqg==",
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.1.5.tgz",
"integrity": "sha512-8sc1ltfKRjy51Q/V/SaDpjptXBamm9LXhixKBYdXdQFZdew4hgqTGYlHIyee/IM9QSqAXk1W+uCtVfkmxPD1EA==",
"dependencies": {
"@livekit/protocol": "1.13.0",
"@livekit/protocol": "1.16.0",
"events": "^3.3.0",
"loglevel": "^1.8.0",
"sdp-transform": "^2.14.1",

4
package.json
View File

@@ -9,12 +9,12 @@
"lint": "next lint"
},
"dependencies": {
"@livekit/components-react": "^2.1.1",
"@livekit/components-react": "^2.3.1",
"@radix-ui/react-dropdown-menu": "^2.0.6",
"cookies-next": "^4.1.1",
"framer-motion": "^10.16.16",
"js-yaml": "^4.1.0",
"livekit-client": "^2.1.0",
"livekit-client": "^2.1.5",
"livekit-server-sdk": "^2.1.2",
"lodash": "^4.17.21",
"next": "^14.0.4",

49
src/components/playground/Playground.tsx
View File

@@ -15,14 +15,13 @@ import {
import { AgentMultibandAudioVisualizer } from "@/components/visualization/AgentMultibandAudioVisualizer";
import { useConfig } from "@/hooks/useConfig";
import { useMultibandTrackVolume } from "@/hooks/useTrackVolume";
import { TranscriptionTile } from "@/transcriptions/TranscriptionTile";
import {
VideoTrack,
useChat,
useConnectionState,
useDataChannel,
useLocalParticipant,
useRemoteParticipants,
useRoomContext,
useRoomInfo,
useTracks,
} from "@livekit/components-react";
@@ -65,7 +64,6 @@ export default function Playground({
const agentParticipant = participants.find((p) => p.isAgent);
const isAgentConnected = agentParticipant !== undefined;
const { send: sendChat, chatMessages } = useChat();
const roomState = useConnectionState();
const tracks = useTracks();
@@ -132,33 +130,6 @@ export default function Playground({
[transcripts]
);
// combine transcripts and chat together
useEffect(() => {
const allMessages = [...transcripts];
for (const msg of chatMessages) {
const isAgent = msg.from?.identity === agentParticipant?.identity;
const isSelf = msg.from?.identity === localParticipant?.identity;
let name = msg.from?.name;
if (!name) {
if (isAgent) {
name = "Agent";
} else if (isSelf) {
name = "You";
} else {
name = "Unknown";
}
}
allMessages.push({
name,
message: msg.message,
timestamp: msg?.timestamp,
isSelf: isSelf,
});
}
allMessages.sort((a, b) => a.timestamp - b.timestamp);
setMessages(allMessages);
}, [transcripts, chatMessages, localParticipant, agentParticipant]);
useDataChannel(onDataReceived);
const videoTileContent = useMemo(() => {
@@ -248,14 +219,16 @@ export default function Playground({
]);
const chatTileContent = useMemo(() => {
return (
<ChatTile
messages={messages}
accentColor={config.settings.theme_color}
onSend={sendChat}
/>
);
}, [config.settings.theme_color, messages, sendChat]);
if (agentAudioTrack) {
return (
<TranscriptionTile
agentAudioTrack={agentAudioTrack}
accentColor={config.settings.theme_color}
/>
);
}
return <></>;
}, [config.settings.theme_color, agentAudioTrack]);
const settingsTileContent = useMemo(() => {
return (

113
src/transcriptions/TranscriptionTile.tsx Normal file
View File

@@ -0,0 +1,113 @@
import { ChatMessageType, ChatTile } from "@/components/chat/ChatTile";
import {
Chat,
ChatMessage as ComponentsChatMessage,
TrackReferenceOrPlaceholder,
useChat,
useLocalParticipant,
useTrackTranscription,
} from "@livekit/components-react";
import {
LocalParticipant,
Participant,
Track,
TranscriptionSegment,
} from "livekit-client";
import { useEffect, useState } from "react";
export function TranscriptionTile({
agentAudioTrack,
accentColor,
}: {
agentAudioTrack: TrackReferenceOrPlaceholder;
accentColor: string;
}) {
const agentMessages = useTrackTranscription(agentAudioTrack);
const localParticipant = useLocalParticipant();
const localMessages = useTrackTranscription({
publication: localParticipant.microphoneTrack,
source: Track.Source.Microphone,
participant: localParticipant.localParticipant,
});
const [transcripts, setTranscripts] = useState<Map<string, ChatMessageType>>(
new Map()
);
const [messages, setMessages] = useState<ChatMessageType[]>([]);
const { chatMessages, send: sendChat } = useChat();
// store transcripts
useEffect(() => {
agentMessages.segments.forEach((s) =>
transcripts.set(
s.id,
segmentToChatMessage(
s,
transcripts.get(s.id),
agentAudioTrack.participant
)
)
);
localMessages.segments.forEach((s) =>
transcripts.set(
s.id,
segmentToChatMessage(
s,
transcripts.get(s.id),
localParticipant.localParticipant
)
)
);
const allMessages = Array.from(transcripts.values());
for (const msg of chatMessages) {
const isAgent =
msg.from?.identity === agentAudioTrack.participant?.identity;
const isSelf =
msg.from?.identity === localParticipant.localParticipant.identity;
let name = msg.from?.name;
if (!name) {
if (isAgent) {
name = "Agent";
} else if (isSelf) {
name = "You";
} else {
name = "Unknown";
}
}
allMessages.push({
name,
message: msg.message,
timestamp: msg.timestamp,
isSelf: isSelf,
});
}
allMessages.sort((a, b) => a.timestamp - b.timestamp);
setMessages(allMessages);
}, [
transcripts,
chatMessages,
localParticipant.localParticipant,
agentAudioTrack.participant,
agentMessages.segments,
localMessages.segments,
]);
return (
<ChatTile messages={messages} accentColor={accentColor} onSend={sendChat} />
);
}
function segmentToChatMessage(
s: TranscriptionSegment,
existingMessage: ChatMessageType | undefined,
participant: Participant
): ChatMessageType {
const msg: ChatMessageType = {
message: s.final ? s.text : `${s.text} ...`,
name: participant instanceof LocalParticipant ? "You" : "Agent",
isSelf: participant instanceof LocalParticipant,
timestamp: existingMessage?.timestamp ?? Date.now(),
};
return msg;
}