Enhance voice interaction and transcript handling in the assistant
- Add a new Docker configuration for the UI in launch.json to facilitate development. - Refactor pipeline.py to integrate a TranscriptProcessor for managing user and assistant transcripts, including event handlers for real-time updates and message handling. - Update useVoicePreview.ts to establish a data channel for sending and receiving text messages, improving interaction flow. - Modify AssistantPage.tsx to support displaying chat messages and sending user input, enhancing the user experience during voice interactions. - Revise DebugTranscriptPanel to dynamically render chat messages with timestamps, improving the visual representation of conversation history.
This commit is contained in:
@@ -9,6 +9,10 @@
|
||||
* client → {type:"ice-candidate", payload:{pc_id, candidate:{...}}}
|
||||
* 音频本身走 WebRTC 媒体流(Opus),不经 ws;后端 TTS 帧从 ontrack 拿到直接播放。
|
||||
*
|
||||
* 另开一条 data channel 与后端管线(pipeline.py)互通应用消息:
|
||||
* client → {type:"user-text", text} 文字输入(打断并触发新回复)
|
||||
* server → {type:"transcript", role, content, timestamp} 用户/助手最终转写(聊天记录)
|
||||
*
|
||||
* 纯本机(localhost)即可跑:localhost 是 secure context,麦克风可用,ws 用明文。
|
||||
* 局域网/别的设备要 https+wss,见 deploy/README.md。
|
||||
*/
|
||||
@@ -19,6 +23,14 @@ import { API_BASE } from "@/lib/api";
|
||||
|
||||
export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed";
|
||||
|
||||
export type ChatMessage = {
|
||||
id: string;
|
||||
role: "user" | "assistant";
|
||||
content: string;
|
||||
/** 后端给的 ISO 时间戳 */
|
||||
timestamp: string;
|
||||
};
|
||||
|
||||
// http→ws、https→wss,自动跟随 API 基址(同源反代时也对)
|
||||
function wsBaseUrl(): string {
|
||||
const url = new URL(API_BASE, window.location.origin);
|
||||
@@ -62,11 +74,14 @@ export function useVoicePreview(assistantId: string | null) {
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [micWarning, setMicWarning] = useState<string | null>(null);
|
||||
const [localStream, setLocalStream] = useState<MediaStream | null>(null);
|
||||
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
const pcRef = useRef<RTCPeerConnection | null>(null);
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
const dataChannelRef = useRef<RTCDataChannel | null>(null);
|
||||
const localStreamRef = useRef<MediaStream | null>(null);
|
||||
const startingRef = useRef(false);
|
||||
const messageSeqRef = useRef(0);
|
||||
|
||||
const releaseResources = useCallback(() => {
|
||||
const ws = wsRef.current;
|
||||
@@ -78,6 +93,13 @@ export function useVoicePreview(assistantId: string | null) {
|
||||
ws.close();
|
||||
}
|
||||
|
||||
const channel = dataChannelRef.current;
|
||||
dataChannelRef.current = null;
|
||||
if (channel) {
|
||||
channel.onmessage = null;
|
||||
channel.close();
|
||||
}
|
||||
|
||||
const pc = pcRef.current;
|
||||
pcRef.current = null;
|
||||
if (pc) {
|
||||
@@ -122,6 +144,7 @@ export function useVoicePreview(assistantId: string | null) {
|
||||
startingRef.current = true;
|
||||
setError(null);
|
||||
setMicWarning(null);
|
||||
setMessages([]); // 新会话清空上一轮聊天记录
|
||||
setStatus("connecting");
|
||||
|
||||
// 麦克风是可选的:获取失败时继续建立仅接收后端音频的 WebRTC 会话。
|
||||
@@ -213,6 +236,36 @@ export function useVoicePreview(assistantId: string | null) {
|
||||
);
|
||||
};
|
||||
|
||||
// 应用消息通道:收后端转写(聊天记录),发文字输入。
|
||||
// 由浏览器侧主动创建,后端 SmallWebRTCConnection 的 on("datachannel") 会接住。
|
||||
const channel = pc.createDataChannel("chat");
|
||||
dataChannelRef.current = channel;
|
||||
channel.onmessage = (event) => {
|
||||
try {
|
||||
const msg = JSON.parse(event.data);
|
||||
if (
|
||||
msg?.type === "transcript" &&
|
||||
(msg.role === "user" || msg.role === "assistant") &&
|
||||
typeof msg.content === "string" &&
|
||||
msg.content.trim()
|
||||
) {
|
||||
messageSeqRef.current += 1;
|
||||
const next: ChatMessage = {
|
||||
id: `msg-${messageSeqRef.current}`,
|
||||
role: msg.role,
|
||||
content: msg.content,
|
||||
timestamp:
|
||||
typeof msg.timestamp === "string"
|
||||
? msg.timestamp
|
||||
: new Date().toISOString(),
|
||||
};
|
||||
setMessages((prev) => [...prev, next]);
|
||||
}
|
||||
} catch {
|
||||
/* 非 JSON / 未知消息,忽略 */
|
||||
}
|
||||
};
|
||||
|
||||
pc.ontrack = (e) => {
|
||||
if (e.track.kind === "audio" && audioRef.current) {
|
||||
audioRef.current.srcObject =
|
||||
@@ -268,6 +321,16 @@ export function useVoicePreview(assistantId: string | null) {
|
||||
}
|
||||
}, [assistantId, fail]);
|
||||
|
||||
// 发送文字消息:后端先打断当前播报,再按用户输入触发新回复。
|
||||
// 成功返回 true;通道未就绪(未开始对话/连接中)返回 false。
|
||||
const sendText = useCallback((text: string): boolean => {
|
||||
const trimmed = text.trim();
|
||||
const channel = dataChannelRef.current;
|
||||
if (!trimmed || !channel || channel.readyState !== "open") return false;
|
||||
channel.send(JSON.stringify({ type: "user-text", text: trimmed }));
|
||||
return true;
|
||||
}, []);
|
||||
|
||||
// 卸载时收尾
|
||||
useEffect(() => releaseResources, [releaseResources]);
|
||||
|
||||
@@ -276,6 +339,8 @@ export function useVoicePreview(assistantId: string | null) {
|
||||
error,
|
||||
micWarning,
|
||||
localStream,
|
||||
messages,
|
||||
sendText,
|
||||
connect,
|
||||
disconnect,
|
||||
audioRef,
|
||||
|
||||
Reference in New Issue
Block a user