b22a9e1045
- Introduce mechanisms in the pipeline to ensure that the end call process waits for the completion of the end speech before hanging up, improving user experience during call termination. - Update the useVoicePreview hook to handle server-initiated call endings gracefully, distinguishing between normal and error disconnections. - Adjust TTS stop frame timeout settings to optimize the timing of call terminations, ensuring timely responses without unnecessary delays. - Refactor related components to support the new end call logic, enhancing overall workflow management and user interaction.
561 lines
19 KiB
TypeScript
561 lines
19 KiB
TypeScript
"use client";
|
|
|
|
/**
|
|
* 语音预览:把麦克风接到后端 /ws/voice(WebRTC 信令),听到助手实时回应。
|
|
*
|
|
* 走原生 RTCPeerConnection + 一条 ws 信令通道,与后端 voice_webrtc.py 的约定对齐:
|
|
* client → {type:"offer", payload:{pc_id, sdp, type, assistant_id}}
|
|
* server → {type:"answer", payload:{pc_id, sdp, type}}
|
|
* client → {type:"ice-candidate", payload:{pc_id, candidate:{...}}}
|
|
* 音频本身走 WebRTC 媒体流(Opus),不经 ws;后端 TTS 帧从 ontrack 拿到直接播放。
|
|
*
|
|
* 另开一条 data channel 与后端管线(pipeline.py)互通应用消息:
|
|
* client → {type:"user-text", text} 文字输入(打断并触发新回复)
|
|
* server → {type:"transcript", role, content, timestamp} 用户/助手最终转写(聊天记录)
|
|
*
|
|
* 纯本机(localhost)即可跑:localhost 是 secure context,麦克风可用,ws 用明文。
|
|
* 局域网/别的设备要 https+wss,见 deploy/README.md。
|
|
*/
|
|
|
|
import { useCallback, useEffect, useRef, useState } from "react";
|
|
|
|
import { API_BASE } from "@/lib/api";
|
|
|
|
export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed";
|
|
|
|
export type ChatMessage = {
|
|
id: string;
|
|
role: "user" | "assistant";
|
|
content: string;
|
|
/** 后端给的 ISO 时间戳 */
|
|
timestamp: string;
|
|
sequence: number;
|
|
turnId?: string;
|
|
streaming?: boolean;
|
|
};
|
|
|
|
// http→ws、https→wss,自动跟随 API 基址(同源反代时也对)
|
|
function wsBaseUrl(): string {
|
|
const url = new URL(API_BASE, window.location.origin);
|
|
url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
|
|
return url.toString().replace(/\/$/, "");
|
|
}
|
|
|
|
function generatePcId(): string {
|
|
const bytes = new Uint8Array(16);
|
|
crypto.getRandomValues(bytes);
|
|
return (
|
|
"PC-" +
|
|
Array.from(bytes)
|
|
.map((b) => b.toString(16).padStart(2, "0"))
|
|
.join("")
|
|
);
|
|
}
|
|
|
|
function errorMessage(error: unknown, fallback: string): string {
|
|
if (error instanceof Error && error.message) return error.message;
|
|
return fallback;
|
|
}
|
|
|
|
function messageOrder(message: ChatMessage): number {
|
|
const timestamp = Date.parse(message.timestamp);
|
|
return Number.isNaN(timestamp) ? Number.MAX_SAFE_INTEGER : timestamp;
|
|
}
|
|
|
|
function sortMessages(messages: ChatMessage[]): ChatMessage[] {
|
|
return messages.sort(
|
|
(a, b) => messageOrder(a) - messageOrder(b) || a.sequence - b.sequence,
|
|
);
|
|
}
|
|
|
|
function microphoneErrorMessage(error: unknown): string {
|
|
if (error instanceof DOMException) {
|
|
if (error.name === "NotAllowedError") {
|
|
return "麦克风权限被拒绝，请在浏览器网站设置中允许 localhost 使用麦克风。";
|
|
}
|
|
if (error.name === "NotFoundError") {
|
|
return "未检测到可用麦克风，请连接或启用麦克风后重试。";
|
|
}
|
|
if (error.name === "NotReadableError") {
|
|
return "麦克风正被其他应用占用，或系统未允许浏览器访问麦克风。";
|
|
}
|
|
}
|
|
return errorMessage(error, "无法访问麦克风。");
|
|
}
|
|
|
|
export function useVoicePreview(
|
|
assistantId: string | null,
|
|
onNodeActive?: (nodeId: string | null) => void,
|
|
) {
|
|
const [status, setStatus] = useState<VoicePreviewStatus>("idle");
|
|
const [error, setError] = useState<string | null>(null);
|
|
const [micWarning, setMicWarning] = useState<string | null>(null);
|
|
const [localStream, setLocalStream] = useState<MediaStream | null>(null);
|
|
// 远端(助手 TTS)媒体流:除挂到 <audio> 播放外,也暴露给波形可视化
|
|
const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
|
|
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
|
// 可选麦克风列表与当前选择(空串表示交给浏览器选默认设备)
|
|
const [audioInputs, setAudioInputs] = useState<MediaDeviceInfo[]>([]);
|
|
const [selectedDeviceId, setSelectedDeviceId] = useState<string>("");
|
|
const selectedDeviceIdRef = useRef("");
|
|
const audioRef = useRef<HTMLAudioElement | null>(null);
|
|
const pcRef = useRef<RTCPeerConnection | null>(null);
|
|
const wsRef = useRef<WebSocket | null>(null);
|
|
const dataChannelRef = useRef<RTCDataChannel | null>(null);
|
|
const localStreamRef = useRef<MediaStream | null>(null);
|
|
const startingRef = useRef(false);
|
|
const messageSeqRef = useRef(0);
|
|
// 后端主动结束(工作流走到结束节点)标记:据此把随后的断开当作正常结束而非报错
|
|
const endedByServerRef = useRef(false);
|
|
// 工作流激活节点回调存进 ref,避免把它挂进 connect 依赖反复重建连接
|
|
const onNodeActiveRef = useRef(onNodeActive);
|
|
useEffect(() => {
|
|
onNodeActiveRef.current = onNodeActive;
|
|
}, [onNodeActive]);
|
|
|
|
// 枚举可用麦克风。未授权前 label 为空,授权(连接)后再刷新即可拿到名称。
|
|
const refreshDevices = useCallback(async () => {
|
|
if (!navigator.mediaDevices?.enumerateDevices) return;
|
|
try {
|
|
const devices = await navigator.mediaDevices.enumerateDevices();
|
|
// 未授权时设备可能没有 deviceId(空串),无法选择,直接过滤掉
|
|
const inputs = devices.filter((d) => d.kind === "audioinput" && d.deviceId);
|
|
setAudioInputs(inputs);
|
|
// 选中的设备已被拔出时,回退到浏览器默认设备
|
|
if (
|
|
selectedDeviceIdRef.current &&
|
|
!inputs.some((d) => d.deviceId === selectedDeviceIdRef.current)
|
|
) {
|
|
setSelectedDeviceId("");
|
|
}
|
|
} catch {
|
|
/* 忽略枚举失败 */
|
|
}
|
|
}, []);
|
|
|
|
// connect/refreshDevices 在回调里读最新选择,避免把它们挂进依赖反复重建
|
|
useEffect(() => {
|
|
selectedDeviceIdRef.current = selectedDeviceId;
|
|
}, [selectedDeviceId]);
|
|
|
|
useEffect(() => {
|
|
if (!navigator.mediaDevices?.enumerateDevices) return;
|
|
// eslint-disable-next-line react-hooks/set-state-in-effect
|
|
void refreshDevices();
|
|
navigator.mediaDevices.addEventListener("devicechange", refreshDevices);
|
|
return () => {
|
|
navigator.mediaDevices.removeEventListener("devicechange", refreshDevices);
|
|
};
|
|
}, [refreshDevices]);
|
|
|
|
const releaseResources = useCallback(() => {
|
|
const ws = wsRef.current;
|
|
wsRef.current = null;
|
|
if (ws) {
|
|
ws.onclose = null;
|
|
ws.onerror = null;
|
|
ws.onmessage = null;
|
|
ws.close();
|
|
}
|
|
|
|
const channel = dataChannelRef.current;
|
|
dataChannelRef.current = null;
|
|
if (channel) {
|
|
channel.onopen = null;
|
|
channel.onmessage = null;
|
|
channel.close();
|
|
}
|
|
|
|
const pc = pcRef.current;
|
|
pcRef.current = null;
|
|
if (pc) {
|
|
pc.onconnectionstatechange = null;
|
|
pc.onicecandidate = null;
|
|
pc.oniceconnectionstatechange = null;
|
|
pc.ontrack = null;
|
|
pc.close();
|
|
}
|
|
|
|
localStreamRef.current?.getTracks().forEach((track) => track.stop());
|
|
localStreamRef.current = null;
|
|
if (audioRef.current) audioRef.current.srcObject = null;
|
|
startingRef.current = false;
|
|
onNodeActiveRef.current?.(null);
|
|
}, []);
|
|
|
|
const disconnect = useCallback(() => {
|
|
releaseResources();
|
|
setLocalStream(null);
|
|
setRemoteStream(null);
|
|
setError(null);
|
|
setMicWarning(null);
|
|
setStatus("idle");
|
|
}, [releaseResources]);
|
|
|
|
const fail = useCallback(
|
|
(message: string) => {
|
|
releaseResources();
|
|
setLocalStream(null);
|
|
setRemoteStream(null);
|
|
setError(message);
|
|
setStatus("failed");
|
|
},
|
|
[releaseResources],
|
|
);
|
|
|
|
// 连接断开时:若是后端主动收尾(call-ended),按正常结束处理(不报错);
|
|
// 否则按异常失败处理。
|
|
const closeOnRemoteEnd = useCallback(
|
|
(message: string) => {
|
|
if (endedByServerRef.current) {
|
|
disconnect();
|
|
} else {
|
|
fail(message);
|
|
}
|
|
},
|
|
[disconnect, fail],
|
|
);
|
|
|
|
const connect = useCallback(async () => {
|
|
if (startingRef.current || pcRef.current || wsRef.current) return;
|
|
if (!assistantId) {
|
|
setError("请先保存助手，再开始语音预览。");
|
|
setStatus("failed");
|
|
return;
|
|
}
|
|
startingRef.current = true;
|
|
setError(null);
|
|
setMicWarning(null);
|
|
setMessages([]); // 新会话清空上一轮聊天记录
|
|
endedByServerRef.current = false;
|
|
setStatus("connecting");
|
|
|
|
// 麦克风是可选的:获取失败时继续建立仅接收后端音频的 WebRTC 会话。
|
|
let stream: MediaStream | null = null;
|
|
if (window.isSecureContext && navigator.mediaDevices?.getUserMedia) {
|
|
try {
|
|
const audioConstraints: MediaTrackConstraints = {
|
|
echoCancellation: true,
|
|
noiseSuppression: true,
|
|
autoGainControl: true,
|
|
};
|
|
// 指定了具体设备就强制用它,否则交给浏览器选默认麦克风
|
|
if (selectedDeviceIdRef.current) {
|
|
audioConstraints.deviceId = { exact: selectedDeviceIdRef.current };
|
|
}
|
|
stream = await navigator.mediaDevices.getUserMedia({
|
|
audio: audioConstraints,
|
|
});
|
|
// 授权后 label 才可见,刷新列表把真实设备名补上
|
|
void refreshDevices();
|
|
} catch (mediaError) {
|
|
setMicWarning(microphoneErrorMessage(mediaError));
|
|
}
|
|
} else {
|
|
setMicWarning("当前页面无法访问麦克风，已进入仅收听模式。");
|
|
}
|
|
if (stream) {
|
|
localStreamRef.current = stream;
|
|
setLocalStream(stream);
|
|
}
|
|
|
|
const pcId = generatePcId();
|
|
const ws = new WebSocket(`${wsBaseUrl()}/ws/voice`);
|
|
wsRef.current = ws;
|
|
|
|
ws.onmessage = async (event) => {
|
|
try {
|
|
const msg = JSON.parse(event.data);
|
|
if (msg.type === "answer") {
|
|
await pcRef.current?.setRemoteDescription({
|
|
type: "answer",
|
|
sdp: msg.payload.sdp,
|
|
});
|
|
} else if (msg.type === "ice-candidate" && msg.payload?.candidate) {
|
|
// 后端当前不主动 trickle,留兼容
|
|
try {
|
|
await pcRef.current?.addIceCandidate(msg.payload.candidate);
|
|
} catch {
|
|
/* 忽略迟到/重复 candidate */
|
|
}
|
|
} else if (msg.type === "error") {
|
|
fail(msg.payload?.message || "后端无法启动语音会话。");
|
|
}
|
|
} catch {
|
|
/* 非 JSON / 未知消息,忽略 */
|
|
}
|
|
};
|
|
|
|
try {
|
|
// 1) 等 ws 连上
|
|
await new Promise<void>((resolve, reject) => {
|
|
ws.onopen = () => resolve();
|
|
ws.onerror = (e) => reject(e);
|
|
ws.onclose = () => reject(new Error("语音信令连接已关闭。"));
|
|
});
|
|
// 连上后,信令异常或关闭都结束当前会话并保留失败状态。
|
|
ws.onerror = () => {
|
|
if (wsRef.current === ws) fail("语音信令连接失败。");
|
|
};
|
|
ws.onclose = () => {
|
|
if (wsRef.current === ws) closeOnRemoteEnd("语音信令连接已断开。");
|
|
};
|
|
|
|
// 2) 建 PeerConnection(纯 STUN,本机/局域网够用)
|
|
const pc = new RTCPeerConnection({
|
|
iceServers: [{ urls: "stun:stun.l.google.com:19302" }],
|
|
});
|
|
pcRef.current = pc;
|
|
|
|
pc.onicecandidate = (e) => {
|
|
if (ws.readyState !== WebSocket.OPEN) return;
|
|
ws.send(
|
|
JSON.stringify({
|
|
type: "ice-candidate",
|
|
payload: {
|
|
pc_id: pcId,
|
|
candidate: e.candidate
|
|
? {
|
|
candidate: e.candidate.candidate,
|
|
sdpMid: e.candidate.sdpMid,
|
|
sdpMLineIndex: e.candidate.sdpMLineIndex,
|
|
}
|
|
: null,
|
|
},
|
|
}),
|
|
);
|
|
};
|
|
|
|
// 应用消息通道:收后端转写(聊天记录),发文字输入。
|
|
// 由浏览器侧主动创建,后端 SmallWebRTCConnection 的 on("datachannel") 会接住。
|
|
const channel = pc.createDataChannel("chat");
|
|
dataChannelRef.current = channel;
|
|
channel.onopen = () => {
|
|
channel.send(JSON.stringify({ type: "client-ready" }));
|
|
};
|
|
channel.onmessage = (event) => {
|
|
try {
|
|
const msg = JSON.parse(event.data);
|
|
if (
|
|
msg?.type === "assistant-text-start" &&
|
|
typeof msg.turn_id === "string"
|
|
) {
|
|
messageSeqRef.current += 1;
|
|
const next: ChatMessage = {
|
|
id: `assistant-${msg.turn_id}`,
|
|
role: "assistant",
|
|
content: "",
|
|
timestamp:
|
|
typeof msg.timestamp === "string"
|
|
? msg.timestamp
|
|
: new Date().toISOString(),
|
|
sequence: messageSeqRef.current,
|
|
turnId: msg.turn_id,
|
|
streaming: true,
|
|
};
|
|
setMessages((prev) => sortMessages([...prev, next]));
|
|
} else if (
|
|
msg?.type === "assistant-text-delta" &&
|
|
typeof msg.turn_id === "string" &&
|
|
typeof msg.delta === "string"
|
|
) {
|
|
setMessages((prev) =>
|
|
prev.map((message) =>
|
|
message.turnId === msg.turn_id
|
|
? { ...message, content: message.content + msg.delta }
|
|
: message,
|
|
),
|
|
);
|
|
} else if (
|
|
msg?.type === "assistant-text-end" &&
|
|
typeof msg.turn_id === "string"
|
|
) {
|
|
setMessages((prev) =>
|
|
prev.map((message) =>
|
|
message.turnId === msg.turn_id
|
|
? {
|
|
...message,
|
|
content:
|
|
typeof msg.content === "string"
|
|
? msg.content
|
|
: message.content,
|
|
streaming: false,
|
|
}
|
|
: message,
|
|
),
|
|
);
|
|
} else if (
|
|
msg?.type === "transcript" &&
|
|
(msg.role === "user" || msg.role === "assistant") &&
|
|
typeof msg.content === "string" &&
|
|
msg.content.trim()
|
|
) {
|
|
messageSeqRef.current += 1;
|
|
const next: ChatMessage = {
|
|
id: `msg-${messageSeqRef.current}`,
|
|
role: msg.role,
|
|
content: msg.content,
|
|
timestamp:
|
|
typeof msg.timestamp === "string"
|
|
? msg.timestamp
|
|
: new Date().toISOString(),
|
|
sequence: messageSeqRef.current,
|
|
};
|
|
setMessages((prev) => sortMessages([...prev, next]));
|
|
} else if (
|
|
msg?.type === "node-active" &&
|
|
typeof msg.nodeId === "string"
|
|
) {
|
|
// 工作流:后端报告当前激活节点,交给画布高亮
|
|
onNodeActiveRef.current?.(msg.nodeId);
|
|
} else if (msg?.type === "call-ended") {
|
|
// 后端走到结束节点、正常收尾:随后的断开按正常结束处理,不报错
|
|
endedByServerRef.current = true;
|
|
onNodeActiveRef.current?.(null);
|
|
}
|
|
} catch {
|
|
/* 非 JSON / 未知消息,忽略 */
|
|
}
|
|
};
|
|
|
|
pc.ontrack = (e) => {
|
|
if (e.track.kind !== "audio") return;
|
|
const remote = e.streams[0] ?? new MediaStream([e.track]);
|
|
setRemoteStream(remote);
|
|
if (audioRef.current) {
|
|
audioRef.current.srcObject = remote;
|
|
void audioRef.current.play().catch(() => {});
|
|
}
|
|
};
|
|
|
|
pc.onconnectionstatechange = () => {
|
|
if (pcRef.current !== pc) return;
|
|
if (pc.connectionState === "connected") setStatus("connected");
|
|
else if (pc.connectionState === "failed")
|
|
closeOnRemoteEnd("WebRTC 音频连接失败。");
|
|
else if (pc.connectionState === "closed")
|
|
closeOnRemoteEnd("WebRTC 音频连接已断开。");
|
|
};
|
|
|
|
pc.oniceconnectionstatechange = () => {
|
|
if (pcRef.current !== pc) return;
|
|
const st = pc.iceConnectionState;
|
|
if (st === "connected" || st === "completed") setStatus("connected");
|
|
else if (st === "failed") closeOnRemoteEnd("WebRTC 音频连接失败。");
|
|
else if (st === "disconnected")
|
|
closeOnRemoteEnd("WebRTC 音频连接已断开。");
|
|
};
|
|
|
|
// 3) 有麦克风时双向音频；否则明确声明只接收后端音频。
|
|
if (stream) {
|
|
stream.getTracks().forEach((track) => pc.addTrack(track, stream));
|
|
} else {
|
|
pc.addTransceiver("audio", { direction: "recvonly" });
|
|
}
|
|
|
|
// 4) 生成 offer 并发给后端(assistant_id 在 payload 顶层)
|
|
const offer = await pc.createOffer();
|
|
await pc.setLocalDescription(offer);
|
|
const localDescription = pc.localDescription;
|
|
if (!localDescription?.sdp) {
|
|
throw new Error("浏览器无法创建 WebRTC offer。");
|
|
}
|
|
ws.send(
|
|
JSON.stringify({
|
|
type: "offer",
|
|
payload: {
|
|
pc_id: pcId,
|
|
sdp: localDescription.sdp,
|
|
type: localDescription.type,
|
|
assistant_id: assistantId,
|
|
},
|
|
}),
|
|
);
|
|
} catch (connectionError) {
|
|
fail(errorMessage(connectionError, "无法连接语音服务。"));
|
|
} finally {
|
|
startingRef.current = false;
|
|
}
|
|
}, [assistantId, fail, closeOnRemoteEnd, refreshDevices]);
|
|
|
|
// 选择麦克风:更新选择;若会话正在发送麦克风音频,则用 WebRTC replaceTrack
|
|
// 热切换轨道(无需重新协商),并把波形可视化重新接到新流。
|
|
// 未连接时仅记下选择,留待下次 connect 生效。
|
|
const selectDevice = useCallback(
|
|
async (deviceId: string) => {
|
|
setSelectedDeviceId(deviceId);
|
|
selectedDeviceIdRef.current = deviceId;
|
|
|
|
const pc = pcRef.current;
|
|
if (!pc) return;
|
|
// 只有本就在发送麦克风音频(存在 audio sender 轨道)时才热切换;
|
|
// 仅收听模式下加麦克风需重新协商,这里不处理,留到下次连接。
|
|
const sender = pc
|
|
.getSenders()
|
|
.find((s) => s.track?.kind === "audio");
|
|
if (!sender) return;
|
|
|
|
try {
|
|
const audioConstraints: MediaTrackConstraints = {
|
|
echoCancellation: true,
|
|
noiseSuppression: true,
|
|
autoGainControl: true,
|
|
};
|
|
if (deviceId) audioConstraints.deviceId = { exact: deviceId };
|
|
const newStream = await navigator.mediaDevices.getUserMedia({
|
|
audio: audioConstraints,
|
|
});
|
|
// 切换期间可能已断开,丢弃刚拿到的流
|
|
if (pcRef.current !== pc) {
|
|
newStream.getTracks().forEach((t) => t.stop());
|
|
return;
|
|
}
|
|
const newTrack = newStream.getAudioTracks()[0];
|
|
if (!newTrack) {
|
|
newStream.getTracks().forEach((t) => t.stop());
|
|
return;
|
|
}
|
|
await sender.replaceTrack(newTrack);
|
|
// 旧轨道停掉,新流替换(波形/分析器随 localStream 变化自动重连)
|
|
localStreamRef.current?.getTracks().forEach((t) => t.stop());
|
|
localStreamRef.current = newStream;
|
|
setLocalStream(newStream);
|
|
setMicWarning(null);
|
|
} catch (mediaError) {
|
|
setMicWarning(microphoneErrorMessage(mediaError));
|
|
}
|
|
},
|
|
[],
|
|
);
|
|
|
|
// 发送文字消息:后端先打断当前播报,再按用户输入触发新回复。
|
|
// 成功返回 true;通道未就绪(未开始对话/连接中)返回 false。
|
|
const sendText = useCallback((text: string): boolean => {
|
|
const trimmed = text.trim();
|
|
const channel = dataChannelRef.current;
|
|
if (!trimmed || !channel || channel.readyState !== "open") return false;
|
|
channel.send(JSON.stringify({ type: "user-text", text: trimmed }));
|
|
return true;
|
|
}, []);
|
|
|
|
// 卸载时收尾
|
|
useEffect(() => releaseResources, [releaseResources]);
|
|
|
|
return {
|
|
status,
|
|
error,
|
|
micWarning,
|
|
localStream,
|
|
remoteStream,
|
|
messages,
|
|
audioInputs,
|
|
selectedDeviceId,
|
|
setSelectedDeviceId,
|
|
selectDevice,
|
|
sendText,
|
|
connect,
|
|
disconnect,
|
|
audioRef,
|
|
};
|
|
}
|