wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix frontend voice preview fallback

Browse Source
This commit is contained in:
Xin Wang
2026-06-10 12:36:18 +08:00
parent 4a948ee609
commit e94d98e947
3 changed files with 72 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/.env.example
View File

@@ -32,4 +32,4 @@ DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/postgres
HOST=0.0.0.0
PORT=8000
# 前端开发地址,允许跨域
CORS_ORIGINS=http://localhost:3000
CORS_ORIGINS=http://localhost:3000,http://127.0.0.1:3000

28
frontend/src/components/pages/AssistantPage.tsx
View File

@@ -1790,9 +1790,15 @@ function DebugVoicePanel({
vizStyle: VizStyle;
assistantId: string | null;
}) {
const [micError, setMicError] = useState(false);
const { status, error, localStream, connect, disconnect, audioRef } =
useVoicePreview(assistantId, { onMicError: () => setMicError(true) });
const {
status,
error,
micWarning,
localStream,
connect,
disconnect,
audioRef,
} = useVoicePreview(assistantId);
// 连接中或已连通都视作"会话进行中"
const recording = status === "connecting" || status === "connected";
@@ -1827,15 +1833,10 @@ function DebugVoicePanel({
<div className="relative flex h-[200px] w-[240px] shrink-0 items-center justify-center">
{(() => {
const onVizError = () => {
setMicError(true);
disconnect();
};
const shared = {
active: Boolean(localStream),
stream: localStream,
className: "relative shrink-0",
onError: onVizError,
} as const;
if (vizStyle === "aura")
return <AuraVisualizer {...shared} size={200} />;
@@ -1852,17 +1853,19 @@ function DebugVoicePanel({
{status === "connecting"
? "连接中…"
: status === "connected"
? "我在聆听"
? micWarning
? "仅收听模式"
: "我在聆听"
: "开始一次语音对话"}
</div>
<p className="mx-auto text-xs leading-5 text-muted-foreground">
{micError
? "无法访问麦克风，请检查浏览器权限后重试。"
: status === "failed"
{status === "failed"
? error ||
"连接失败，请确认后端已启动且助手已保存后重试。"
: !assistantId
? "请先保存助手，再开始语音预览。"
: micWarning
? `${micWarning} 可接收助手播报，但无法发送语音。`
: recording
? "直接说话即可。助手会在您停顿后自然回应。"
: "测试语音识别、响应速度与助手的播报效果。"}
@@ -1872,7 +1875,6 @@ function DebugVoicePanel({
<Button
disabled={!assistantId || status === "connecting"}
onClick={() => {
setMicError(false);
if (recording) {
disconnect();
} else {

85
frontend/src/hooks/use-voice-preview.ts
View File

@@ -37,22 +37,30 @@ function generatePcId(): string {
);
}
type UseVoicePreviewOptions = {
/** 取麦克风失败(权限/无设备)时回调,供 UI 提示。 */
onMicError?: () => void;
};
function errorMessage(error: unknown, fallback: string): string {
if (error instanceof Error && error.message) return error.message;
return fallback;
}
export function useVoicePreview(
assistantId: string | null,
{ onMicError }: UseVoicePreviewOptions = {},
) {
function microphoneErrorMessage(error: unknown): string {
if (error instanceof DOMException) {
if (error.name === "NotAllowedError") {
return "麦克风权限被拒绝，请在浏览器网站设置中允许 localhost 使用麦克风。";
}
if (error.name === "NotFoundError") {
return "未检测到可用麦克风，请连接或启用麦克风后重试。";
}
if (error.name === "NotReadableError") {
return "麦克风正被其他应用占用，或系统未允许浏览器访问麦克风。";
}
}
return errorMessage(error, "无法访问麦克风。");
}
export function useVoicePreview(assistantId: string | null) {
const [status, setStatus] = useState<VoicePreviewStatus>("idle");
const [error, setError] = useState<string | null>(null);
const [micWarning, setMicWarning] = useState<string | null>(null);
const [localStream, setLocalStream] = useState<MediaStream | null>(null);
const audioRef = useRef<HTMLAudioElement | null>(null);
const pcRef = useRef<RTCPeerConnection | null>(null);
@@ -90,6 +98,7 @@ export function useVoicePreview(
releaseResources();
setLocalStream(null);
setError(null);
setMicWarning(null);
setStatus("idle");
}, [releaseResources]);
@@ -110,11 +119,33 @@ export function useVoicePreview(
setStatus("failed");
return;
}
startingRef.current = true;
setError(null);
setMicWarning(null);
setStatus("connecting");
// 麦克风是可选的:获取失败时继续建立仅接收后端音频的 WebRTC 会话。
let stream: MediaStream | null = null;
if (window.isSecureContext && navigator.mediaDevices?.getUserMedia) {
try {
stream = await navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
});
} catch (mediaError) {
setMicWarning(microphoneErrorMessage(mediaError));
}
} else {
setMicWarning("当前页面无法访问麦克风，已进入仅收听模式。");
}
if (stream) {
localStreamRef.current = stream;
setLocalStream(stream);
}
const pcId = generatePcId();
const ws = new WebSocket(`${wsBaseUrl()}/ws/voice`);
wsRef.current = ws;
@@ -205,24 +236,12 @@ export function useVoicePreview(
else if (st === "disconnected") fail("WebRTC 音频连接已断开。");
};
// 3) 取麦克风 → 加入连接
let stream: MediaStream;
try {
stream = await navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
});
} catch (mediaError) {
onMicError?.();
fail(errorMessage(mediaError, "无法访问麦克风。"));
return;
// 3) 有麦克风时双向音频；否则明确声明只接收后端音频。
if (stream) {
stream.getTracks().forEach((track) => pc.addTrack(track, stream));
} else {
pc.addTransceiver("audio", { direction: "recvonly" });
}
localStreamRef.current = stream;
setLocalStream(stream);
stream.getTracks().forEach((track) => pc.addTrack(track, stream));
// 4) 生成 offer 并发给后端(assistant_id 在 payload 顶层)
const offer = await pc.createOffer();
@@ -247,10 +266,18 @@ export function useVoicePreview(
} finally {
startingRef.current = false;
}
}, [assistantId, fail, onMicError]);
}, [assistantId, fail]);
// 卸载时收尾
useEffect(() => releaseResources, [releaseResources]);
return { status, error, localStream, connect, disconnect, audioRef };
return {
status,
error,
micWarning,
localStream,
connect,
disconnect,
audioRef,
};
}