"use client"; /** * 语音预览:把麦克风接到后端 /ws/voice(WebRTC 信令),听到助手实时回应。 * * 走原生 RTCPeerConnection + 一条 ws 信令通道,与后端 voice_webrtc.py 的约定对齐: * client → {type:"offer", payload:{pc_id, sdp, type, assistant_id}} * server → {type:"answer", payload:{pc_id, sdp, type}} * client → {type:"ice-candidate", payload:{pc_id, candidate:{...}}} * 音频本身走 WebRTC 媒体流(Opus),不经 ws;后端 TTS 帧从 ontrack 拿到直接播放。 * * 另开一条 data channel 与后端管线(pipeline.py)互通应用消息: * client → {type:"user-text", text} 文字输入(打断并触发新回复) * server → {type:"transcript", role, content, timestamp} 用户/助手最终转写(聊天记录) * * 纯本机(localhost)即可跑:localhost 是 secure context,麦克风可用,ws 用明文。 * 局域网/别的设备要 https+wss,见 deploy/README.md。 */ import { useCallback, useEffect, useRef, useState } from "react"; import { API_BASE } from "@/lib/api"; export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed"; export type ChatMessage = { id: string; role: "user" | "assistant"; content: string; /** 后端给的 ISO 时间戳 */ timestamp: string; sequence: number; turnId?: string; streaming?: boolean; }; // http→ws、https→wss,自动跟随 API 基址(同源反代时也对) function wsBaseUrl(): string { const url = new URL(API_BASE, window.location.origin); url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; return url.toString().replace(/\/$/, ""); } function generatePcId(): string { const bytes = new Uint8Array(16); crypto.getRandomValues(bytes); return ( "PC-" + Array.from(bytes) .map((b) => b.toString(16).padStart(2, "0")) .join("") ); } function errorMessage(error: unknown, fallback: string): string { if (error instanceof Error && error.message) return error.message; return fallback; } function messageOrder(message: ChatMessage): number { const timestamp = Date.parse(message.timestamp); return Number.isNaN(timestamp) ? Number.MAX_SAFE_INTEGER : timestamp; } function sortMessages(messages: ChatMessage[]): ChatMessage[] { return messages.sort( (a, b) => messageOrder(a) - messageOrder(b) || a.sequence - b.sequence, ); } function microphoneErrorMessage(error: unknown): string { if (error instanceof DOMException) { if (error.name === "NotAllowedError") { return "麦克风权限被拒绝,请在浏览器网站设置中允许 localhost 使用麦克风。"; } if (error.name === "NotFoundError") { return "未检测到可用麦克风,请连接或启用麦克风后重试。"; } if (error.name === "NotReadableError") { return "麦克风正被其他应用占用,或系统未允许浏览器访问麦克风。"; } } return errorMessage(error, "无法访问麦克风。"); } export function useVoicePreview( assistantId: string | null, onNodeActive?: (nodeId: string | null) => void, ) { const [status, setStatus] = useState("idle"); const [error, setError] = useState(null); const [micWarning, setMicWarning] = useState(null); const [localStream, setLocalStream] = useState(null); // 远端(助手 TTS)媒体流:除挂到