"use client"; /** * 语音预览。 * * WebRTC 生命周期交给 Pipecat 官方 SmallWebRTCTransport:它负责固定的 * audio/video transceiver、trackStatus、keepalive、重协商和 ICE 恢复。 * 本文件只把平台已有的业务消息映射到 React 状态。 */ import { useCallback, useEffect, useRef, useState } from "react"; import Daily from "@daily-co/daily-js"; import type { PipecatClientOptions, RTVIMessage } from "@pipecat-ai/client-js"; import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport"; import { API_BASE, webrtcApi } from "@/lib/api"; export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed"; export type NetworkQuality = "unknown" | "good" | "fair" | "poor"; const NETWORK_SAMPLE_INTERVAL_MS = 2_000; const SESSION_UPDATE_TIMEOUT_MS = 10_000; type NetworkStatsSample = { packetsSent: number; packetsLost: number; }; type NetworkMetrics = NetworkStatsSample & { roundTripTime: number | null; availableOutgoingBitrate: number | null; }; type ConnectOptions = { visionEnabled?: boolean; videoStream?: MediaStream | null; dynamicVariables?: Record; }; export type ChatMessage = { id: string; role: "user" | "assistant"; content: string; timestamp: string; sequence: number; turnId?: string; streaming?: boolean; }; type AppMessage = Record & { type?: string }; export type DynamicVariableValue = string | number | boolean; export type UserInputPart = | { type: "input_text"; text: string } | { type: "input_image"; source: { type: "camera_frame"; frame: "current" }; }; export type UserInputResult = { inputId: string; status: "accepted"; }; export type SessionUpdateRequest = { dynamicVariables: Record; }; export type SessionUpdateResult = { updateId: string; status: "accepted"; changed: string[]; dynamicVariables: Record; }; export type ClientToolHandler = ( argumentsValue: Record, ) => unknown | Promise; export type ClientToolDefinition = { name: string; label: string; description: string; parameters: readonly ClientToolParameter[]; }; export type ClientToolParameter = { name: string; type: string; required: boolean; description: string; }; type RegisteredClientTool = { definition: ClientToolDefinition; handler: ClientToolHandler; }; type PendingUserInput = { resolve: (result: UserInputResult) => void; reject: (error: Error) => void; timeout: number; }; type PendingSessionUpdate = { resolve: (result: SessionUpdateResult) => void; reject: (error: Error) => void; timeout: number; }; function publicVariableSnapshot( value: unknown, ): Record { if (!value || typeof value !== "object" || Array.isArray(value)) return {}; return Object.fromEntries( Object.entries(value).filter(([name, item]) => !name.startsWith("system__") && !name.startsWith("secret__") && (typeof item === "string" || typeof item === "number" || typeof item === "boolean"), ) as [string, DynamicVariableValue][], ); } class AppSmallWebRTCTransport extends SmallWebRTCTransport { onAppMessage?: (message: AppMessage) => void; /** * SmallWebRTCTransport 的媒体管理器会复用 Daily 的全局 call object, * 但 disconnect() 不会移除构造时注册的设备和 track 监听器。销毁 call * object 后,下一次连接会得到干净的实例,避免监听器跨会话累积。 */ async dispose(): Promise { this.onAppMessage = undefined; try { await this.disconnect(); } finally { const call = Daily.getCallInstance(); if (call && !call.isDestroyed()) await call.destroy(); } } override handleMessage(raw: string): void { try { const message = JSON.parse(raw) as AppMessage; if (message.type === "signalling" || message.label === "rtvi-ai") { super.handleMessage(raw); } else { this.onAppMessage?.(message); } } catch { super.handleMessage(raw); } } sendAppMessage(message: AppMessage): void { super.sendMessage(message as unknown as RTVIMessage); } async connectionStats(): Promise { const peerConnection = ( this as unknown as { pc: RTCPeerConnection | null } ).pc; return peerConnection ? peerConnection.getStats() : null; } } function errorMessage(error: unknown, fallback: string): string { if (error instanceof Error && error.message) return error.message; return fallback; } function messageOrder(message: ChatMessage): number { const timestamp = Date.parse(message.timestamp); return Number.isNaN(timestamp) ? Number.MAX_SAFE_INTEGER : timestamp; } function sortMessages(messages: ChatMessage[]): ChatMessage[] { return messages.sort( (a, b) => messageOrder(a) - messageOrder(b) || a.sequence - b.sequence, ); } function encodeHeaderJson(value: unknown): string { const bytes = new TextEncoder().encode(JSON.stringify(value)); let binary = ""; bytes.forEach((byte) => { binary += String.fromCharCode(byte); }); return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); } function newInputId(): string { if (typeof crypto !== "undefined" && "randomUUID" in crypto) { return `input_${crypto.randomUUID()}`; } return `input_${Date.now()}_${Math.random().toString(16).slice(2)}`; } function newUpdateId(): string { if (typeof crypto !== "undefined" && "randomUUID" in crypto) { return `update_${crypto.randomUUID()}`; } return `update_${Date.now()}_${Math.random().toString(16).slice(2)}`; } function readNetworkMetrics(report: RTCStatsReport): NetworkMetrics | null { const outbound: Partial> = {}; const lost: Partial> = {}; let roundTripTime: number | null = null; let availableOutgoingBitrate: number | null = null; report.forEach((stat) => { const value = stat as RTCStats & Record; const kind = value.kind ?? value.mediaType; if ( value.type === "outbound-rtp" && (kind === "audio" || kind === "video") && typeof value.packetsSent === "number" ) { outbound[kind] = value.packetsSent; } else if ( value.type === "remote-inbound-rtp" && (kind === "audio" || kind === "video") ) { if (typeof value.packetsLost === "number") lost[kind] = value.packetsLost; if (typeof value.roundTripTime === "number") { roundTripTime = value.roundTripTime; } } else if ( value.type === "candidate-pair" && value.state === "succeeded" && (value.nominated === true || value.selected === true) ) { if (typeof value.currentRoundTripTime === "number") { roundTripTime = value.currentRoundTripTime; } if (typeof value.availableOutgoingBitrate === "number") { availableOutgoingBitrate = value.availableOutgoingBitrate; } } }); const kind = outbound.video !== undefined ? "video" : "audio"; if (outbound[kind] === undefined && roundTripTime === null) return null; return { packetsSent: outbound[kind] ?? 0, packetsLost: lost[kind] ?? 0, roundTripTime, availableOutgoingBitrate, }; } function classifyNetworkQuality( metrics: NetworkMetrics, previous: NetworkStatsSample | null, ): NetworkQuality { const sentDelta = previous ? Math.max(0, metrics.packetsSent - previous.packetsSent) : 0; const lostDelta = previous ? Math.max(0, metrics.packetsLost - previous.packetsLost) : 0; const lossRate = sentDelta > 0 ? lostDelta / sentDelta : null; const { roundTripTime: rtt, availableOutgoingBitrate: bandwidth } = metrics; if (lossRate === null && rtt === null && bandwidth === null) return "unknown"; if ( (lossRate !== null && lossRate >= 0.08) || (rtt !== null && rtt >= 0.45) || (bandwidth !== null && bandwidth < 350_000) ) return "poor"; if ( (lossRate !== null && lossRate >= 0.03) || (rtt !== null && rtt >= 0.25) || (bandwidth !== null && bandwidth < 550_000) ) return "fair"; return "good"; } export function useVoicePreview( assistantId: string | null, onNodeActive?: (nodeId: string | null) => void, ) { const [status, setStatus] = useState("idle"); const [error, setError] = useState(null); const [micWarning, setMicWarning] = useState(null); const [localStream, setLocalStream] = useState(null); const [videoStream, setVideoStream] = useState(null); const [remoteStream, setRemoteStream] = useState(null); const [messages, setMessages] = useState([]); const [sessionVariables, setSessionVariables] = useState< Record >({}); const [callEnded, setCallEnded] = useState(false); const [networkQuality, setNetworkQuality] = useState("unknown"); const [audioInputs, setAudioInputs] = useState([]); const [audioOutputs, setAudioOutputs] = useState([]); const [selectedDeviceId, setSelectedDeviceId] = useState(""); const [selectedOutputDeviceId, setSelectedOutputDeviceId] = useState(""); const [clientTools, setClientTools] = useState([]); const transportRef = useRef(null); const cleanupPromiseRef = useRef>(Promise.resolve()); const connectionGenerationRef = useRef(0); const startingRef = useRef(false); const messageSeqRef = useRef(0); const networkStatsRef = useRef(null); const pendingAssistantTurnsRef = useRef( new Map(), ); const pendingUserInputsRef = useRef(new Map()); const pendingSessionUpdatesRef = useRef( new Map(), ); const clientToolHandlersRef = useRef( new Map(), ); const endedByServerRef = useRef(false); const selectedDeviceIdRef = useRef(""); const selectedOutputDeviceIdRef = useRef(""); const onNodeActiveRef = useRef(onNodeActive); const audioRef = useRef(null); useEffect(() => { onNodeActiveRef.current = onNodeActive; }, [onNodeActive]); const supportsOutputSelection = typeof HTMLAudioElement !== "undefined" && "setSinkId" in HTMLAudioElement.prototype; const applyOutputDevice = useCallback(async (deviceId: string) => { const audio = audioRef.current; if (!audio || !supportsOutputSelection) return; try { await audio.setSinkId(deviceId); } catch { // 设备可能已拔出,交给浏览器回退默认输出。 } }, [supportsOutputSelection]); const refreshDevices = useCallback(async () => { if (!navigator.mediaDevices?.enumerateDevices) return; try { const devices = await navigator.mediaDevices.enumerateDevices(); setAudioInputs( devices.filter((item) => item.kind === "audioinput" && item.deviceId), ); setAudioOutputs( devices.filter((item) => item.kind === "audiooutput" && item.deviceId), ); } catch { // 设备枚举失败不影响仅接收模式。 } }, []); useEffect(() => { // eslint-disable-next-line react-hooks/set-state-in-effect void refreshDevices(); navigator.mediaDevices?.addEventListener("devicechange", refreshDevices); return () => navigator.mediaDevices?.removeEventListener("devicechange", refreshDevices); }, [refreshDevices]); const releaseResources = useCallback(() => { connectionGenerationRef.current += 1; const transport = transportRef.current; transportRef.current = null; if (transport) { cleanupPromiseRef.current = cleanupPromiseRef.current .then(() => transport.dispose()) .catch(() => {}); } if (audioRef.current) audioRef.current.srcObject = null; startingRef.current = false; pendingAssistantTurnsRef.current.clear(); pendingUserInputsRef.current.forEach(({ reject, timeout }) => { window.clearTimeout(timeout); reject(new Error("连接已关闭,用户输入未完成")); }); pendingUserInputsRef.current.clear(); pendingSessionUpdatesRef.current.forEach(({ reject, timeout }) => { window.clearTimeout(timeout); reject(new Error("连接已关闭,会话状态更新未完成")); }); pendingSessionUpdatesRef.current.clear(); networkStatsRef.current = null; onNodeActiveRef.current?.(null); }, []); const disconnect = useCallback(() => { releaseResources(); setLocalStream(null); setVideoStream(null); setRemoteStream(null); setMessages([]); messageSeqRef.current = 0; pendingAssistantTurnsRef.current.clear(); setError(null); setMicWarning(null); setNetworkQuality("unknown"); setStatus("idle"); }, [releaseResources]); const fail = useCallback((message: string) => { releaseResources(); setLocalStream(null); setVideoStream(null); setRemoteStream(null); setError(message); setNetworkQuality("unknown"); setStatus("failed"); }, [releaseResources]); const dispatchClientTool = useCallback(async (msg: AppMessage) => { const toolCallId = typeof msg.tool_call_id === "string" ? msg.tool_call_id : ""; const functionName = typeof msg.function_name === "string" ? msg.function_name : ""; const waitForResponse = msg.wait_for_response !== false; if (!toolCallId || !functionName) return; const registeredTool = clientToolHandlersRef.current.get(functionName); const transport = transportRef.current; if (!transport || transport.state !== "connected") return; if (!registeredTool) { if (waitForResponse) { transport.sendAppMessage({ type: "client-tool-result", tool_call_id: toolCallId, status: "error", message: `客户端未注册工具: ${functionName}`, }); } return; } try { const argumentsValue = msg.arguments && typeof msg.arguments === "object" && !Array.isArray(msg.arguments) ? (msg.arguments as Record) : {}; const data = await registeredTool.handler(argumentsValue); if (transportRef.current !== transport) return; if (!waitForResponse) return; transport.sendAppMessage({ type: "client-tool-result", tool_call_id: toolCallId, status: "ok", data, }); } catch (toolError) { if (transportRef.current !== transport) return; if (!waitForResponse) return; transport.sendAppMessage({ type: "client-tool-result", tool_call_id: toolCallId, status: "error", message: errorMessage(toolError, "客户端工具执行失败"), }); } }, []); const handleAppMessage = useCallback((msg: AppMessage) => { if (msg.type === "client-tool-call") { void dispatchClientTool(msg); } else if ( msg.type === "user-input-result" && typeof msg.input_id === "string" ) { const pending = pendingUserInputsRef.current.get(msg.input_id); if (!pending) return; pendingUserInputsRef.current.delete(msg.input_id); window.clearTimeout(pending.timeout); if (msg.status === "accepted") { pending.resolve({ inputId: msg.input_id, status: "accepted" }); } else { pending.reject( new Error( typeof msg.message === "string" ? msg.message : "用户输入处理失败", ), ); } } else if ( msg.type === "session-update-result" && typeof msg.update_id === "string" ) { const pending = pendingSessionUpdatesRef.current.get(msg.update_id); if (!pending) return; pendingSessionUpdatesRef.current.delete(msg.update_id); window.clearTimeout(pending.timeout); if (msg.status === "accepted") { const dynamicVariables = publicVariableSnapshot(msg.dynamic_variables); const changed = Array.isArray(msg.changed) ? msg.changed.filter((name): name is string => typeof name === "string") : []; setSessionVariables(dynamicVariables); pending.resolve({ updateId: msg.update_id, status: "accepted", changed, dynamicVariables, }); } else { pending.reject( new Error( typeof msg.message === "string" ? msg.message : "会话状态更新失败", ), ); } } else if ( msg.type === "assistant-text-start" && typeof msg.turn_id === "string" ) { pendingAssistantTurnsRef.current.set(msg.turn_id, { timestamp: typeof msg.timestamp === "string" ? msg.timestamp : new Date().toISOString(), }); } else if ( msg.type === "assistant-text-delta" && typeof msg.turn_id === "string" && typeof msg.delta === "string" && msg.delta.length > 0 ) { const pending = pendingAssistantTurnsRef.current.get(msg.turn_id); if (pending) { pendingAssistantTurnsRef.current.delete(msg.turn_id); messageSeqRef.current += 1; const sequence = messageSeqRef.current; setMessages((previous) => sortMessages([ ...previous, { id: `assistant-${msg.turn_id as string}`, role: "assistant", content: msg.delta as string, timestamp: pending.timestamp, sequence, turnId: msg.turn_id as string, streaming: true, }, ]), ); } else { setMessages((previous) => previous.map((message) => message.turnId === msg.turn_id ? { ...message, content: message.content + msg.delta } : message, ), ); } } else if ( msg.type === "assistant-text-end" && typeof msg.turn_id === "string" ) { const pending = pendingAssistantTurnsRef.current.get(msg.turn_id); pendingAssistantTurnsRef.current.delete(msg.turn_id); const finalContent = typeof msg.content === "string" ? msg.content.trim() : ""; setMessages((previous) => { const existing = previous.some((message) => message.turnId === msg.turn_id); if (existing) { return previous.map((message) => message.turnId === msg.turn_id ? { ...message, content: finalContent || message.content, streaming: false, } : message, ); } if (!finalContent) return previous; messageSeqRef.current += 1; return sortMessages([ ...previous, { id: `assistant-${msg.turn_id as string}`, role: "assistant", content: finalContent, timestamp: pending?.timestamp ?? new Date().toISOString(), sequence: messageSeqRef.current, turnId: msg.turn_id as string, streaming: false, }, ]); }); } else if ( msg.type === "transcript" && (msg.role === "user" || msg.role === "assistant") && typeof msg.content === "string" && msg.content.trim() ) { messageSeqRef.current += 1; setMessages((previous) => sortMessages([ ...previous, { id: `msg-${messageSeqRef.current}`, role: msg.role as "user" | "assistant", content: msg.content as string, timestamp: typeof msg.timestamp === "string" ? msg.timestamp : new Date().toISOString(), sequence: messageSeqRef.current, }, ]), ); } else if (msg.type === "node-active" && typeof msg.nodeId === "string") { onNodeActiveRef.current?.(msg.nodeId); } else if (msg.type === "workflow-variables") { setSessionVariables(publicVariableSnapshot(msg.variables)); } else if (msg.type === "call-ended") { endedByServerRef.current = true; setCallEnded(true); disconnect(); } }, [disconnect, dispatchClientTool]); const connect = useCallback(async (options: ConnectOptions = {}) => { if (startingRef.current || transportRef.current) return; if (!assistantId) { setError("请先保存助手,再开始语音预览。"); setStatus("failed"); return; } startingRef.current = true; const generation = connectionGenerationRef.current + 1; connectionGenerationRef.current = generation; setStatus("connecting"); setError(null); setMicWarning(null); setMessages([]); setSessionVariables(publicVariableSnapshot(options.dynamicVariables ?? {})); pendingAssistantTurnsRef.current.clear(); setCallEnded(false); endedByServerRef.current = false; await cleanupPromiseRef.current; if (connectionGenerationRef.current !== generation) return; const iceServers = await webrtcApi .iceServers() .then((response) => response.iceServers) .catch(() => [{ urls: "stun:stun.l.google.com:19302" }]); if (connectionGenerationRef.current !== generation) return; const transport = new AppSmallWebRTCTransport({ iceServers }); transportRef.current = transport; transport.onAppMessage = handleAppMessage; const callbacks: NonNullable = { onConnected: () => setStatus("connected"), onDisconnected: () => { if (transportRef.current !== transport) return; if (endedByServerRef.current) disconnect(); else fail("WebRTC 连接已断开。"); }, onTrackStarted: (track) => { if (track.kind !== "audio") return; const stream = new MediaStream([track]); setRemoteStream(stream); if (audioRef.current) { audioRef.current.srcObject = stream; void applyOutputDevice(selectedOutputDeviceIdRef.current).then(() => audioRef.current?.play().catch(() => {}), ); } }, onDeviceError: (deviceError) => { setMicWarning(deviceError.message || "无法访问麦克风。已尝试继续连接。"); }, onCamUpdated: () => { const track = transport.tracks().local.video; setVideoStream(track ? new MediaStream([track]) : null); }, onTransportStateChanged: (nextState) => { if (nextState === "connected" || nextState === "ready") { setStatus("connected"); } else if (nextState === "error") { fail("WebRTC 连接失败。"); } }, }; transport.initialize( { transport, enableMic: true, enableCam: Boolean(options.visionEnabled), callbacks, }, () => {}, ); try { await transport.initDevices(); if (connectionGenerationRef.current !== generation) return; if (selectedDeviceIdRef.current) { await transport.updateMic(selectedDeviceIdRef.current); } if (connectionGenerationRef.current !== generation) return; const localAudio = transport.tracks().local.audio; const localVideo = transport.tracks().local.video; setLocalStream(localAudio ? new MediaStream([localAudio]) : null); setVideoStream(localVideo ? new MediaStream([localVideo]) : null); void refreshDevices(); const request = new Request(`${API_BASE}/api/webrtc/offer`, { method: "POST", headers: { "Content-Type": "application/json", "X-Pipecat-Assistant-ID": assistantId, "X-Pipecat-Dynamic-Variables": encodeHeaderJson( options.dynamicVariables ?? {}, ), }, credentials: "include", }); await transport.connect({ webrtcRequestParams: { endpoint: request, requestData: { assistant_id: assistantId, vision_enabled: Boolean(options.visionEnabled), dynamic_variables: options.dynamicVariables ?? {}, }, }, }); if (connectionGenerationRef.current !== generation) return; transport.sendAppMessage({ type: "client-ready" }); setStatus("connected"); } catch (connectionError) { if (connectionGenerationRef.current === generation) { fail(errorMessage(connectionError, "无法连接语音服务。")); } } finally { if (connectionGenerationRef.current === generation) { startingRef.current = false; } } }, [ assistantId, applyOutputDevice, disconnect, fail, handleAppMessage, refreshDevices, ]); const replaceVideoStream = useCallback(async (videoStream: MediaStream | null) => { const transport = transportRef.current; const deviceId = videoStream?.getVideoTracks()[0]?.getSettings().deviceId; if (transport && deviceId) await transport.updateCam(deviceId); }, []); const selectCamera = useCallback((deviceId: string) => { transportRef.current?.updateCam(deviceId); }, []); const selectDevice = useCallback(async (deviceId: string) => { setSelectedDeviceId(deviceId); selectedDeviceIdRef.current = deviceId; const transport = transportRef.current; if (!transport) return; try { await transport.updateMic(deviceId); const audio = transport.tracks().local.audio; setLocalStream(audio ? new MediaStream([audio]) : null); setMicWarning(null); } catch (deviceError) { setMicWarning(errorMessage(deviceError, "无法切换麦克风。")); } }, []); const selectOutputDevice = useCallback((deviceId: string) => { setSelectedOutputDeviceId(deviceId); selectedOutputDeviceIdRef.current = deviceId; void applyOutputDevice(deviceId); }, [applyOutputDevice]); useEffect(() => { if (status !== "connected") return; let cancelled = false; const sample = async () => { const transport = transportRef.current; if (!transport) return; try { const report = await transport.connectionStats(); const metrics = report ? readNetworkMetrics(report) : null; if (cancelled || transportRef.current !== transport) return; if (!metrics) { setNetworkQuality("unknown"); return; } setNetworkQuality( classifyNetworkQuality(metrics, networkStatsRef.current), ); networkStatsRef.current = { packetsSent: metrics.packetsSent, packetsLost: metrics.packetsLost, }; } catch { if (!cancelled) setNetworkQuality("unknown"); } }; void sample(); const interval = window.setInterval(() => void sample(), NETWORK_SAMPLE_INTERVAL_MS); return () => { cancelled = true; window.clearInterval(interval); }; }, [status]); const sendText = useCallback((text: string): boolean => { const trimmed = text.trim(); const transport = transportRef.current; if (!trimmed || !transport || transport.state !== "connected") return false; const inputId = newInputId(); transport.sendAppMessage({ type: "user-input", schema_version: 1, input_id: inputId, parts: [{ type: "input_text", text: trimmed }], options: { run_immediately: true, interrupt: true }, }); return true; }, []); const sendUserInput = useCallback( ( parts: UserInputPart[], options: { runImmediately?: boolean; interrupt?: boolean } = {}, ): Promise => { const transport = transportRef.current; if (!transport || transport.state !== "connected") { return Promise.reject(new Error("当前未连接语音服务")); } const inputId = newInputId(); return new Promise((resolve, reject) => { const timeout = window.setTimeout(() => { pendingUserInputsRef.current.delete(inputId); reject(new Error("等待用户输入处理结果超时")); }, 30_000); pendingUserInputsRef.current.set(inputId, { resolve, reject, timeout }); try { transport.sendAppMessage({ type: "user-input", schema_version: 1, input_id: inputId, parts, options: { run_immediately: options.runImmediately ?? true, interrupt: options.interrupt ?? true, }, }); } catch (sendError) { window.clearTimeout(timeout); pendingUserInputsRef.current.delete(inputId); reject( new Error(errorMessage(sendError, "发送用户输入失败")), ); } }); }, [], ); const updateSession = useCallback( ({ dynamicVariables, }: SessionUpdateRequest): Promise => { if (Object.keys(dynamicVariables).length === 0) { return Promise.reject(new Error("至少需要更新一个动态变量")); } const transport = transportRef.current; if (!transport || transport.state !== "connected") { return Promise.reject(new Error("当前未连接语音服务")); } const updateId = newUpdateId(); return new Promise((resolve, reject) => { const timeout = window.setTimeout(() => { pendingSessionUpdatesRef.current.delete(updateId); reject(new Error("等待会话状态更新结果超时")); }, SESSION_UPDATE_TIMEOUT_MS); pendingSessionUpdatesRef.current.set(updateId, { resolve, reject, timeout, }); try { transport.sendAppMessage({ type: "session-update", schema_version: 1, update_id: updateId, dynamic_variables: dynamicVariables, }); } catch (sendError) { window.clearTimeout(timeout); pendingSessionUpdatesRef.current.delete(updateId); reject(new Error(errorMessage(sendError, "发送会话状态更新失败"))); } }); }, [], ); const registerClientTool = useCallback( ( definition: ClientToolDefinition, handler: ClientToolHandler, ): (() => void) => { const registeredTool = { definition, handler }; clientToolHandlersRef.current.set(definition.name, registeredTool); setClientTools( [...clientToolHandlersRef.current.values()].map( (item) => item.definition, ), ); return () => { if ( clientToolHandlersRef.current.get(definition.name) === registeredTool ) { clientToolHandlersRef.current.delete(definition.name); setClientTools( [...clientToolHandlersRef.current.values()].map( (item) => item.definition, ), ); } }; }, [], ); useEffect(() => releaseResources, [releaseResources]); return { status, error, micWarning, localStream, videoStream, remoteStream, messages, sessionVariables, clientTools, callEnded, networkQuality, audioInputs, audioOutputs, selectedDeviceId, selectedOutputDeviceId, setSelectedDeviceId, selectDevice, selectOutputDevice, supportsOutputSelection, sendText, sendUserInput, updateSession, registerClientTool, connect, replaceVideoStream, selectCamera, disconnect, audioRef, }; } export type VoicePreview = ReturnType;