974 lines
31 KiB
TypeScript
974 lines
31 KiB
TypeScript
"use client";
|
||
|
||
/**
|
||
* 语音预览。
|
||
*
|
||
* WebRTC 生命周期交给 Pipecat 官方 SmallWebRTCTransport：它负责固定的
|
||
* audio/video transceiver、trackStatus、keepalive、重协商和 ICE 恢复。
|
||
* 本文件只把平台已有的业务消息映射到 React 状态。
|
||
*/
|
||
|
||
import { useCallback, useEffect, useRef, useState } from "react";
|
||
import Daily from "@daily-co/daily-js";
|
||
import type { PipecatClientOptions, RTVIMessage } from "@pipecat-ai/client-js";
|
||
import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport";
|
||
|
||
import { API_BASE, webrtcApi } from "@/lib/api";
|
||
|
||
export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed";
|
||
export type NetworkQuality = "unknown" | "good" | "fair" | "poor";
|
||
|
||
const NETWORK_SAMPLE_INTERVAL_MS = 2_000;
|
||
const SESSION_UPDATE_TIMEOUT_MS = 10_000;
|
||
|
||
type NetworkStatsSample = {
|
||
packetsSent: number;
|
||
packetsLost: number;
|
||
};
|
||
|
||
type NetworkMetrics = NetworkStatsSample & {
|
||
roundTripTime: number | null;
|
||
availableOutgoingBitrate: number | null;
|
||
};
|
||
|
||
type ConnectOptions = {
|
||
visionEnabled?: boolean;
|
||
videoStream?: MediaStream | null;
|
||
dynamicVariables?: Record<string, string | number | boolean>;
|
||
};
|
||
|
||
export type ChatMessage = {
|
||
id: string;
|
||
role: "user" | "assistant";
|
||
content: string;
|
||
timestamp: string;
|
||
sequence: number;
|
||
turnId?: string;
|
||
streaming?: boolean;
|
||
};
|
||
|
||
type AppMessage = Record<string, unknown> & { type?: string };
|
||
export type DynamicVariableValue = string | number | boolean;
|
||
export type UserInputPart =
|
||
| { type: "input_text"; text: string }
|
||
| {
|
||
type: "input_image";
|
||
source: { type: "camera_frame"; frame: "current" };
|
||
};
|
||
export type UserInputResult = {
|
||
inputId: string;
|
||
status: "accepted";
|
||
};
|
||
export type SessionUpdateRequest = {
|
||
dynamicVariables: Record<string, DynamicVariableValue>;
|
||
};
|
||
export type SessionUpdateResult = {
|
||
updateId: string;
|
||
status: "accepted";
|
||
changed: string[];
|
||
dynamicVariables: Record<string, DynamicVariableValue>;
|
||
};
|
||
export type ClientToolHandler = (
|
||
argumentsValue: Record<string, unknown>,
|
||
) => unknown | Promise<unknown>;
|
||
export type ClientToolDefinition = {
|
||
name: string;
|
||
label: string;
|
||
description: string;
|
||
parameters: readonly ClientToolParameter[];
|
||
};
|
||
export type ClientToolParameter = {
|
||
name: string;
|
||
type: string;
|
||
required: boolean;
|
||
description: string;
|
||
};
|
||
|
||
type RegisteredClientTool = {
|
||
definition: ClientToolDefinition;
|
||
handler: ClientToolHandler;
|
||
};
|
||
|
||
type PendingUserInput = {
|
||
resolve: (result: UserInputResult) => void;
|
||
reject: (error: Error) => void;
|
||
timeout: number;
|
||
};
|
||
|
||
type PendingSessionUpdate = {
|
||
resolve: (result: SessionUpdateResult) => void;
|
||
reject: (error: Error) => void;
|
||
timeout: number;
|
||
};
|
||
|
||
function publicVariableSnapshot(
|
||
value: unknown,
|
||
): Record<string, DynamicVariableValue> {
|
||
if (!value || typeof value !== "object" || Array.isArray(value)) return {};
|
||
return Object.fromEntries(
|
||
Object.entries(value).filter(([name, item]) =>
|
||
!name.startsWith("system__") &&
|
||
!name.startsWith("secret__") &&
|
||
(typeof item === "string" ||
|
||
typeof item === "number" ||
|
||
typeof item === "boolean"),
|
||
) as [string, DynamicVariableValue][],
|
||
);
|
||
}
|
||
|
||
class AppSmallWebRTCTransport extends SmallWebRTCTransport {
|
||
onAppMessage?: (message: AppMessage) => void;
|
||
|
||
/**
|
||
* SmallWebRTCTransport 的媒体管理器会复用 Daily 的全局 call object，
|
||
* 但 disconnect() 不会移除构造时注册的设备和 track 监听器。销毁 call
|
||
* object 后，下一次连接会得到干净的实例，避免监听器跨会话累积。
|
||
*/
|
||
async dispose(): Promise<void> {
|
||
this.onAppMessage = undefined;
|
||
try {
|
||
await this.disconnect();
|
||
} finally {
|
||
const call = Daily.getCallInstance();
|
||
if (call && !call.isDestroyed()) await call.destroy();
|
||
}
|
||
}
|
||
|
||
override handleMessage(raw: string): void {
|
||
try {
|
||
const message = JSON.parse(raw) as AppMessage;
|
||
if (message.type === "signalling" || message.label === "rtvi-ai") {
|
||
super.handleMessage(raw);
|
||
} else {
|
||
this.onAppMessage?.(message);
|
||
}
|
||
} catch {
|
||
super.handleMessage(raw);
|
||
}
|
||
}
|
||
|
||
sendAppMessage(message: AppMessage): void {
|
||
super.sendMessage(message as unknown as RTVIMessage);
|
||
}
|
||
|
||
async connectionStats(): Promise<RTCStatsReport | null> {
|
||
const peerConnection = (
|
||
this as unknown as { pc: RTCPeerConnection | null }
|
||
).pc;
|
||
return peerConnection ? peerConnection.getStats() : null;
|
||
}
|
||
}
|
||
|
||
function errorMessage(error: unknown, fallback: string): string {
|
||
if (error instanceof Error && error.message) return error.message;
|
||
return fallback;
|
||
}
|
||
|
||
function messageOrder(message: ChatMessage): number {
|
||
const timestamp = Date.parse(message.timestamp);
|
||
return Number.isNaN(timestamp) ? Number.MAX_SAFE_INTEGER : timestamp;
|
||
}
|
||
|
||
function sortMessages(messages: ChatMessage[]): ChatMessage[] {
|
||
return messages.sort(
|
||
(a, b) => messageOrder(a) - messageOrder(b) || a.sequence - b.sequence,
|
||
);
|
||
}
|
||
|
||
function encodeHeaderJson(value: unknown): string {
|
||
const bytes = new TextEncoder().encode(JSON.stringify(value));
|
||
let binary = "";
|
||
bytes.forEach((byte) => {
|
||
binary += String.fromCharCode(byte);
|
||
});
|
||
return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
||
}
|
||
|
||
function newInputId(): string {
|
||
if (typeof crypto !== "undefined" && "randomUUID" in crypto) {
|
||
return `input_${crypto.randomUUID()}`;
|
||
}
|
||
return `input_${Date.now()}_${Math.random().toString(16).slice(2)}`;
|
||
}
|
||
|
||
function newUpdateId(): string {
|
||
if (typeof crypto !== "undefined" && "randomUUID" in crypto) {
|
||
return `update_${crypto.randomUUID()}`;
|
||
}
|
||
return `update_${Date.now()}_${Math.random().toString(16).slice(2)}`;
|
||
}
|
||
|
||
function readNetworkMetrics(report: RTCStatsReport): NetworkMetrics | null {
|
||
const outbound: Partial<Record<"audio" | "video", number>> = {};
|
||
const lost: Partial<Record<"audio" | "video", number>> = {};
|
||
let roundTripTime: number | null = null;
|
||
let availableOutgoingBitrate: number | null = null;
|
||
|
||
report.forEach((stat) => {
|
||
const value = stat as RTCStats & Record<string, unknown>;
|
||
const kind = value.kind ?? value.mediaType;
|
||
if (
|
||
value.type === "outbound-rtp" &&
|
||
(kind === "audio" || kind === "video") &&
|
||
typeof value.packetsSent === "number"
|
||
) {
|
||
outbound[kind] = value.packetsSent;
|
||
} else if (
|
||
value.type === "remote-inbound-rtp" &&
|
||
(kind === "audio" || kind === "video")
|
||
) {
|
||
if (typeof value.packetsLost === "number") lost[kind] = value.packetsLost;
|
||
if (typeof value.roundTripTime === "number") {
|
||
roundTripTime = value.roundTripTime;
|
||
}
|
||
} else if (
|
||
value.type === "candidate-pair" &&
|
||
value.state === "succeeded" &&
|
||
(value.nominated === true || value.selected === true)
|
||
) {
|
||
if (typeof value.currentRoundTripTime === "number") {
|
||
roundTripTime = value.currentRoundTripTime;
|
||
}
|
||
if (typeof value.availableOutgoingBitrate === "number") {
|
||
availableOutgoingBitrate = value.availableOutgoingBitrate;
|
||
}
|
||
}
|
||
});
|
||
|
||
const kind = outbound.video !== undefined ? "video" : "audio";
|
||
if (outbound[kind] === undefined && roundTripTime === null) return null;
|
||
return {
|
||
packetsSent: outbound[kind] ?? 0,
|
||
packetsLost: lost[kind] ?? 0,
|
||
roundTripTime,
|
||
availableOutgoingBitrate,
|
||
};
|
||
}
|
||
|
||
function classifyNetworkQuality(
|
||
metrics: NetworkMetrics,
|
||
previous: NetworkStatsSample | null,
|
||
): NetworkQuality {
|
||
const sentDelta = previous
|
||
? Math.max(0, metrics.packetsSent - previous.packetsSent)
|
||
: 0;
|
||
const lostDelta = previous
|
||
? Math.max(0, metrics.packetsLost - previous.packetsLost)
|
||
: 0;
|
||
const lossRate = sentDelta > 0 ? lostDelta / sentDelta : null;
|
||
const { roundTripTime: rtt, availableOutgoingBitrate: bandwidth } = metrics;
|
||
|
||
if (lossRate === null && rtt === null && bandwidth === null) return "unknown";
|
||
if (
|
||
(lossRate !== null && lossRate >= 0.08) ||
|
||
(rtt !== null && rtt >= 0.45) ||
|
||
(bandwidth !== null && bandwidth < 350_000)
|
||
) return "poor";
|
||
if (
|
||
(lossRate !== null && lossRate >= 0.03) ||
|
||
(rtt !== null && rtt >= 0.25) ||
|
||
(bandwidth !== null && bandwidth < 550_000)
|
||
) return "fair";
|
||
return "good";
|
||
}
|
||
|
||
export function useVoicePreview(
|
||
assistantId: string | null,
|
||
onNodeActive?: (nodeId: string | null) => void,
|
||
) {
|
||
const [status, setStatus] = useState<VoicePreviewStatus>("idle");
|
||
const [error, setError] = useState<string | null>(null);
|
||
const [micWarning, setMicWarning] = useState<string | null>(null);
|
||
const [localStream, setLocalStream] = useState<MediaStream | null>(null);
|
||
const [videoStream, setVideoStream] = useState<MediaStream | null>(null);
|
||
const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
|
||
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
||
const [sessionVariables, setSessionVariables] = useState<
|
||
Record<string, DynamicVariableValue>
|
||
>({});
|
||
const [callEnded, setCallEnded] = useState(false);
|
||
const [networkQuality, setNetworkQuality] =
|
||
useState<NetworkQuality>("unknown");
|
||
const [audioInputs, setAudioInputs] = useState<MediaDeviceInfo[]>([]);
|
||
const [audioOutputs, setAudioOutputs] = useState<MediaDeviceInfo[]>([]);
|
||
const [selectedDeviceId, setSelectedDeviceId] = useState("");
|
||
const [selectedOutputDeviceId, setSelectedOutputDeviceId] = useState("");
|
||
const [clientTools, setClientTools] = useState<ClientToolDefinition[]>([]);
|
||
|
||
const transportRef = useRef<AppSmallWebRTCTransport | null>(null);
|
||
const cleanupPromiseRef = useRef<Promise<void>>(Promise.resolve());
|
||
const connectionGenerationRef = useRef(0);
|
||
const startingRef = useRef(false);
|
||
const messageSeqRef = useRef(0);
|
||
const networkStatsRef = useRef<NetworkStatsSample | null>(null);
|
||
const pendingAssistantTurnsRef = useRef(
|
||
new Map<string, { timestamp: string }>(),
|
||
);
|
||
const pendingUserInputsRef = useRef(new Map<string, PendingUserInput>());
|
||
const pendingSessionUpdatesRef = useRef(
|
||
new Map<string, PendingSessionUpdate>(),
|
||
);
|
||
const clientToolHandlersRef = useRef(
|
||
new Map<string, RegisteredClientTool>(),
|
||
);
|
||
const endedByServerRef = useRef(false);
|
||
const selectedDeviceIdRef = useRef("");
|
||
const selectedOutputDeviceIdRef = useRef("");
|
||
const onNodeActiveRef = useRef(onNodeActive);
|
||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||
|
||
useEffect(() => {
|
||
onNodeActiveRef.current = onNodeActive;
|
||
}, [onNodeActive]);
|
||
|
||
const supportsOutputSelection =
|
||
typeof HTMLAudioElement !== "undefined" &&
|
||
"setSinkId" in HTMLAudioElement.prototype;
|
||
|
||
const applyOutputDevice = useCallback(async (deviceId: string) => {
|
||
const audio = audioRef.current;
|
||
if (!audio || !supportsOutputSelection) return;
|
||
try {
|
||
await audio.setSinkId(deviceId);
|
||
} catch {
|
||
// 设备可能已拔出，交给浏览器回退默认输出。
|
||
}
|
||
}, [supportsOutputSelection]);
|
||
|
||
const refreshDevices = useCallback(async () => {
|
||
if (!navigator.mediaDevices?.enumerateDevices) return;
|
||
try {
|
||
const devices = await navigator.mediaDevices.enumerateDevices();
|
||
setAudioInputs(
|
||
devices.filter((item) => item.kind === "audioinput" && item.deviceId),
|
||
);
|
||
setAudioOutputs(
|
||
devices.filter((item) => item.kind === "audiooutput" && item.deviceId),
|
||
);
|
||
} catch {
|
||
// 设备枚举失败不影响仅接收模式。
|
||
}
|
||
}, []);
|
||
|
||
useEffect(() => {
|
||
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||
void refreshDevices();
|
||
navigator.mediaDevices?.addEventListener("devicechange", refreshDevices);
|
||
return () =>
|
||
navigator.mediaDevices?.removeEventListener("devicechange", refreshDevices);
|
||
}, [refreshDevices]);
|
||
|
||
const releaseResources = useCallback(() => {
|
||
connectionGenerationRef.current += 1;
|
||
const transport = transportRef.current;
|
||
transportRef.current = null;
|
||
if (transport) {
|
||
cleanupPromiseRef.current = cleanupPromiseRef.current
|
||
.then(() => transport.dispose())
|
||
.catch(() => {});
|
||
}
|
||
if (audioRef.current) audioRef.current.srcObject = null;
|
||
startingRef.current = false;
|
||
pendingAssistantTurnsRef.current.clear();
|
||
pendingUserInputsRef.current.forEach(({ reject, timeout }) => {
|
||
window.clearTimeout(timeout);
|
||
reject(new Error("连接已关闭，用户输入未完成"));
|
||
});
|
||
pendingUserInputsRef.current.clear();
|
||
pendingSessionUpdatesRef.current.forEach(({ reject, timeout }) => {
|
||
window.clearTimeout(timeout);
|
||
reject(new Error("连接已关闭，会话状态更新未完成"));
|
||
});
|
||
pendingSessionUpdatesRef.current.clear();
|
||
networkStatsRef.current = null;
|
||
onNodeActiveRef.current?.(null);
|
||
}, []);
|
||
|
||
const disconnect = useCallback(() => {
|
||
releaseResources();
|
||
setLocalStream(null);
|
||
setVideoStream(null);
|
||
setRemoteStream(null);
|
||
setMessages([]);
|
||
messageSeqRef.current = 0;
|
||
pendingAssistantTurnsRef.current.clear();
|
||
setError(null);
|
||
setMicWarning(null);
|
||
setNetworkQuality("unknown");
|
||
setStatus("idle");
|
||
}, [releaseResources]);
|
||
|
||
const fail = useCallback((message: string) => {
|
||
releaseResources();
|
||
setLocalStream(null);
|
||
setVideoStream(null);
|
||
setRemoteStream(null);
|
||
setError(message);
|
||
setNetworkQuality("unknown");
|
||
setStatus("failed");
|
||
}, [releaseResources]);
|
||
|
||
const dispatchClientTool = useCallback(async (msg: AppMessage) => {
|
||
const toolCallId =
|
||
typeof msg.tool_call_id === "string" ? msg.tool_call_id : "";
|
||
const functionName =
|
||
typeof msg.function_name === "string" ? msg.function_name : "";
|
||
const waitForResponse = msg.wait_for_response !== false;
|
||
if (!toolCallId || !functionName) return;
|
||
|
||
const registeredTool = clientToolHandlersRef.current.get(functionName);
|
||
const transport = transportRef.current;
|
||
if (!transport || transport.state !== "connected") return;
|
||
if (!registeredTool) {
|
||
if (waitForResponse) {
|
||
transport.sendAppMessage({
|
||
type: "client-tool-result",
|
||
tool_call_id: toolCallId,
|
||
status: "error",
|
||
message: `客户端未注册工具: ${functionName}`,
|
||
});
|
||
}
|
||
return;
|
||
}
|
||
|
||
try {
|
||
const argumentsValue =
|
||
msg.arguments &&
|
||
typeof msg.arguments === "object" &&
|
||
!Array.isArray(msg.arguments)
|
||
? (msg.arguments as Record<string, unknown>)
|
||
: {};
|
||
const data = await registeredTool.handler(argumentsValue);
|
||
if (transportRef.current !== transport) return;
|
||
if (!waitForResponse) return;
|
||
transport.sendAppMessage({
|
||
type: "client-tool-result",
|
||
tool_call_id: toolCallId,
|
||
status: "ok",
|
||
data,
|
||
});
|
||
} catch (toolError) {
|
||
if (transportRef.current !== transport) return;
|
||
if (!waitForResponse) return;
|
||
transport.sendAppMessage({
|
||
type: "client-tool-result",
|
||
tool_call_id: toolCallId,
|
||
status: "error",
|
||
message: errorMessage(toolError, "客户端工具执行失败"),
|
||
});
|
||
}
|
||
}, []);
|
||
|
||
const handleAppMessage = useCallback((msg: AppMessage) => {
|
||
if (msg.type === "client-tool-call") {
|
||
void dispatchClientTool(msg);
|
||
} else if (
|
||
msg.type === "user-input-result" &&
|
||
typeof msg.input_id === "string"
|
||
) {
|
||
const pending = pendingUserInputsRef.current.get(msg.input_id);
|
||
if (!pending) return;
|
||
pendingUserInputsRef.current.delete(msg.input_id);
|
||
window.clearTimeout(pending.timeout);
|
||
if (msg.status === "accepted") {
|
||
pending.resolve({ inputId: msg.input_id, status: "accepted" });
|
||
} else {
|
||
pending.reject(
|
||
new Error(
|
||
typeof msg.message === "string" ? msg.message : "用户输入处理失败",
|
||
),
|
||
);
|
||
}
|
||
} else if (
|
||
msg.type === "session-update-result" &&
|
||
typeof msg.update_id === "string"
|
||
) {
|
||
const pending = pendingSessionUpdatesRef.current.get(msg.update_id);
|
||
if (!pending) return;
|
||
pendingSessionUpdatesRef.current.delete(msg.update_id);
|
||
window.clearTimeout(pending.timeout);
|
||
if (msg.status === "accepted") {
|
||
const dynamicVariables = publicVariableSnapshot(msg.dynamic_variables);
|
||
const changed = Array.isArray(msg.changed)
|
||
? msg.changed.filter((name): name is string => typeof name === "string")
|
||
: [];
|
||
setSessionVariables(dynamicVariables);
|
||
pending.resolve({
|
||
updateId: msg.update_id,
|
||
status: "accepted",
|
||
changed,
|
||
dynamicVariables,
|
||
});
|
||
} else {
|
||
pending.reject(
|
||
new Error(
|
||
typeof msg.message === "string" ? msg.message : "会话状态更新失败",
|
||
),
|
||
);
|
||
}
|
||
} else if (
|
||
msg.type === "assistant-text-start" &&
|
||
typeof msg.turn_id === "string"
|
||
) {
|
||
pendingAssistantTurnsRef.current.set(msg.turn_id, {
|
||
timestamp:
|
||
typeof msg.timestamp === "string"
|
||
? msg.timestamp
|
||
: new Date().toISOString(),
|
||
});
|
||
} else if (
|
||
msg.type === "assistant-text-delta" &&
|
||
typeof msg.turn_id === "string" &&
|
||
typeof msg.delta === "string" &&
|
||
msg.delta.length > 0
|
||
) {
|
||
const pending = pendingAssistantTurnsRef.current.get(msg.turn_id);
|
||
if (pending) {
|
||
pendingAssistantTurnsRef.current.delete(msg.turn_id);
|
||
messageSeqRef.current += 1;
|
||
const sequence = messageSeqRef.current;
|
||
setMessages((previous) =>
|
||
sortMessages([
|
||
...previous,
|
||
{
|
||
id: `assistant-${msg.turn_id as string}`,
|
||
role: "assistant",
|
||
content: msg.delta as string,
|
||
timestamp: pending.timestamp,
|
||
sequence,
|
||
turnId: msg.turn_id as string,
|
||
streaming: true,
|
||
},
|
||
]),
|
||
);
|
||
} else {
|
||
setMessages((previous) =>
|
||
previous.map((message) =>
|
||
message.turnId === msg.turn_id
|
||
? { ...message, content: message.content + msg.delta }
|
||
: message,
|
||
),
|
||
);
|
||
}
|
||
} else if (
|
||
msg.type === "assistant-text-end" &&
|
||
typeof msg.turn_id === "string"
|
||
) {
|
||
const pending = pendingAssistantTurnsRef.current.get(msg.turn_id);
|
||
pendingAssistantTurnsRef.current.delete(msg.turn_id);
|
||
const finalContent =
|
||
typeof msg.content === "string" ? msg.content.trim() : "";
|
||
setMessages((previous) => {
|
||
const existing = previous.some((message) => message.turnId === msg.turn_id);
|
||
if (existing) {
|
||
return previous.map((message) =>
|
||
message.turnId === msg.turn_id
|
||
? {
|
||
...message,
|
||
content: finalContent || message.content,
|
||
streaming: false,
|
||
}
|
||
: message,
|
||
);
|
||
}
|
||
if (!finalContent) return previous;
|
||
messageSeqRef.current += 1;
|
||
return sortMessages([
|
||
...previous,
|
||
{
|
||
id: `assistant-${msg.turn_id as string}`,
|
||
role: "assistant",
|
||
content: finalContent,
|
||
timestamp: pending?.timestamp ?? new Date().toISOString(),
|
||
sequence: messageSeqRef.current,
|
||
turnId: msg.turn_id as string,
|
||
streaming: false,
|
||
},
|
||
]);
|
||
});
|
||
} else if (
|
||
msg.type === "transcript" &&
|
||
(msg.role === "user" || msg.role === "assistant") &&
|
||
typeof msg.content === "string" &&
|
||
msg.content.trim()
|
||
) {
|
||
messageSeqRef.current += 1;
|
||
setMessages((previous) =>
|
||
sortMessages([
|
||
...previous,
|
||
{
|
||
id: `msg-${messageSeqRef.current}`,
|
||
role: msg.role as "user" | "assistant",
|
||
content: msg.content as string,
|
||
timestamp:
|
||
typeof msg.timestamp === "string"
|
||
? msg.timestamp
|
||
: new Date().toISOString(),
|
||
sequence: messageSeqRef.current,
|
||
},
|
||
]),
|
||
);
|
||
} else if (msg.type === "node-active" && typeof msg.nodeId === "string") {
|
||
onNodeActiveRef.current?.(msg.nodeId);
|
||
} else if (msg.type === "workflow-variables") {
|
||
setSessionVariables(publicVariableSnapshot(msg.variables));
|
||
} else if (msg.type === "call-ended") {
|
||
endedByServerRef.current = true;
|
||
setCallEnded(true);
|
||
disconnect();
|
||
}
|
||
}, [disconnect, dispatchClientTool]);
|
||
|
||
const connect = useCallback(async (options: ConnectOptions = {}) => {
|
||
if (startingRef.current || transportRef.current) return;
|
||
if (!assistantId) {
|
||
setError("请先保存助手，再开始语音预览。");
|
||
setStatus("failed");
|
||
return;
|
||
}
|
||
|
||
startingRef.current = true;
|
||
const generation = connectionGenerationRef.current + 1;
|
||
connectionGenerationRef.current = generation;
|
||
setStatus("connecting");
|
||
setError(null);
|
||
setMicWarning(null);
|
||
setMessages([]);
|
||
setSessionVariables(publicVariableSnapshot(options.dynamicVariables ?? {}));
|
||
pendingAssistantTurnsRef.current.clear();
|
||
setCallEnded(false);
|
||
endedByServerRef.current = false;
|
||
|
||
await cleanupPromiseRef.current;
|
||
if (connectionGenerationRef.current !== generation) return;
|
||
|
||
const iceServers = await webrtcApi
|
||
.iceServers()
|
||
.then((response) => response.iceServers)
|
||
.catch(() => [{ urls: "stun:stun.l.google.com:19302" }]);
|
||
if (connectionGenerationRef.current !== generation) return;
|
||
|
||
const transport = new AppSmallWebRTCTransport({ iceServers });
|
||
transportRef.current = transport;
|
||
transport.onAppMessage = handleAppMessage;
|
||
|
||
const callbacks: NonNullable<PipecatClientOptions["callbacks"]> = {
|
||
onConnected: () => setStatus("connected"),
|
||
onDisconnected: () => {
|
||
if (transportRef.current !== transport) return;
|
||
if (endedByServerRef.current) disconnect();
|
||
else fail("WebRTC 连接已断开。");
|
||
},
|
||
onTrackStarted: (track) => {
|
||
if (track.kind !== "audio") return;
|
||
const stream = new MediaStream([track]);
|
||
setRemoteStream(stream);
|
||
if (audioRef.current) {
|
||
audioRef.current.srcObject = stream;
|
||
void applyOutputDevice(selectedOutputDeviceIdRef.current).then(() =>
|
||
audioRef.current?.play().catch(() => {}),
|
||
);
|
||
}
|
||
},
|
||
onDeviceError: (deviceError) => {
|
||
setMicWarning(deviceError.message || "无法访问麦克风。已尝试继续连接。");
|
||
},
|
||
onCamUpdated: () => {
|
||
const track = transport.tracks().local.video;
|
||
setVideoStream(track ? new MediaStream([track]) : null);
|
||
},
|
||
onTransportStateChanged: (nextState) => {
|
||
if (nextState === "connected" || nextState === "ready") {
|
||
setStatus("connected");
|
||
} else if (nextState === "error") {
|
||
fail("WebRTC 连接失败。");
|
||
}
|
||
},
|
||
};
|
||
|
||
transport.initialize(
|
||
{
|
||
transport,
|
||
enableMic: true,
|
||
enableCam: Boolean(options.visionEnabled),
|
||
callbacks,
|
||
},
|
||
() => {},
|
||
);
|
||
|
||
try {
|
||
await transport.initDevices();
|
||
if (connectionGenerationRef.current !== generation) return;
|
||
if (selectedDeviceIdRef.current) {
|
||
await transport.updateMic(selectedDeviceIdRef.current);
|
||
}
|
||
if (connectionGenerationRef.current !== generation) return;
|
||
const localAudio = transport.tracks().local.audio;
|
||
const localVideo = transport.tracks().local.video;
|
||
setLocalStream(localAudio ? new MediaStream([localAudio]) : null);
|
||
setVideoStream(localVideo ? new MediaStream([localVideo]) : null);
|
||
void refreshDevices();
|
||
|
||
const request = new Request(`${API_BASE}/api/webrtc/offer`, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
"X-Pipecat-Assistant-ID": assistantId,
|
||
"X-Pipecat-Dynamic-Variables": encodeHeaderJson(
|
||
options.dynamicVariables ?? {},
|
||
),
|
||
},
|
||
credentials: "include",
|
||
});
|
||
await transport.connect({
|
||
webrtcRequestParams: {
|
||
endpoint: request,
|
||
requestData: {
|
||
assistant_id: assistantId,
|
||
vision_enabled: Boolean(options.visionEnabled),
|
||
dynamic_variables: options.dynamicVariables ?? {},
|
||
},
|
||
},
|
||
});
|
||
if (connectionGenerationRef.current !== generation) return;
|
||
transport.sendAppMessage({ type: "client-ready" });
|
||
setStatus("connected");
|
||
} catch (connectionError) {
|
||
if (connectionGenerationRef.current === generation) {
|
||
fail(errorMessage(connectionError, "无法连接语音服务。"));
|
||
}
|
||
} finally {
|
||
if (connectionGenerationRef.current === generation) {
|
||
startingRef.current = false;
|
||
}
|
||
}
|
||
}, [
|
||
assistantId,
|
||
applyOutputDevice,
|
||
disconnect,
|
||
fail,
|
||
handleAppMessage,
|
||
refreshDevices,
|
||
]);
|
||
|
||
const replaceVideoStream = useCallback(async (videoStream: MediaStream | null) => {
|
||
const transport = transportRef.current;
|
||
const deviceId = videoStream?.getVideoTracks()[0]?.getSettings().deviceId;
|
||
if (transport && deviceId) await transport.updateCam(deviceId);
|
||
}, []);
|
||
|
||
const selectCamera = useCallback((deviceId: string) => {
|
||
transportRef.current?.updateCam(deviceId);
|
||
}, []);
|
||
|
||
const selectDevice = useCallback(async (deviceId: string) => {
|
||
setSelectedDeviceId(deviceId);
|
||
selectedDeviceIdRef.current = deviceId;
|
||
const transport = transportRef.current;
|
||
if (!transport) return;
|
||
try {
|
||
await transport.updateMic(deviceId);
|
||
const audio = transport.tracks().local.audio;
|
||
setLocalStream(audio ? new MediaStream([audio]) : null);
|
||
setMicWarning(null);
|
||
} catch (deviceError) {
|
||
setMicWarning(errorMessage(deviceError, "无法切换麦克风。"));
|
||
}
|
||
}, []);
|
||
|
||
const selectOutputDevice = useCallback((deviceId: string) => {
|
||
setSelectedOutputDeviceId(deviceId);
|
||
selectedOutputDeviceIdRef.current = deviceId;
|
||
void applyOutputDevice(deviceId);
|
||
}, [applyOutputDevice]);
|
||
|
||
useEffect(() => {
|
||
if (status !== "connected") return;
|
||
let cancelled = false;
|
||
const sample = async () => {
|
||
const transport = transportRef.current;
|
||
if (!transport) return;
|
||
try {
|
||
const report = await transport.connectionStats();
|
||
const metrics = report ? readNetworkMetrics(report) : null;
|
||
if (cancelled || transportRef.current !== transport) return;
|
||
if (!metrics) {
|
||
setNetworkQuality("unknown");
|
||
return;
|
||
}
|
||
setNetworkQuality(
|
||
classifyNetworkQuality(metrics, networkStatsRef.current),
|
||
);
|
||
networkStatsRef.current = {
|
||
packetsSent: metrics.packetsSent,
|
||
packetsLost: metrics.packetsLost,
|
||
};
|
||
} catch {
|
||
if (!cancelled) setNetworkQuality("unknown");
|
||
}
|
||
};
|
||
void sample();
|
||
const interval = window.setInterval(() => void sample(), NETWORK_SAMPLE_INTERVAL_MS);
|
||
return () => {
|
||
cancelled = true;
|
||
window.clearInterval(interval);
|
||
};
|
||
}, [status]);
|
||
|
||
const sendText = useCallback((text: string): boolean => {
|
||
const trimmed = text.trim();
|
||
const transport = transportRef.current;
|
||
if (!trimmed || !transport || transport.state !== "connected") return false;
|
||
const inputId = newInputId();
|
||
transport.sendAppMessage({
|
||
type: "user-input",
|
||
schema_version: 1,
|
||
input_id: inputId,
|
||
parts: [{ type: "input_text", text: trimmed }],
|
||
options: { run_immediately: true, interrupt: true },
|
||
});
|
||
return true;
|
||
}, []);
|
||
|
||
const sendUserInput = useCallback(
|
||
(
|
||
parts: UserInputPart[],
|
||
options: { runImmediately?: boolean; interrupt?: boolean } = {},
|
||
): Promise<UserInputResult> => {
|
||
const transport = transportRef.current;
|
||
if (!transport || transport.state !== "connected") {
|
||
return Promise.reject(new Error("当前未连接语音服务"));
|
||
}
|
||
const inputId = newInputId();
|
||
return new Promise<UserInputResult>((resolve, reject) => {
|
||
const timeout = window.setTimeout(() => {
|
||
pendingUserInputsRef.current.delete(inputId);
|
||
reject(new Error("等待用户输入处理结果超时"));
|
||
}, 30_000);
|
||
pendingUserInputsRef.current.set(inputId, { resolve, reject, timeout });
|
||
try {
|
||
transport.sendAppMessage({
|
||
type: "user-input",
|
||
schema_version: 1,
|
||
input_id: inputId,
|
||
parts,
|
||
options: {
|
||
run_immediately: options.runImmediately ?? true,
|
||
interrupt: options.interrupt ?? true,
|
||
},
|
||
});
|
||
} catch (sendError) {
|
||
window.clearTimeout(timeout);
|
||
pendingUserInputsRef.current.delete(inputId);
|
||
reject(
|
||
new Error(errorMessage(sendError, "发送用户输入失败")),
|
||
);
|
||
}
|
||
});
|
||
},
|
||
[],
|
||
);
|
||
|
||
const updateSession = useCallback(
|
||
({
|
||
dynamicVariables,
|
||
}: SessionUpdateRequest): Promise<SessionUpdateResult> => {
|
||
if (Object.keys(dynamicVariables).length === 0) {
|
||
return Promise.reject(new Error("至少需要更新一个动态变量"));
|
||
}
|
||
const transport = transportRef.current;
|
||
if (!transport || transport.state !== "connected") {
|
||
return Promise.reject(new Error("当前未连接语音服务"));
|
||
}
|
||
const updateId = newUpdateId();
|
||
return new Promise<SessionUpdateResult>((resolve, reject) => {
|
||
const timeout = window.setTimeout(() => {
|
||
pendingSessionUpdatesRef.current.delete(updateId);
|
||
reject(new Error("等待会话状态更新结果超时"));
|
||
}, SESSION_UPDATE_TIMEOUT_MS);
|
||
pendingSessionUpdatesRef.current.set(updateId, {
|
||
resolve,
|
||
reject,
|
||
timeout,
|
||
});
|
||
try {
|
||
transport.sendAppMessage({
|
||
type: "session-update",
|
||
schema_version: 1,
|
||
update_id: updateId,
|
||
dynamic_variables: dynamicVariables,
|
||
});
|
||
} catch (sendError) {
|
||
window.clearTimeout(timeout);
|
||
pendingSessionUpdatesRef.current.delete(updateId);
|
||
reject(new Error(errorMessage(sendError, "发送会话状态更新失败")));
|
||
}
|
||
});
|
||
},
|
||
[],
|
||
);
|
||
|
||
const registerClientTool = useCallback(
|
||
(
|
||
definition: ClientToolDefinition,
|
||
handler: ClientToolHandler,
|
||
): (() => void) => {
|
||
const registeredTool = { definition, handler };
|
||
clientToolHandlersRef.current.set(definition.name, registeredTool);
|
||
setClientTools(
|
||
[...clientToolHandlersRef.current.values()].map(
|
||
(item) => item.definition,
|
||
),
|
||
);
|
||
return () => {
|
||
if (
|
||
clientToolHandlersRef.current.get(definition.name) === registeredTool
|
||
) {
|
||
clientToolHandlersRef.current.delete(definition.name);
|
||
setClientTools(
|
||
[...clientToolHandlersRef.current.values()].map(
|
||
(item) => item.definition,
|
||
),
|
||
);
|
||
}
|
||
};
|
||
},
|
||
[],
|
||
);
|
||
|
||
useEffect(() => releaseResources, [releaseResources]);
|
||
|
||
return {
|
||
status,
|
||
error,
|
||
micWarning,
|
||
localStream,
|
||
videoStream,
|
||
remoteStream,
|
||
messages,
|
||
sessionVariables,
|
||
clientTools,
|
||
callEnded,
|
||
networkQuality,
|
||
audioInputs,
|
||
audioOutputs,
|
||
selectedDeviceId,
|
||
selectedOutputDeviceId,
|
||
setSelectedDeviceId,
|
||
selectDevice,
|
||
selectOutputDevice,
|
||
supportsOutputSelection,
|
||
sendText,
|
||
sendUserInput,
|
||
updateSession,
|
||
registerClientTool,
|
||
connect,
|
||
replaceVideoStream,
|
||
selectCamera,
|
||
disconnect,
|
||
audioRef,
|
||
};
|
||
}
|
||
|
||
export type VoicePreview = ReturnType<typeof useVoicePreview>;
|