Add Network Quality Indicator component and integrate into Assistant and Mobile Call pages
- Introduce a new `NetworkQualityIndicator` component to visually represent network quality status during calls. - Integrate the `NetworkQualityIndicator` into `AssistantPage` and `MobileCallPage`, enhancing user awareness of connection quality. - Update `use-voice-preview` hook to manage network quality metrics, improving overall call experience and performance monitoring.
This commit is contained in:
70
frontend/src/components/network-quality-indicator.tsx
Normal file
70
frontend/src/components/network-quality-indicator.tsx
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { Wifi, WifiOff } from "lucide-react";
|
||||||
|
|
||||||
|
import type {
|
||||||
|
NetworkQuality,
|
||||||
|
VoicePreviewStatus,
|
||||||
|
} from "@/hooks/use-voice-preview";
|
||||||
|
|
||||||
|
const QUALITY_VIEW: Record<
|
||||||
|
NetworkQuality,
|
||||||
|
{ label: string; color: string; darkColor: string }
|
||||||
|
> = {
|
||||||
|
unknown: {
|
||||||
|
label: "检测中",
|
||||||
|
color: "text-muted-foreground",
|
||||||
|
darkColor: "text-white/55",
|
||||||
|
},
|
||||||
|
good: {
|
||||||
|
label: "网络良好",
|
||||||
|
color: "text-emerald-600",
|
||||||
|
darkColor: "text-emerald-300",
|
||||||
|
},
|
||||||
|
fair: {
|
||||||
|
label: "网络一般",
|
||||||
|
color: "text-amber-600",
|
||||||
|
darkColor: "text-amber-300",
|
||||||
|
},
|
||||||
|
poor: {
|
||||||
|
label: "网络较差",
|
||||||
|
color: "text-red-600",
|
||||||
|
darkColor: "text-red-300",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
export function NetworkQualityIndicator({
|
||||||
|
quality,
|
||||||
|
status,
|
||||||
|
dark = false,
|
||||||
|
className = "",
|
||||||
|
}: {
|
||||||
|
quality: NetworkQuality;
|
||||||
|
status: VoicePreviewStatus;
|
||||||
|
dark?: boolean;
|
||||||
|
className?: string;
|
||||||
|
}) {
|
||||||
|
const connected = status === "connected";
|
||||||
|
const qualityView = QUALITY_VIEW[quality];
|
||||||
|
const view = connected
|
||||||
|
? {
|
||||||
|
label: qualityView.label,
|
||||||
|
color: dark ? qualityView.darkColor : qualityView.color,
|
||||||
|
}
|
||||||
|
: {
|
||||||
|
label: status === "connecting" ? "检测中" : "未连接",
|
||||||
|
color: dark ? "text-white/55" : "text-muted-foreground",
|
||||||
|
};
|
||||||
|
const Icon = connected || status === "connecting" ? Wifi : WifiOff;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<span
|
||||||
|
className={`inline-flex items-center gap-1.5 whitespace-nowrap text-xs ${view.color} ${className}`}
|
||||||
|
title={view.label}
|
||||||
|
aria-label={view.label}
|
||||||
|
>
|
||||||
|
<Icon className="size-3.5" aria-hidden="true" />
|
||||||
|
<span>{view.label}</span>
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -40,6 +40,7 @@ import {
|
|||||||
|
|
||||||
import { Button } from "@/components/ui/button";
|
import { Button } from "@/components/ui/button";
|
||||||
import { Badge } from "@/components/ui/badge";
|
import { Badge } from "@/components/ui/badge";
|
||||||
|
import { NetworkQualityIndicator } from "@/components/network-quality-indicator";
|
||||||
import { Input } from "@/components/ui/input";
|
import { Input } from "@/components/ui/input";
|
||||||
import { Textarea } from "@/components/ui/textarea";
|
import { Textarea } from "@/components/ui/textarea";
|
||||||
import { Switch } from "@/components/ui/switch";
|
import { Switch } from "@/components/ui/switch";
|
||||||
@@ -2037,6 +2038,10 @@ function DebugDrawer({
|
|||||||
<div className="shrink-0 text-sm font-medium text-foreground">
|
<div className="shrink-0 text-sm font-medium text-foreground">
|
||||||
调试与预览
|
调试与预览
|
||||||
</div>
|
</div>
|
||||||
|
<NetworkQualityIndicator
|
||||||
|
quality={preview.networkQuality}
|
||||||
|
status={preview.status}
|
||||||
|
/>
|
||||||
<DebugConnectionStatus
|
<DebugConnectionStatus
|
||||||
status={preview.status}
|
status={preview.status}
|
||||||
micWarning={preview.micWarning}
|
micWarning={preview.micWarning}
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import {
|
|||||||
Video,
|
Video,
|
||||||
} from "lucide-react";
|
} from "lucide-react";
|
||||||
import { Button } from "@/components/ui/button";
|
import { Button } from "@/components/ui/button";
|
||||||
|
import { NetworkQualityIndicator } from "@/components/network-quality-indicator";
|
||||||
import {
|
import {
|
||||||
DropdownMenu,
|
DropdownMenu,
|
||||||
DropdownMenuContent,
|
DropdownMenuContent,
|
||||||
@@ -247,6 +248,7 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
|
|||||||
selectOutputDevice,
|
selectOutputDevice,
|
||||||
supportsOutputSelection,
|
supportsOutputSelection,
|
||||||
messages,
|
messages,
|
||||||
|
networkQuality,
|
||||||
connect,
|
connect,
|
||||||
replaceVideoStream,
|
replaceVideoStream,
|
||||||
disconnect,
|
disconnect,
|
||||||
@@ -325,6 +327,13 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
|
|||||||
|
|
||||||
{view === "chat" && <MobileCallTranscript messages={messages} />}
|
{view === "chat" && <MobileCallTranscript messages={messages} />}
|
||||||
|
|
||||||
|
<NetworkQualityIndicator
|
||||||
|
quality={networkQuality}
|
||||||
|
status={status}
|
||||||
|
dark
|
||||||
|
className="absolute left-4 top-[max(1rem,env(safe-area-inset-top))] z-10 rounded-full border border-white/10 bg-black/30 px-3 py-1.5 backdrop-blur-md"
|
||||||
|
/>
|
||||||
|
|
||||||
<div className="absolute left-1/2 top-[max(1rem,env(safe-area-inset-top))] z-10 flex -translate-x-1/2 items-center gap-2 rounded-full border border-white/10 bg-black/30 px-3 py-1.5 text-xs text-white/85 backdrop-blur-md">
|
<div className="absolute left-1/2 top-[max(1rem,env(safe-area-inset-top))] z-10 flex -translate-x-1/2 items-center gap-2 rounded-full border border-white/10 bg-black/30 px-3 py-1.5 text-xs text-white/85 backdrop-blur-md">
|
||||||
<span
|
<span
|
||||||
className={[
|
className={[
|
||||||
|
|||||||
@@ -9,6 +9,12 @@
|
|||||||
|
|
||||||
import { useCallback, useEffect, useRef, useState } from "react";
|
import { useCallback, useEffect, useRef, useState } from "react";
|
||||||
|
|
||||||
|
const CAMERA_VIDEO_CONSTRAINTS: MediaTrackConstraints = {
|
||||||
|
width: { ideal: 640, max: 640 },
|
||||||
|
height: { ideal: 360, max: 480 },
|
||||||
|
frameRate: { ideal: 12, max: 15 },
|
||||||
|
};
|
||||||
|
|
||||||
function cameraErrorMessage(error: unknown): string {
|
function cameraErrorMessage(error: unknown): string {
|
||||||
if (error instanceof DOMException) {
|
if (error instanceof DOMException) {
|
||||||
if (error.name === "NotAllowedError")
|
if (error.name === "NotAllowedError")
|
||||||
@@ -59,8 +65,12 @@ export function useCameraPreview() {
|
|||||||
setStarting(true);
|
setStarting(true);
|
||||||
try {
|
try {
|
||||||
const id = deviceIdRef.current;
|
const id = deviceIdRef.current;
|
||||||
|
const videoConstraints: MediaTrackConstraints = {
|
||||||
|
...CAMERA_VIDEO_CONSTRAINTS,
|
||||||
|
...(id ? { deviceId: { exact: id } } : {}),
|
||||||
|
};
|
||||||
const media = await navigator.mediaDevices.getUserMedia({
|
const media = await navigator.mediaDevices.getUserMedia({
|
||||||
video: id ? { deviceId: { exact: id } } : true,
|
video: videoConstraints,
|
||||||
});
|
});
|
||||||
streamRef.current = media;
|
streamRef.current = media;
|
||||||
setStream(media);
|
setStream(media);
|
||||||
|
|||||||
@@ -22,6 +22,30 @@ import { useCallback, useEffect, useRef, useState } from "react";
|
|||||||
import { API_BASE, webrtcApi } from "@/lib/api";
|
import { API_BASE, webrtcApi } from "@/lib/api";
|
||||||
|
|
||||||
export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed";
|
export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed";
|
||||||
|
export type NetworkQuality = "unknown" | "good" | "fair" | "poor";
|
||||||
|
|
||||||
|
const NETWORK_SAMPLE_INTERVAL_MS = 2_000;
|
||||||
|
const VIDEO_BITRATE_BY_QUALITY: Record<
|
||||||
|
Exclude<NetworkQuality, "unknown">,
|
||||||
|
number
|
||||||
|
> = {
|
||||||
|
good: 600_000,
|
||||||
|
fair: 500_000,
|
||||||
|
poor: 300_000,
|
||||||
|
};
|
||||||
|
const INITIAL_VIDEO_BITRATE = VIDEO_BITRATE_BY_QUALITY.fair;
|
||||||
|
|
||||||
|
type NetworkStatsSample = {
|
||||||
|
packetsSent: number;
|
||||||
|
packetsLost: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
type VideoNetworkMetrics = {
|
||||||
|
packetsSent: number | null;
|
||||||
|
packetsLost: number | null;
|
||||||
|
roundTripTime: number | null;
|
||||||
|
availableOutgoingBitrate: number | null;
|
||||||
|
};
|
||||||
|
|
||||||
type ConnectOptions = {
|
type ConnectOptions = {
|
||||||
visionEnabled?: boolean;
|
visionEnabled?: boolean;
|
||||||
@@ -88,6 +112,97 @@ function microphoneErrorMessage(error: unknown): string {
|
|||||||
return errorMessage(error, "无法访问麦克风。");
|
return errorMessage(error, "无法访问麦克风。");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function setVideoBitrate(
|
||||||
|
sender: RTCRtpSender,
|
||||||
|
bitrate: number,
|
||||||
|
): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
const parameters = sender.getParameters();
|
||||||
|
if (!parameters.encodings.length) parameters.encodings = [{}];
|
||||||
|
parameters.encodings[0].maxBitrate = bitrate;
|
||||||
|
parameters.degradationPreference = "maintain-resolution";
|
||||||
|
await sender.setParameters(parameters);
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
// 不支持发送参数的浏览器继续使用 WebRTC 自带的拥塞控制。
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function readVideoNetworkMetrics(report: RTCStatsReport): VideoNetworkMetrics {
|
||||||
|
const metrics: VideoNetworkMetrics = {
|
||||||
|
packetsSent: null,
|
||||||
|
packetsLost: null,
|
||||||
|
roundTripTime: null,
|
||||||
|
availableOutgoingBitrate: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
report.forEach((stat) => {
|
||||||
|
const values = stat as RTCStats & Record<string, unknown>;
|
||||||
|
const kind = values.kind ?? values.mediaType;
|
||||||
|
if (values.type === "outbound-rtp" && kind === "video") {
|
||||||
|
if (typeof values.packetsSent === "number") {
|
||||||
|
metrics.packetsSent = values.packetsSent;
|
||||||
|
}
|
||||||
|
} else if (values.type === "remote-inbound-rtp" && kind === "video") {
|
||||||
|
if (typeof values.packetsLost === "number") {
|
||||||
|
metrics.packetsLost = values.packetsLost;
|
||||||
|
}
|
||||||
|
if (typeof values.roundTripTime === "number") {
|
||||||
|
metrics.roundTripTime = values.roundTripTime;
|
||||||
|
}
|
||||||
|
} else if (
|
||||||
|
values.type === "candidate-pair" &&
|
||||||
|
values.state === "succeeded" &&
|
||||||
|
(values.nominated === true || values.selected === true)
|
||||||
|
) {
|
||||||
|
if (typeof values.currentRoundTripTime === "number") {
|
||||||
|
metrics.roundTripTime = values.currentRoundTripTime;
|
||||||
|
}
|
||||||
|
if (typeof values.availableOutgoingBitrate === "number") {
|
||||||
|
metrics.availableOutgoingBitrate = values.availableOutgoingBitrate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return metrics;
|
||||||
|
}
|
||||||
|
|
||||||
|
function classifyNetworkQuality(
|
||||||
|
metrics: VideoNetworkMetrics,
|
||||||
|
previous: NetworkStatsSample | null,
|
||||||
|
): NetworkQuality {
|
||||||
|
const hasPacketSample =
|
||||||
|
previous !== null &&
|
||||||
|
metrics.packetsSent !== null &&
|
||||||
|
metrics.packetsLost !== null;
|
||||||
|
const sentDelta = hasPacketSample
|
||||||
|
? Math.max(0, metrics.packetsSent! - previous.packetsSent)
|
||||||
|
: 0;
|
||||||
|
const lostDelta = hasPacketSample
|
||||||
|
? Math.max(0, metrics.packetsLost! - previous.packetsLost)
|
||||||
|
: 0;
|
||||||
|
const lossRate = sentDelta > 0 ? lostDelta / sentDelta : null;
|
||||||
|
const { roundTripTime: rtt, availableOutgoingBitrate: bandwidth } = metrics;
|
||||||
|
|
||||||
|
if (lossRate === null && rtt === null && bandwidth === null) return "unknown";
|
||||||
|
if (
|
||||||
|
(lossRate !== null && lossRate >= 0.08) ||
|
||||||
|
(rtt !== null && rtt >= 0.45) ||
|
||||||
|
(bandwidth !== null && bandwidth < 350_000)
|
||||||
|
) {
|
||||||
|
return "poor";
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
(lossRate !== null && lossRate >= 0.03) ||
|
||||||
|
(rtt !== null && rtt >= 0.25) ||
|
||||||
|
(bandwidth !== null && bandwidth < 550_000)
|
||||||
|
) {
|
||||||
|
return "fair";
|
||||||
|
}
|
||||||
|
return "good";
|
||||||
|
}
|
||||||
|
|
||||||
export function useVoicePreview(
|
export function useVoicePreview(
|
||||||
assistantId: string | null,
|
assistantId: string | null,
|
||||||
onNodeActive?: (nodeId: string | null) => void,
|
onNodeActive?: (nodeId: string | null) => void,
|
||||||
@@ -99,6 +214,8 @@ export function useVoicePreview(
|
|||||||
// 远端(助手 TTS)媒体流:除挂到 <audio> 播放外,也暴露给波形可视化
|
// 远端(助手 TTS)媒体流:除挂到 <audio> 播放外,也暴露给波形可视化
|
||||||
const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
|
const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
|
||||||
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
||||||
|
const [networkQuality, setNetworkQuality] =
|
||||||
|
useState<NetworkQuality>("unknown");
|
||||||
// 可选麦克风/扬声器列表与当前选择(空串表示交给浏览器选默认设备)
|
// 可选麦克风/扬声器列表与当前选择(空串表示交给浏览器选默认设备)
|
||||||
const [audioInputs, setAudioInputs] = useState<MediaDeviceInfo[]>([]);
|
const [audioInputs, setAudioInputs] = useState<MediaDeviceInfo[]>([]);
|
||||||
const [audioOutputs, setAudioOutputs] = useState<MediaDeviceInfo[]>([]);
|
const [audioOutputs, setAudioOutputs] = useState<MediaDeviceInfo[]>([]);
|
||||||
@@ -114,6 +231,8 @@ export function useVoicePreview(
|
|||||||
const localStreamRef = useRef<MediaStream | null>(null);
|
const localStreamRef = useRef<MediaStream | null>(null);
|
||||||
const startingRef = useRef(false);
|
const startingRef = useRef(false);
|
||||||
const messageSeqRef = useRef(0);
|
const messageSeqRef = useRef(0);
|
||||||
|
const networkStatsRef = useRef<NetworkStatsSample | null>(null);
|
||||||
|
const videoBitrateRef = useRef(INITIAL_VIDEO_BITRATE);
|
||||||
// 后端主动结束(工作流走到结束节点)标记:据此把随后的断开当作正常结束而非报错
|
// 后端主动结束(工作流走到结束节点)标记:据此把随后的断开当作正常结束而非报错
|
||||||
const endedByServerRef = useRef(false);
|
const endedByServerRef = useRef(false);
|
||||||
// 工作流激活节点回调存进 ref,避免把它挂进 connect 依赖反复重建连接
|
// 工作流激活节点回调存进 ref,避免把它挂进 connect 依赖反复重建连接
|
||||||
@@ -233,6 +352,9 @@ export function useVoicePreview(
|
|||||||
messageSeqRef.current = 0;
|
messageSeqRef.current = 0;
|
||||||
setError(null);
|
setError(null);
|
||||||
setMicWarning(null);
|
setMicWarning(null);
|
||||||
|
setNetworkQuality("unknown");
|
||||||
|
networkStatsRef.current = null;
|
||||||
|
videoBitrateRef.current = INITIAL_VIDEO_BITRATE;
|
||||||
setStatus("idle");
|
setStatus("idle");
|
||||||
}, [releaseResources]);
|
}, [releaseResources]);
|
||||||
|
|
||||||
@@ -242,6 +364,9 @@ export function useVoicePreview(
|
|||||||
setLocalStream(null);
|
setLocalStream(null);
|
||||||
setRemoteStream(null);
|
setRemoteStream(null);
|
||||||
setError(message);
|
setError(message);
|
||||||
|
setNetworkQuality("unknown");
|
||||||
|
networkStatsRef.current = null;
|
||||||
|
videoBitrateRef.current = INITIAL_VIDEO_BITRATE;
|
||||||
setStatus("failed");
|
setStatus("failed");
|
||||||
},
|
},
|
||||||
[releaseResources],
|
[releaseResources],
|
||||||
@@ -500,9 +625,10 @@ export function useVoicePreview(
|
|||||||
pc.addTransceiver("audio", { direction: "recvonly" });
|
pc.addTransceiver("audio", { direction: "recvonly" });
|
||||||
}
|
}
|
||||||
if (options.videoStream) {
|
if (options.videoStream) {
|
||||||
options.videoStream
|
for (const track of options.videoStream.getVideoTracks()) {
|
||||||
.getVideoTracks()
|
const sender = pc.addTrack(track, options.videoStream);
|
||||||
.forEach((track) => pc.addTrack(track, options.videoStream!));
|
await setVideoBitrate(sender, INITIAL_VIDEO_BITRATE);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4) 生成 offer 并发给后端(assistant_id 在 payload 顶层)
|
// 4) 生成 offer 并发给后端(assistant_id 在 payload 顶层)
|
||||||
@@ -531,6 +657,51 @@ export function useVoicePreview(
|
|||||||
}
|
}
|
||||||
}, [assistantId, applyOutputDevice, fail, closeOnRemoteEnd, refreshDevices]);
|
}, [assistantId, applyOutputDevice, fail, closeOnRemoteEnd, refreshDevices]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (status !== "connected") return;
|
||||||
|
const pc = pcRef.current;
|
||||||
|
if (!pc) return;
|
||||||
|
|
||||||
|
let cancelled = false;
|
||||||
|
const sampleNetwork = async () => {
|
||||||
|
try {
|
||||||
|
const metrics = readVideoNetworkMetrics(await pc.getStats());
|
||||||
|
if (cancelled || pcRef.current !== pc) return;
|
||||||
|
const quality = classifyNetworkQuality(metrics, networkStatsRef.current);
|
||||||
|
if (metrics.packetsSent !== null && metrics.packetsLost !== null) {
|
||||||
|
networkStatsRef.current = {
|
||||||
|
packetsSent: metrics.packetsSent,
|
||||||
|
packetsLost: metrics.packetsLost,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
setNetworkQuality(quality);
|
||||||
|
|
||||||
|
if (quality === "unknown") return;
|
||||||
|
const bitrate = VIDEO_BITRATE_BY_QUALITY[quality];
|
||||||
|
if (bitrate === videoBitrateRef.current) return;
|
||||||
|
const sender = pc
|
||||||
|
.getSenders()
|
||||||
|
.find((item) => item.track?.kind === "video");
|
||||||
|
if (!sender) return;
|
||||||
|
if (await setVideoBitrate(sender, bitrate)) {
|
||||||
|
videoBitrateRef.current = bitrate;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// 个别浏览器不提供完整 WebRTC 统计信息时,保留底层拥塞控制。
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
void sampleNetwork();
|
||||||
|
const interval = window.setInterval(
|
||||||
|
() => void sampleNetwork(),
|
||||||
|
NETWORK_SAMPLE_INTERVAL_MS,
|
||||||
|
);
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
window.clearInterval(interval);
|
||||||
|
};
|
||||||
|
}, [status]);
|
||||||
|
|
||||||
const replaceVideoStream = useCallback(
|
const replaceVideoStream = useCallback(
|
||||||
async (videoStream: MediaStream | null) => {
|
async (videoStream: MediaStream | null) => {
|
||||||
const pc = pcRef.current;
|
const pc = pcRef.current;
|
||||||
@@ -621,6 +792,7 @@ export function useVoicePreview(
|
|||||||
localStream,
|
localStream,
|
||||||
remoteStream,
|
remoteStream,
|
||||||
messages,
|
messages,
|
||||||
|
networkQuality,
|
||||||
audioInputs,
|
audioInputs,
|
||||||
audioOutputs,
|
audioOutputs,
|
||||||
selectedDeviceId,
|
selectedDeviceId,
|
||||||
|
|||||||
Reference in New Issue
Block a user