Add Network Quality Indicator component and integrate into Assistant and Mobile Call pages
- Introduce a new `NetworkQualityIndicator` component to visually represent network quality status during calls. - Integrate the `NetworkQualityIndicator` into `AssistantPage` and `MobileCallPage`, enhancing user awareness of connection quality. - Update `use-voice-preview` hook to manage network quality metrics, improving overall call experience and performance monitoring.
This commit is contained in:
70
frontend/src/components/network-quality-indicator.tsx
Normal file
70
frontend/src/components/network-quality-indicator.tsx
Normal file
@@ -0,0 +1,70 @@
|
||||
"use client";
|
||||
|
||||
import { Wifi, WifiOff } from "lucide-react";
|
||||
|
||||
import type {
|
||||
NetworkQuality,
|
||||
VoicePreviewStatus,
|
||||
} from "@/hooks/use-voice-preview";
|
||||
|
||||
const QUALITY_VIEW: Record<
|
||||
NetworkQuality,
|
||||
{ label: string; color: string; darkColor: string }
|
||||
> = {
|
||||
unknown: {
|
||||
label: "检测中",
|
||||
color: "text-muted-foreground",
|
||||
darkColor: "text-white/55",
|
||||
},
|
||||
good: {
|
||||
label: "网络良好",
|
||||
color: "text-emerald-600",
|
||||
darkColor: "text-emerald-300",
|
||||
},
|
||||
fair: {
|
||||
label: "网络一般",
|
||||
color: "text-amber-600",
|
||||
darkColor: "text-amber-300",
|
||||
},
|
||||
poor: {
|
||||
label: "网络较差",
|
||||
color: "text-red-600",
|
||||
darkColor: "text-red-300",
|
||||
},
|
||||
};
|
||||
|
||||
export function NetworkQualityIndicator({
|
||||
quality,
|
||||
status,
|
||||
dark = false,
|
||||
className = "",
|
||||
}: {
|
||||
quality: NetworkQuality;
|
||||
status: VoicePreviewStatus;
|
||||
dark?: boolean;
|
||||
className?: string;
|
||||
}) {
|
||||
const connected = status === "connected";
|
||||
const qualityView = QUALITY_VIEW[quality];
|
||||
const view = connected
|
||||
? {
|
||||
label: qualityView.label,
|
||||
color: dark ? qualityView.darkColor : qualityView.color,
|
||||
}
|
||||
: {
|
||||
label: status === "connecting" ? "检测中" : "未连接",
|
||||
color: dark ? "text-white/55" : "text-muted-foreground",
|
||||
};
|
||||
const Icon = connected || status === "connecting" ? Wifi : WifiOff;
|
||||
|
||||
return (
|
||||
<span
|
||||
className={`inline-flex items-center gap-1.5 whitespace-nowrap text-xs ${view.color} ${className}`}
|
||||
title={view.label}
|
||||
aria-label={view.label}
|
||||
>
|
||||
<Icon className="size-3.5" aria-hidden="true" />
|
||||
<span>{view.label}</span>
|
||||
</span>
|
||||
);
|
||||
}
|
||||
@@ -40,6 +40,7 @@ import {
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { NetworkQualityIndicator } from "@/components/network-quality-indicator";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
@@ -2037,6 +2038,10 @@ function DebugDrawer({
|
||||
<div className="shrink-0 text-sm font-medium text-foreground">
|
||||
调试与预览
|
||||
</div>
|
||||
<NetworkQualityIndicator
|
||||
quality={preview.networkQuality}
|
||||
status={preview.status}
|
||||
/>
|
||||
<DebugConnectionStatus
|
||||
status={preview.status}
|
||||
micWarning={preview.micWarning}
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
Video,
|
||||
} from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { NetworkQualityIndicator } from "@/components/network-quality-indicator";
|
||||
import {
|
||||
DropdownMenu,
|
||||
DropdownMenuContent,
|
||||
@@ -247,6 +248,7 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
|
||||
selectOutputDevice,
|
||||
supportsOutputSelection,
|
||||
messages,
|
||||
networkQuality,
|
||||
connect,
|
||||
replaceVideoStream,
|
||||
disconnect,
|
||||
@@ -325,6 +327,13 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
|
||||
|
||||
{view === "chat" && <MobileCallTranscript messages={messages} />}
|
||||
|
||||
<NetworkQualityIndicator
|
||||
quality={networkQuality}
|
||||
status={status}
|
||||
dark
|
||||
className="absolute left-4 top-[max(1rem,env(safe-area-inset-top))] z-10 rounded-full border border-white/10 bg-black/30 px-3 py-1.5 backdrop-blur-md"
|
||||
/>
|
||||
|
||||
<div className="absolute left-1/2 top-[max(1rem,env(safe-area-inset-top))] z-10 flex -translate-x-1/2 items-center gap-2 rounded-full border border-white/10 bg-black/30 px-3 py-1.5 text-xs text-white/85 backdrop-blur-md">
|
||||
<span
|
||||
className={[
|
||||
|
||||
@@ -9,6 +9,12 @@
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
|
||||
const CAMERA_VIDEO_CONSTRAINTS: MediaTrackConstraints = {
|
||||
width: { ideal: 640, max: 640 },
|
||||
height: { ideal: 360, max: 480 },
|
||||
frameRate: { ideal: 12, max: 15 },
|
||||
};
|
||||
|
||||
function cameraErrorMessage(error: unknown): string {
|
||||
if (error instanceof DOMException) {
|
||||
if (error.name === "NotAllowedError")
|
||||
@@ -59,8 +65,12 @@ export function useCameraPreview() {
|
||||
setStarting(true);
|
||||
try {
|
||||
const id = deviceIdRef.current;
|
||||
const videoConstraints: MediaTrackConstraints = {
|
||||
...CAMERA_VIDEO_CONSTRAINTS,
|
||||
...(id ? { deviceId: { exact: id } } : {}),
|
||||
};
|
||||
const media = await navigator.mediaDevices.getUserMedia({
|
||||
video: id ? { deviceId: { exact: id } } : true,
|
||||
video: videoConstraints,
|
||||
});
|
||||
streamRef.current = media;
|
||||
setStream(media);
|
||||
|
||||
@@ -22,6 +22,30 @@ import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { API_BASE, webrtcApi } from "@/lib/api";
|
||||
|
||||
export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed";
|
||||
export type NetworkQuality = "unknown" | "good" | "fair" | "poor";
|
||||
|
||||
const NETWORK_SAMPLE_INTERVAL_MS = 2_000;
|
||||
const VIDEO_BITRATE_BY_QUALITY: Record<
|
||||
Exclude<NetworkQuality, "unknown">,
|
||||
number
|
||||
> = {
|
||||
good: 600_000,
|
||||
fair: 500_000,
|
||||
poor: 300_000,
|
||||
};
|
||||
const INITIAL_VIDEO_BITRATE = VIDEO_BITRATE_BY_QUALITY.fair;
|
||||
|
||||
type NetworkStatsSample = {
|
||||
packetsSent: number;
|
||||
packetsLost: number;
|
||||
};
|
||||
|
||||
type VideoNetworkMetrics = {
|
||||
packetsSent: number | null;
|
||||
packetsLost: number | null;
|
||||
roundTripTime: number | null;
|
||||
availableOutgoingBitrate: number | null;
|
||||
};
|
||||
|
||||
type ConnectOptions = {
|
||||
visionEnabled?: boolean;
|
||||
@@ -88,6 +112,97 @@ function microphoneErrorMessage(error: unknown): string {
|
||||
return errorMessage(error, "无法访问麦克风。");
|
||||
}
|
||||
|
||||
async function setVideoBitrate(
|
||||
sender: RTCRtpSender,
|
||||
bitrate: number,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const parameters = sender.getParameters();
|
||||
if (!parameters.encodings.length) parameters.encodings = [{}];
|
||||
parameters.encodings[0].maxBitrate = bitrate;
|
||||
parameters.degradationPreference = "maintain-resolution";
|
||||
await sender.setParameters(parameters);
|
||||
return true;
|
||||
} catch {
|
||||
// 不支持发送参数的浏览器继续使用 WebRTC 自带的拥塞控制。
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readVideoNetworkMetrics(report: RTCStatsReport): VideoNetworkMetrics {
|
||||
const metrics: VideoNetworkMetrics = {
|
||||
packetsSent: null,
|
||||
packetsLost: null,
|
||||
roundTripTime: null,
|
||||
availableOutgoingBitrate: null,
|
||||
};
|
||||
|
||||
report.forEach((stat) => {
|
||||
const values = stat as RTCStats & Record<string, unknown>;
|
||||
const kind = values.kind ?? values.mediaType;
|
||||
if (values.type === "outbound-rtp" && kind === "video") {
|
||||
if (typeof values.packetsSent === "number") {
|
||||
metrics.packetsSent = values.packetsSent;
|
||||
}
|
||||
} else if (values.type === "remote-inbound-rtp" && kind === "video") {
|
||||
if (typeof values.packetsLost === "number") {
|
||||
metrics.packetsLost = values.packetsLost;
|
||||
}
|
||||
if (typeof values.roundTripTime === "number") {
|
||||
metrics.roundTripTime = values.roundTripTime;
|
||||
}
|
||||
} else if (
|
||||
values.type === "candidate-pair" &&
|
||||
values.state === "succeeded" &&
|
||||
(values.nominated === true || values.selected === true)
|
||||
) {
|
||||
if (typeof values.currentRoundTripTime === "number") {
|
||||
metrics.roundTripTime = values.currentRoundTripTime;
|
||||
}
|
||||
if (typeof values.availableOutgoingBitrate === "number") {
|
||||
metrics.availableOutgoingBitrate = values.availableOutgoingBitrate;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
||||
function classifyNetworkQuality(
|
||||
metrics: VideoNetworkMetrics,
|
||||
previous: NetworkStatsSample | null,
|
||||
): NetworkQuality {
|
||||
const hasPacketSample =
|
||||
previous !== null &&
|
||||
metrics.packetsSent !== null &&
|
||||
metrics.packetsLost !== null;
|
||||
const sentDelta = hasPacketSample
|
||||
? Math.max(0, metrics.packetsSent! - previous.packetsSent)
|
||||
: 0;
|
||||
const lostDelta = hasPacketSample
|
||||
? Math.max(0, metrics.packetsLost! - previous.packetsLost)
|
||||
: 0;
|
||||
const lossRate = sentDelta > 0 ? lostDelta / sentDelta : null;
|
||||
const { roundTripTime: rtt, availableOutgoingBitrate: bandwidth } = metrics;
|
||||
|
||||
if (lossRate === null && rtt === null && bandwidth === null) return "unknown";
|
||||
if (
|
||||
(lossRate !== null && lossRate >= 0.08) ||
|
||||
(rtt !== null && rtt >= 0.45) ||
|
||||
(bandwidth !== null && bandwidth < 350_000)
|
||||
) {
|
||||
return "poor";
|
||||
}
|
||||
if (
|
||||
(lossRate !== null && lossRate >= 0.03) ||
|
||||
(rtt !== null && rtt >= 0.25) ||
|
||||
(bandwidth !== null && bandwidth < 550_000)
|
||||
) {
|
||||
return "fair";
|
||||
}
|
||||
return "good";
|
||||
}
|
||||
|
||||
export function useVoicePreview(
|
||||
assistantId: string | null,
|
||||
onNodeActive?: (nodeId: string | null) => void,
|
||||
@@ -99,6 +214,8 @@ export function useVoicePreview(
|
||||
// 远端(助手 TTS)媒体流:除挂到 <audio> 播放外,也暴露给波形可视化
|
||||
const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
|
||||
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
||||
const [networkQuality, setNetworkQuality] =
|
||||
useState<NetworkQuality>("unknown");
|
||||
// 可选麦克风/扬声器列表与当前选择(空串表示交给浏览器选默认设备)
|
||||
const [audioInputs, setAudioInputs] = useState<MediaDeviceInfo[]>([]);
|
||||
const [audioOutputs, setAudioOutputs] = useState<MediaDeviceInfo[]>([]);
|
||||
@@ -114,6 +231,8 @@ export function useVoicePreview(
|
||||
const localStreamRef = useRef<MediaStream | null>(null);
|
||||
const startingRef = useRef(false);
|
||||
const messageSeqRef = useRef(0);
|
||||
const networkStatsRef = useRef<NetworkStatsSample | null>(null);
|
||||
const videoBitrateRef = useRef(INITIAL_VIDEO_BITRATE);
|
||||
// 后端主动结束(工作流走到结束节点)标记:据此把随后的断开当作正常结束而非报错
|
||||
const endedByServerRef = useRef(false);
|
||||
// 工作流激活节点回调存进 ref,避免把它挂进 connect 依赖反复重建连接
|
||||
@@ -233,6 +352,9 @@ export function useVoicePreview(
|
||||
messageSeqRef.current = 0;
|
||||
setError(null);
|
||||
setMicWarning(null);
|
||||
setNetworkQuality("unknown");
|
||||
networkStatsRef.current = null;
|
||||
videoBitrateRef.current = INITIAL_VIDEO_BITRATE;
|
||||
setStatus("idle");
|
||||
}, [releaseResources]);
|
||||
|
||||
@@ -242,6 +364,9 @@ export function useVoicePreview(
|
||||
setLocalStream(null);
|
||||
setRemoteStream(null);
|
||||
setError(message);
|
||||
setNetworkQuality("unknown");
|
||||
networkStatsRef.current = null;
|
||||
videoBitrateRef.current = INITIAL_VIDEO_BITRATE;
|
||||
setStatus("failed");
|
||||
},
|
||||
[releaseResources],
|
||||
@@ -500,9 +625,10 @@ export function useVoicePreview(
|
||||
pc.addTransceiver("audio", { direction: "recvonly" });
|
||||
}
|
||||
if (options.videoStream) {
|
||||
options.videoStream
|
||||
.getVideoTracks()
|
||||
.forEach((track) => pc.addTrack(track, options.videoStream!));
|
||||
for (const track of options.videoStream.getVideoTracks()) {
|
||||
const sender = pc.addTrack(track, options.videoStream);
|
||||
await setVideoBitrate(sender, INITIAL_VIDEO_BITRATE);
|
||||
}
|
||||
}
|
||||
|
||||
// 4) 生成 offer 并发给后端(assistant_id 在 payload 顶层)
|
||||
@@ -531,6 +657,51 @@ export function useVoicePreview(
|
||||
}
|
||||
}, [assistantId, applyOutputDevice, fail, closeOnRemoteEnd, refreshDevices]);
|
||||
|
||||
useEffect(() => {
|
||||
if (status !== "connected") return;
|
||||
const pc = pcRef.current;
|
||||
if (!pc) return;
|
||||
|
||||
let cancelled = false;
|
||||
const sampleNetwork = async () => {
|
||||
try {
|
||||
const metrics = readVideoNetworkMetrics(await pc.getStats());
|
||||
if (cancelled || pcRef.current !== pc) return;
|
||||
const quality = classifyNetworkQuality(metrics, networkStatsRef.current);
|
||||
if (metrics.packetsSent !== null && metrics.packetsLost !== null) {
|
||||
networkStatsRef.current = {
|
||||
packetsSent: metrics.packetsSent,
|
||||
packetsLost: metrics.packetsLost,
|
||||
};
|
||||
}
|
||||
setNetworkQuality(quality);
|
||||
|
||||
if (quality === "unknown") return;
|
||||
const bitrate = VIDEO_BITRATE_BY_QUALITY[quality];
|
||||
if (bitrate === videoBitrateRef.current) return;
|
||||
const sender = pc
|
||||
.getSenders()
|
||||
.find((item) => item.track?.kind === "video");
|
||||
if (!sender) return;
|
||||
if (await setVideoBitrate(sender, bitrate)) {
|
||||
videoBitrateRef.current = bitrate;
|
||||
}
|
||||
} catch {
|
||||
// 个别浏览器不提供完整 WebRTC 统计信息时,保留底层拥塞控制。
|
||||
}
|
||||
};
|
||||
|
||||
void sampleNetwork();
|
||||
const interval = window.setInterval(
|
||||
() => void sampleNetwork(),
|
||||
NETWORK_SAMPLE_INTERVAL_MS,
|
||||
);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
window.clearInterval(interval);
|
||||
};
|
||||
}, [status]);
|
||||
|
||||
const replaceVideoStream = useCallback(
|
||||
async (videoStream: MediaStream | null) => {
|
||||
const pc = pcRef.current;
|
||||
@@ -621,6 +792,7 @@ export function useVoicePreview(
|
||||
localStream,
|
||||
remoteStream,
|
||||
messages,
|
||||
networkQuality,
|
||||
audioInputs,
|
||||
audioOutputs,
|
||||
selectedDeviceId,
|
||||
|
||||
Reference in New Issue
Block a user