Enhance MobileCallPage with video functionality and improved view management
- Update CallView type from "camera" to "video" for clarity. - Introduce MobileVideoSurface component to handle video stream rendering. - Implement MobileCallVisualWorkspace for managing chat and video views with drag-and-drop functionality for the video panel. - Refactor state management to streamline view transitions between chat and video modes. - Enhance network metrics handling in useVoicePreview for better performance monitoring during calls.
This commit is contained in:
Xin Wang
@@ -17,6 +17,18 @@ import { API_BASE, webrtcApi } from "@/lib/api";
|
||||
export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed";
|
||||
export type NetworkQuality = "unknown" | "good" | "fair" | "poor";
|
||||
|
||||
const NETWORK_SAMPLE_INTERVAL_MS = 2_000;
|
||||
|
||||
type NetworkStatsSample = {
|
||||
packetsSent: number;
|
||||
packetsLost: number;
|
||||
};
|
||||
|
||||
type NetworkMetrics = NetworkStatsSample & {
|
||||
roundTripTime: number | null;
|
||||
availableOutgoingBitrate: number | null;
|
||||
};
|
||||
|
||||
type ConnectOptions = {
|
||||
visionEnabled?: boolean;
|
||||
videoStream?: MediaStream | null;
|
||||
@@ -54,6 +66,13 @@ class AppSmallWebRTCTransport extends SmallWebRTCTransport {
|
||||
sendAppMessage(message: AppMessage): void {
|
||||
super.sendMessage(message as unknown as RTVIMessage);
|
||||
}
|
||||
|
||||
async connectionStats(): Promise<RTCStatsReport | null> {
|
||||
const peerConnection = (
|
||||
this as unknown as { pc: RTCPeerConnection | null }
|
||||
).pc;
|
||||
return peerConnection ? peerConnection.getStats() : null;
|
||||
}
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown, fallback: string): string {
|
||||
@@ -81,6 +100,80 @@ function encodeHeaderJson(value: unknown): string {
|
||||
return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
||||
}
|
||||
|
||||
function readNetworkMetrics(report: RTCStatsReport): NetworkMetrics | null {
|
||||
const outbound: Partial<Record<"audio" | "video", number>> = {};
|
||||
const lost: Partial<Record<"audio" | "video", number>> = {};
|
||||
let roundTripTime: number | null = null;
|
||||
let availableOutgoingBitrate: number | null = null;
|
||||
|
||||
report.forEach((stat) => {
|
||||
const value = stat as RTCStats & Record<string, unknown>;
|
||||
const kind = value.kind ?? value.mediaType;
|
||||
if (
|
||||
value.type === "outbound-rtp" &&
|
||||
(kind === "audio" || kind === "video") &&
|
||||
typeof value.packetsSent === "number"
|
||||
) {
|
||||
outbound[kind] = value.packetsSent;
|
||||
} else if (
|
||||
value.type === "remote-inbound-rtp" &&
|
||||
(kind === "audio" || kind === "video")
|
||||
) {
|
||||
if (typeof value.packetsLost === "number") lost[kind] = value.packetsLost;
|
||||
if (typeof value.roundTripTime === "number") {
|
||||
roundTripTime = value.roundTripTime;
|
||||
}
|
||||
} else if (
|
||||
value.type === "candidate-pair" &&
|
||||
value.state === "succeeded" &&
|
||||
(value.nominated === true || value.selected === true)
|
||||
) {
|
||||
if (typeof value.currentRoundTripTime === "number") {
|
||||
roundTripTime = value.currentRoundTripTime;
|
||||
}
|
||||
if (typeof value.availableOutgoingBitrate === "number") {
|
||||
availableOutgoingBitrate = value.availableOutgoingBitrate;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const kind = outbound.video !== undefined ? "video" : "audio";
|
||||
if (outbound[kind] === undefined && roundTripTime === null) return null;
|
||||
return {
|
||||
packetsSent: outbound[kind] ?? 0,
|
||||
packetsLost: lost[kind] ?? 0,
|
||||
roundTripTime,
|
||||
availableOutgoingBitrate,
|
||||
};
|
||||
}
|
||||
|
||||
function classifyNetworkQuality(
|
||||
metrics: NetworkMetrics,
|
||||
previous: NetworkStatsSample | null,
|
||||
): NetworkQuality {
|
||||
const sentDelta = previous
|
||||
? Math.max(0, metrics.packetsSent - previous.packetsSent)
|
||||
: 0;
|
||||
const lostDelta = previous
|
||||
? Math.max(0, metrics.packetsLost - previous.packetsLost)
|
||||
: 0;
|
||||
const lossRate = sentDelta > 0 ? lostDelta / sentDelta : null;
|
||||
const { roundTripTime: rtt, availableOutgoingBitrate: bandwidth } = metrics;
|
||||
|
||||
if (lossRate === null && rtt === null && bandwidth === null) return "unknown";
|
||||
if (
|
||||
(lossRate !== null && lossRate >= 0.08) ||
|
||||
(rtt !== null && rtt >= 0.45) ||
|
||||
(bandwidth !== null && bandwidth < 350_000)
|
||||
) return "poor";
|
||||
if (
|
||||
(lossRate !== null && lossRate >= 0.03) ||
|
||||
(rtt !== null && rtt >= 0.25) ||
|
||||
(bandwidth !== null && bandwidth < 550_000)
|
||||
) return "fair";
|
||||
return "good";
|
||||
}
|
||||
|
||||
export function useVoicePreview(
|
||||
assistantId: string | null,
|
||||
onNodeActive?: (nodeId: string | null) => void,
|
||||
@@ -103,6 +196,10 @@ export function useVoicePreview(
|
||||
const transportRef = useRef<AppSmallWebRTCTransport | null>(null);
|
||||
const startingRef = useRef(false);
|
||||
const messageSeqRef = useRef(0);
|
||||
const networkStatsRef = useRef<NetworkStatsSample | null>(null);
|
||||
const pendingAssistantTurnsRef = useRef(
|
||||
new Map<string, { timestamp: string }>(),
|
||||
);
|
||||
const endedByServerRef = useRef(false);
|
||||
const selectedDeviceIdRef = useRef("");
|
||||
const selectedOutputDeviceIdRef = useRef("");
|
||||
@@ -156,6 +253,8 @@ export function useVoicePreview(
|
||||
transport?.disconnect().catch(() => {});
|
||||
if (audioRef.current) audioRef.current.srcObject = null;
|
||||
startingRef.current = false;
|
||||
pendingAssistantTurnsRef.current.clear();
|
||||
networkStatsRef.current = null;
|
||||
onNodeActiveRef.current?.(null);
|
||||
}, []);
|
||||
|
||||
@@ -166,6 +265,7 @@ export function useVoicePreview(
|
||||
setRemoteStream(null);
|
||||
setMessages([]);
|
||||
messageSeqRef.current = 0;
|
||||
pendingAssistantTurnsRef.current.clear();
|
||||
setError(null);
|
||||
setMicWarning(null);
|
||||
setNetworkQuality("unknown");
|
||||
@@ -187,52 +287,82 @@ export function useVoicePreview(
|
||||
msg.type === "assistant-text-start" &&
|
||||
typeof msg.turn_id === "string"
|
||||
) {
|
||||
messageSeqRef.current += 1;
|
||||
setMessages((previous) =>
|
||||
sortMessages([
|
||||
...previous,
|
||||
{
|
||||
id: `assistant-${msg.turn_id as string}`,
|
||||
role: "assistant",
|
||||
content: "",
|
||||
timestamp:
|
||||
typeof msg.timestamp === "string"
|
||||
? msg.timestamp
|
||||
: new Date().toISOString(),
|
||||
sequence: messageSeqRef.current,
|
||||
turnId: msg.turn_id as string,
|
||||
streaming: true,
|
||||
},
|
||||
]),
|
||||
);
|
||||
pendingAssistantTurnsRef.current.set(msg.turn_id, {
|
||||
timestamp:
|
||||
typeof msg.timestamp === "string"
|
||||
? msg.timestamp
|
||||
: new Date().toISOString(),
|
||||
});
|
||||
} else if (
|
||||
msg.type === "assistant-text-delta" &&
|
||||
typeof msg.turn_id === "string" &&
|
||||
typeof msg.delta === "string"
|
||||
typeof msg.delta === "string" &&
|
||||
msg.delta.length > 0
|
||||
) {
|
||||
setMessages((previous) =>
|
||||
previous.map((message) =>
|
||||
message.turnId === msg.turn_id
|
||||
? { ...message, content: message.content + msg.delta }
|
||||
: message,
|
||||
),
|
||||
);
|
||||
const pending = pendingAssistantTurnsRef.current.get(msg.turn_id);
|
||||
if (pending) {
|
||||
pendingAssistantTurnsRef.current.delete(msg.turn_id);
|
||||
messageSeqRef.current += 1;
|
||||
const sequence = messageSeqRef.current;
|
||||
setMessages((previous) =>
|
||||
sortMessages([
|
||||
...previous,
|
||||
{
|
||||
id: `assistant-${msg.turn_id as string}`,
|
||||
role: "assistant",
|
||||
content: msg.delta as string,
|
||||
timestamp: pending.timestamp,
|
||||
sequence,
|
||||
turnId: msg.turn_id as string,
|
||||
streaming: true,
|
||||
},
|
||||
]),
|
||||
);
|
||||
} else {
|
||||
setMessages((previous) =>
|
||||
previous.map((message) =>
|
||||
message.turnId === msg.turn_id
|
||||
? { ...message, content: message.content + msg.delta }
|
||||
: message,
|
||||
),
|
||||
);
|
||||
}
|
||||
} else if (
|
||||
msg.type === "assistant-text-end" &&
|
||||
typeof msg.turn_id === "string"
|
||||
) {
|
||||
setMessages((previous) =>
|
||||
previous.map((message) =>
|
||||
message.turnId === msg.turn_id
|
||||
? {
|
||||
...message,
|
||||
content:
|
||||
typeof msg.content === "string" ? msg.content : message.content,
|
||||
streaming: false,
|
||||
}
|
||||
: message,
|
||||
),
|
||||
);
|
||||
const pending = pendingAssistantTurnsRef.current.get(msg.turn_id);
|
||||
pendingAssistantTurnsRef.current.delete(msg.turn_id);
|
||||
const finalContent =
|
||||
typeof msg.content === "string" ? msg.content.trim() : "";
|
||||
setMessages((previous) => {
|
||||
const existing = previous.some((message) => message.turnId === msg.turn_id);
|
||||
if (existing) {
|
||||
return previous.map((message) =>
|
||||
message.turnId === msg.turn_id
|
||||
? {
|
||||
...message,
|
||||
content: finalContent || message.content,
|
||||
streaming: false,
|
||||
}
|
||||
: message,
|
||||
);
|
||||
}
|
||||
if (!finalContent) return previous;
|
||||
messageSeqRef.current += 1;
|
||||
return sortMessages([
|
||||
...previous,
|
||||
{
|
||||
id: `assistant-${msg.turn_id as string}`,
|
||||
role: "assistant",
|
||||
content: finalContent,
|
||||
timestamp: pending?.timestamp ?? new Date().toISOString(),
|
||||
sequence: messageSeqRef.current,
|
||||
turnId: msg.turn_id as string,
|
||||
streaming: false,
|
||||
},
|
||||
]);
|
||||
});
|
||||
} else if (
|
||||
msg.type === "transcript" &&
|
||||
(msg.role === "user" || msg.role === "assistant") &&
|
||||
@@ -277,6 +407,7 @@ export function useVoicePreview(
|
||||
setError(null);
|
||||
setMicWarning(null);
|
||||
setMessages([]);
|
||||
pendingAssistantTurnsRef.current.clear();
|
||||
setCallEnded(false);
|
||||
endedByServerRef.current = false;
|
||||
|
||||
@@ -411,6 +542,39 @@ export function useVoicePreview(
|
||||
void applyOutputDevice(deviceId);
|
||||
}, [applyOutputDevice]);
|
||||
|
||||
useEffect(() => {
|
||||
if (status !== "connected") return;
|
||||
let cancelled = false;
|
||||
const sample = async () => {
|
||||
const transport = transportRef.current;
|
||||
if (!transport) return;
|
||||
try {
|
||||
const report = await transport.connectionStats();
|
||||
const metrics = report ? readNetworkMetrics(report) : null;
|
||||
if (cancelled || transportRef.current !== transport) return;
|
||||
if (!metrics) {
|
||||
setNetworkQuality("unknown");
|
||||
return;
|
||||
}
|
||||
setNetworkQuality(
|
||||
classifyNetworkQuality(metrics, networkStatsRef.current),
|
||||
);
|
||||
networkStatsRef.current = {
|
||||
packetsSent: metrics.packetsSent,
|
||||
packetsLost: metrics.packetsLost,
|
||||
};
|
||||
} catch {
|
||||
if (!cancelled) setNetworkQuality("unknown");
|
||||
}
|
||||
};
|
||||
void sample();
|
||||
const interval = window.setInterval(() => void sample(), NETWORK_SAMPLE_INTERVAL_MS);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
window.clearInterval(interval);
|
||||
};
|
||||
}, [status]);
|
||||
|
||||
const sendText = useCallback((text: string): boolean => {
|
||||
const trimmed = text.trim();
|
||||
const transport = transportRef.current;
|
||||
|
||||
Reference in New Issue
Block a user