Enhance MobileCallPage with video functionality and improved view management

- Update CallView type from "camera" to "video" for clarity. - Introduce MobileVideoSurface component to handle video stream rendering. - Implement MobileCallVisualWorkspace for managing chat and video views with drag-and-drop functionality for the video panel. - Refactor state management to streamline view transitions between chat and video modes. - Enhance network metrics handling in useVoicePreview for better performance monitoring during calls.
2026-07-13 08:01:39 +08:00
parent a25b149f65
commit 313387778a
2 changed files with 381 additions and 96 deletions
--- a/frontend/src/components/pages/MobileCallPage.tsx
+++ b/frontend/src/components/pages/MobileCallPage.tsx
@@ -35,7 +35,7 @@ import {
  type ChatMessage,
 } from "@/hooks/use-voice-preview";

-type CallView = "camera" | "chat";
+type CallView = "video" | "chat";

 function messageTime(timestamp: string): string {
  const date = new Date(timestamp);
@@ -106,6 +106,171 @@ function MobileCallTranscript({ messages }: { messages: ChatMessage[] }) {
  );
 }

+function MobileVideoSurface({
+  stream,
+  compact = false,
+}: {
+  stream: MediaStream | null;
+  compact?: boolean;
+}) {
+  const videoRef = useRef<HTMLVideoElement>(null);
+
+  useEffect(() => {
+    if (videoRef.current) videoRef.current.srcObject = stream;
+  }, [stream]);
+
+  return (
+    <video
+      ref={videoRef}
+      autoPlay
+      playsInline
+      muted
+      className={[
+        "h-full w-full -scale-x-100",
+        compact ? "object-cover" : "object-cover lg:object-contain",
+        stream ? "" : "hidden",
+      ].join(" ")}
+    />
+  );
+}
+
+function MobileCallVisualWorkspace({
+  view,
+  onViewChange,
+  messages,
+  videoStream,
+  cameraStarting,
+  cameraError,
+}: {
+  view: CallView;
+  onViewChange: (view: CallView) => void;
+  messages: ChatMessage[];
+  videoStream: MediaStream | null;
+  cameraStarting: boolean;
+  cameraError: string | null;
+}) {
+  const containerRef = useRef<HTMLDivElement>(null);
+  const [position, setPosition] = useState({ x: 16, y: 72 });
+  const positionedRef = useRef(false);
+  const dragRef = useRef<{
+    pointerId: number;
+    startX: number;
+    startY: number;
+    originX: number;
+    originY: number;
+    moved: boolean;
+  } | null>(null);
+  const latestMessage = [...messages]
+    .reverse()
+    .find((message) => message.content.trim());
+
+  useEffect(() => {
+    const container = containerRef.current;
+    if (!container) return;
+    const rect = container.getBoundingClientRect();
+    if (!positionedRef.current) {
+      positionedRef.current = true;
+      setPosition({ x: Math.max(16, rect.width - 168), y: 72 });
+      return;
+    }
+    setPosition((current) => ({
+      x: Math.min(Math.max(16, current.x), Math.max(16, rect.width - 168)),
+      y: Math.min(Math.max(72, current.y), Math.max(72, rect.height - 200)),
+    }));
+  }, [view]);
+
+  const moveFloatingVideo = useCallback((event: React.PointerEvent<HTMLButtonElement>) => {
+    const drag = dragRef.current;
+    const container = containerRef.current;
+    if (!drag || !container || drag.pointerId !== event.pointerId) return;
+    const dx = event.clientX - drag.startX;
+    const dy = event.clientY - drag.startY;
+    if (Math.abs(dx) + Math.abs(dy) > 4) drag.moved = true;
+    const rect = container.getBoundingClientRect();
+    setPosition({
+      x: Math.min(Math.max(16, drag.originX + dx), Math.max(16, rect.width - 168)),
+      y: Math.min(Math.max(72, drag.originY + dy), Math.max(72, rect.height - 200)),
+    });
+  }, []);
+
+  return (
+    <div ref={containerRef} className="absolute inset-0 z-[1] overflow-hidden">
+      {view === "chat" ? (
+        <>
+          <MobileCallTranscript messages={messages} />
+          <button
+            type="button"
+            style={{ left: position.x, top: position.y }}
+            className="absolute z-10 h-24 w-38 touch-none overflow-hidden rounded-2xl border border-white/20 bg-black shadow-2xl ring-1 ring-black/20 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-white/50"
+            aria-label="拖动摄像头浮窗，点击切换到视频流"
+            onPointerDown={(event) => {
+              event.currentTarget.setPointerCapture(event.pointerId);
+              dragRef.current = {
+                pointerId: event.pointerId,
+                startX: event.clientX,
+                startY: event.clientY,
+                originX: position.x,
+                originY: position.y,
+                moved: false,
+              };
+            }}
+            onPointerMove={moveFloatingVideo}
+            onPointerUp={(event) => {
+              const drag = dragRef.current;
+              event.currentTarget.releasePointerCapture(event.pointerId);
+              dragRef.current = null;
+              if (drag && !drag.moved) onViewChange("video");
+            }}
+            onPointerCancel={() => {
+              dragRef.current = null;
+            }}
+          >
+            <MobileVideoSurface stream={videoStream} compact />
+            {!videoStream && (
+              <span className="absolute inset-0 flex items-center justify-center text-white/60">
+                {cameraStarting ? <Loader2 className="size-5 animate-spin" /> : <Video className="size-5" />}
+              </span>
+            )}
+          </button>
+        </>
+      ) : (
+        <>
+          <MobileVideoSurface stream={videoStream} />
+          {!videoStream && (
+            <div className="absolute inset-0 flex items-center justify-center px-8 text-center">
+              <div className="flex max-w-sm flex-col items-center gap-3">
+                {cameraStarting ? (
+                  <Loader2 size={34} className="animate-spin text-white/70" />
+                ) : (
+                  <Video size={34} className="text-white/60" />
+                )}
+                <p className="text-sm leading-6 text-white/75">
+                  {cameraStarting ? "正在开启摄像头…" : cameraError || "等待摄像头画面…"}
+                </p>
+              </div>
+            </div>
+          )}
+          <button
+            type="button"
+            onClick={() => onViewChange("chat")}
+            className="absolute left-4 right-4 top-[max(4.5rem,calc(env(safe-area-inset-top)+4.5rem))] z-10 flex h-16 items-center gap-3 overflow-hidden rounded-2xl border border-white/15 bg-[#07101a]/55 px-4 py-2 text-left shadow-xl backdrop-blur-md focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-white/50"
+            aria-label="切换到聊天记录"
+          >
+            <MessageSquareText className="size-[17px] shrink-0 text-white/65" />
+            <span className="line-clamp-2 min-w-0 flex-1 text-xs leading-5 text-white/90">
+              <span className="mr-1.5 font-medium text-white/55">
+                {latestMessage?.role === "user" ? "我：" : "助手："}
+              </span>
+              {latestMessage?.content || "暂无消息，点击返回聊天记录"}
+            </span>
+          </button>
+        </>
+      )}
+      <div className="pointer-events-none absolute inset-0 z-[2] bg-gradient-to-b from-black/35 via-transparent to-black/65" />
+    </div>
+  );
+}
+
 function CallAudioDeviceSelect({
  micValue,
  micDevices,
@@ -263,15 +428,10 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
    deviceId: cameraDeviceId,
    selectCamera: changeCamera,
  } = camera;
-  const videoRef = useRef<HTMLVideoElement>(null);
-  const [view, setView] = useState<CallView>("camera");
+  const [view, setView] = useState<CallView>("chat");
  const connecting = status === "connecting";
  const inCall = !callEnded && (connecting || status === "connected");

-  useEffect(() => {
-    if (videoRef.current) videoRef.current.srcObject = videoStream;
-  }, [videoStream]);
-
  const startCall = useCallback(async () => {
    await connect({
      visionEnabled: true,
@@ -279,7 +439,7 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
  }, [connect]);

  const endCall = useCallback(() => {
-    setView("camera");
+    setView("chat");
    disconnect();
  }, [disconnect]);

@@ -305,20 +465,16 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
        data-testid="mobile-call-viewport"
        className="relative isolate h-dvh min-h-80 w-full overflow-hidden bg-[#07101a] text-white lg:h-[calc(100dvh-2rem)] lg:min-h-0 lg:w-auto lg:aspect-[9/16] lg:rounded-[2rem] lg:ring-1 lg:ring-white/15 lg:shadow-2xl"
      >
-      <video
-        ref={videoRef}
-        autoPlay
-        playsInline
-        muted
-        className={[
-          "absolute inset-0 h-full w-full -scale-x-100 object-cover transition-opacity duration-300 lg:object-contain",
-          videoStream && view === "camera" ? "opacity-100" : "opacity-0",
-        ].join(" ")}
-      />
-
-      <div className="pointer-events-none absolute inset-0 bg-gradient-to-b from-black/35 via-transparent to-black/65" />
-
-      {view === "chat" && inCall && <MobileCallTranscript messages={messages} />}
+      {inCall && (
+        <MobileCallVisualWorkspace
+          view={view}
+          onViewChange={setView}
+          messages={messages}
+          videoStream={videoStream}
+          cameraStarting={cameraStarting}
+          cameraError={cameraError}
+        />
+      )}

      <NetworkQualityIndicator
        quality={networkQuality}
@@ -347,48 +503,13 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
              : "通话已结束"}
      </div>

-      {inCall && (
-        <button
-          type="button"
-          onClick={() =>
-            setView((current) => current === "camera" ? "chat" : "camera")
-          }
-          aria-label={view === "camera" ? "显示聊天记录" : "显示摄像头画面"}
-          title={view === "camera" ? "聊天记录" : "摄像头画面"}
-          className="absolute right-4 top-[max(1rem,env(safe-area-inset-top))] z-10 flex size-9 items-center justify-center rounded-full border border-white/15 bg-black/40 text-white shadow-lg backdrop-blur-md transition-colors hover:bg-black/60 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-white/40"
-        >
-          {view === "camera" ? (
-            <MessageSquareText className="size-[18px]" />
-          ) : (
-            <Video className="size-[18px]" />
-          )}
-        </button>
-      )}
-
-      {!videoStream && inCall && view === "camera" && (
-        <div className="absolute inset-0 z-[2] flex items-center justify-center px-8 text-center">
-          <div className="flex max-w-sm flex-col items-center gap-3">
-            {cameraStarting ? (
-              <Loader2 size={34} className="animate-spin text-white/70" />
-            ) : (
-              <Video size={34} className="text-white/60" />
-            )}
-            <p className="text-sm leading-6 text-white/75">
-              {cameraStarting
-                ? "正在开启摄像头…"
-                : cameraError || "等待摄像头画面…"}
-            </p>
-          </div>
-        </div>
-      )}
-
      {!inCall && (
        <div className="absolute inset-0 z-[2] flex items-center justify-center px-8">
          <div className="flex max-w-sm flex-col items-center gap-4 text-center">
            <Button
              type="button"
              onClick={() => {
-                setView("camera");
+                setView("chat");
                void startCall();
              }}
              className="h-12 gap-2 rounded-full bg-white px-6 text-[#07101a] shadow-2xl hover:bg-white/90"
--- a/frontend/src/hooks/use-voice-preview.ts
+++ b/frontend/src/hooks/use-voice-preview.ts
@@ -17,6 +17,18 @@ import { API_BASE, webrtcApi } from "@/lib/api";
 export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed";
 export type NetworkQuality = "unknown" | "good" | "fair" | "poor";

+const NETWORK_SAMPLE_INTERVAL_MS = 2_000;
+
+type NetworkStatsSample = {
+  packetsSent: number;
+  packetsLost: number;
+};
+
+type NetworkMetrics = NetworkStatsSample & {
+  roundTripTime: number | null;
+  availableOutgoingBitrate: number | null;
+};
+
 type ConnectOptions = {
  visionEnabled?: boolean;
  videoStream?: MediaStream | null;
@@ -54,6 +66,13 @@ class AppSmallWebRTCTransport extends SmallWebRTCTransport {
  sendAppMessage(message: AppMessage): void {
    super.sendMessage(message as unknown as RTVIMessage);
  }
+
+  async connectionStats(): Promise<RTCStatsReport | null> {
+    const peerConnection = (
+      this as unknown as { pc: RTCPeerConnection | null }
+    ).pc;
+    return peerConnection ? peerConnection.getStats() : null;
+  }
 }

 function errorMessage(error: unknown, fallback: string): string {
@@ -81,6 +100,80 @@ function encodeHeaderJson(value: unknown): string {
  return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
 }

+function readNetworkMetrics(report: RTCStatsReport): NetworkMetrics | null {
+  const outbound: Partial<Record<"audio" | "video", number>> = {};
+  const lost: Partial<Record<"audio" | "video", number>> = {};
+  let roundTripTime: number | null = null;
+  let availableOutgoingBitrate: number | null = null;
+
+  report.forEach((stat) => {
+    const value = stat as RTCStats & Record<string, unknown>;
+    const kind = value.kind ?? value.mediaType;
+    if (
+      value.type === "outbound-rtp" &&
+      (kind === "audio" || kind === "video") &&
+      typeof value.packetsSent === "number"
+    ) {
+      outbound[kind] = value.packetsSent;
+    } else if (
+      value.type === "remote-inbound-rtp" &&
+      (kind === "audio" || kind === "video")
+    ) {
+      if (typeof value.packetsLost === "number") lost[kind] = value.packetsLost;
+      if (typeof value.roundTripTime === "number") {
+        roundTripTime = value.roundTripTime;
+      }
+    } else if (
+      value.type === "candidate-pair" &&
+      value.state === "succeeded" &&
+      (value.nominated === true || value.selected === true)
+    ) {
+      if (typeof value.currentRoundTripTime === "number") {
+        roundTripTime = value.currentRoundTripTime;
+      }
+      if (typeof value.availableOutgoingBitrate === "number") {
+        availableOutgoingBitrate = value.availableOutgoingBitrate;
+      }
+    }
+  });
+
+  const kind = outbound.video !== undefined ? "video" : "audio";
+  if (outbound[kind] === undefined && roundTripTime === null) return null;
+  return {
+    packetsSent: outbound[kind] ?? 0,
+    packetsLost: lost[kind] ?? 0,
+    roundTripTime,
+    availableOutgoingBitrate,
+  };
+}
+
+function classifyNetworkQuality(
+  metrics: NetworkMetrics,
+  previous: NetworkStatsSample | null,
+): NetworkQuality {
+  const sentDelta = previous
+    ? Math.max(0, metrics.packetsSent - previous.packetsSent)
+    : 0;
+  const lostDelta = previous
+    ? Math.max(0, metrics.packetsLost - previous.packetsLost)
+    : 0;
+  const lossRate = sentDelta > 0 ? lostDelta / sentDelta : null;
+  const { roundTripTime: rtt, availableOutgoingBitrate: bandwidth } = metrics;
+
+  if (lossRate === null && rtt === null && bandwidth === null) return "unknown";
+  if (
+    (lossRate !== null && lossRate >= 0.08) ||
+    (rtt !== null && rtt >= 0.45) ||
+    (bandwidth !== null && bandwidth < 350_000)
+  ) return "poor";
+  if (
+    (lossRate !== null && lossRate >= 0.03) ||
+    (rtt !== null && rtt >= 0.25) ||
+    (bandwidth !== null && bandwidth < 550_000)
+  ) return "fair";
+  return "good";
+}
+
 export function useVoicePreview(
  assistantId: string | null,
  onNodeActive?: (nodeId: string | null) => void,
@@ -103,6 +196,10 @@ export function useVoicePreview(
  const transportRef = useRef<AppSmallWebRTCTransport | null>(null);
  const startingRef = useRef(false);
  const messageSeqRef = useRef(0);
+  const networkStatsRef = useRef<NetworkStatsSample | null>(null);
+  const pendingAssistantTurnsRef = useRef(
+    new Map<string, { timestamp: string }>(),
+  );
  const endedByServerRef = useRef(false);
  const selectedDeviceIdRef = useRef("");
  const selectedOutputDeviceIdRef = useRef("");
@@ -156,6 +253,8 @@ export function useVoicePreview(
    transport?.disconnect().catch(() => {});
    if (audioRef.current) audioRef.current.srcObject = null;
    startingRef.current = false;
+    pendingAssistantTurnsRef.current.clear();
+    networkStatsRef.current = null;
    onNodeActiveRef.current?.(null);
  }, []);

@@ -166,6 +265,7 @@ export function useVoicePreview(
    setRemoteStream(null);
    setMessages([]);
    messageSeqRef.current = 0;
+    pendingAssistantTurnsRef.current.clear();
    setError(null);
    setMicWarning(null);
    setNetworkQuality("unknown");
@@ -187,52 +287,82 @@ export function useVoicePreview(
      msg.type === "assistant-text-start" &&
      typeof msg.turn_id === "string"
    ) {
-      messageSeqRef.current += 1;
-      setMessages((previous) =>
-        sortMessages([
-          ...previous,
-          {
-            id: `assistant-${msg.turn_id as string}`,
-            role: "assistant",
-            content: "",
-            timestamp:
-              typeof msg.timestamp === "string"
-                ? msg.timestamp
-                : new Date().toISOString(),
-            sequence: messageSeqRef.current,
-            turnId: msg.turn_id as string,
-            streaming: true,
-          },
-        ]),
-      );
+      pendingAssistantTurnsRef.current.set(msg.turn_id, {
+        timestamp:
+          typeof msg.timestamp === "string"
+            ? msg.timestamp
+            : new Date().toISOString(),
+      });
    } else if (
      msg.type === "assistant-text-delta" &&
      typeof msg.turn_id === "string" &&
-      typeof msg.delta === "string"
+      typeof msg.delta === "string" &&
+      msg.delta.length > 0
    ) {
-      setMessages((previous) =>
-        previous.map((message) =>
-          message.turnId === msg.turn_id
-            ? { ...message, content: message.content + msg.delta }
-            : message,
-        ),
-      );
+      const pending = pendingAssistantTurnsRef.current.get(msg.turn_id);
+      if (pending) {
+        pendingAssistantTurnsRef.current.delete(msg.turn_id);
+        messageSeqRef.current += 1;
+        const sequence = messageSeqRef.current;
+        setMessages((previous) =>
+          sortMessages([
+            ...previous,
+            {
+              id: `assistant-${msg.turn_id as string}`,
+              role: "assistant",
+              content: msg.delta as string,
+              timestamp: pending.timestamp,
+              sequence,
+              turnId: msg.turn_id as string,
+              streaming: true,
+            },
+          ]),
+        );
+      } else {
+        setMessages((previous) =>
+          previous.map((message) =>
+            message.turnId === msg.turn_id
+              ? { ...message, content: message.content + msg.delta }
+              : message,
+          ),
+        );
+      }
    } else if (
      msg.type === "assistant-text-end" &&
      typeof msg.turn_id === "string"
    ) {
-      setMessages((previous) =>
-        previous.map((message) =>
-          message.turnId === msg.turn_id
-            ? {
-                ...message,
-                content:
-                  typeof msg.content === "string" ? msg.content : message.content,
-                streaming: false,
-              }
-            : message,
-        ),
-      );
+      const pending = pendingAssistantTurnsRef.current.get(msg.turn_id);
+      pendingAssistantTurnsRef.current.delete(msg.turn_id);
+      const finalContent =
+        typeof msg.content === "string" ? msg.content.trim() : "";
+      setMessages((previous) => {
+        const existing = previous.some((message) => message.turnId === msg.turn_id);
+        if (existing) {
+          return previous.map((message) =>
+            message.turnId === msg.turn_id
+              ? {
+                  ...message,
+                  content: finalContent || message.content,
+                  streaming: false,
+                }
+              : message,
+          );
+        }
+        if (!finalContent) return previous;
+        messageSeqRef.current += 1;
+        return sortMessages([
+          ...previous,
+          {
+            id: `assistant-${msg.turn_id as string}`,
+            role: "assistant",
+            content: finalContent,
+            timestamp: pending?.timestamp ?? new Date().toISOString(),
+            sequence: messageSeqRef.current,
+            turnId: msg.turn_id as string,
+            streaming: false,
+          },
+        ]);
+      });
    } else if (
      msg.type === "transcript" &&
      (msg.role === "user" || msg.role === "assistant") &&
@@ -277,6 +407,7 @@ export function useVoicePreview(
    setError(null);
    setMicWarning(null);
    setMessages([]);
+    pendingAssistantTurnsRef.current.clear();
    setCallEnded(false);
    endedByServerRef.current = false;

@@ -411,6 +542,39 @@ export function useVoicePreview(
    void applyOutputDevice(deviceId);
  }, [applyOutputDevice]);

+  useEffect(() => {
+    if (status !== "connected") return;
+    let cancelled = false;
+    const sample = async () => {
+      const transport = transportRef.current;
+      if (!transport) return;
+      try {
+        const report = await transport.connectionStats();
+        const metrics = report ? readNetworkMetrics(report) : null;
+        if (cancelled || transportRef.current !== transport) return;
+        if (!metrics) {
+          setNetworkQuality("unknown");
+          return;
+        }
+        setNetworkQuality(
+          classifyNetworkQuality(metrics, networkStatsRef.current),
+        );
+        networkStatsRef.current = {
+          packetsSent: metrics.packetsSent,
+          packetsLost: metrics.packetsLost,
+        };
+      } catch {
+        if (!cancelled) setNetworkQuality("unknown");
+      }
+    };
+    void sample();
+    const interval = window.setInterval(() => void sample(), NETWORK_SAMPLE_INTERVAL_MS);
+    return () => {
+      cancelled = true;
+      window.clearInterval(interval);
+    };
+  }, [status]);
+
  const sendText = useCallback((text: string): boolean => {
    const trimmed = text.trim();
    const transport = transportRef.current;