From fd57f34a559fd9c15e9d0179f494e1142df1886d Mon Sep 17 00:00:00 2001
From: Xin Wang <your-email@example.com>
Date: Mon, 13 Jul 2026 07:05:06 +0800
Subject: [PATCH] Refactor camera handling and video stream management in
 Assistant and MobileCall pages

- Remove direct camera start/stop calls and replace them with streamlined video stream handling.
- Update DebugVideoPanel to accept a streamOverride for better flexibility in video source management.
- Adjust MobileCallPage to utilize the new videoStream state, enhancing the call experience.
- Introduce videoStream state management in useVoicePreview for improved media handling during voice interactions.
---
 .../src/components/pages/AssistantPage.tsx    | 40 ++++++++-----------
 .../src/components/pages/MobileCallPage.tsx   | 33 +++++----------
 frontend/src/hooks/use-voice-preview.ts       | 15 +++++++
 3 files changed, 43 insertions(+), 45 deletions(-)

diff --git a/frontend/src/components/pages/AssistantPage.tsx b/frontend/src/components/pages/AssistantPage.tsx
index a13ff37..b46fcb5 100644
--- a/frontend/src/components/pages/AssistantPage.tsx
+++ b/frontend/src/components/pages/AssistantPage.tsx
@@ -2155,11 +2155,8 @@ function DebugDrawer({
   const [dynamicVariableValues, setDynamicVariableValues] = useState<
     Record<string, string | number | boolean>
   >({});
-  const stopCamera = camera.stop;
-  const startCamera = camera.start;
   const recording =
     preview.status === "connecting" || preview.status === "connected";
-  const shouldKeepCamera = vision && recording;
   const dynamicVariableEntries = Object.entries(dynamicVariableDefinitions);
   const resolvedDynamicVariables: Record<string, string | number | boolean> = {};
   let dynamicVariablesError = "";
@@ -2174,18 +2171,10 @@ function DebugDrawer({
     resolvedDynamicVariables[name] = value;
   }
 
-  useEffect(() => {
-    if (shouldKeepCamera) {
-      void startCamera();
-    } else {
-      stopCamera();
-    }
-  }, [shouldKeepCamera, startCamera, stopCamera]);
-
   const selectCamera = useCallback(
     async (deviceId: string) => {
-      const nextStream = await camera.selectCamera(deviceId);
-      await preview.replaceVideoStream(nextStream);
+      await camera.selectCamera(deviceId);
+      preview.selectCamera(deviceId);
     },
     [camera, preview],
   );
@@ -2666,15 +2655,12 @@ function DebugVoicePanel({
   const startConversation = useCallback(async () => {
     if (!assistantId || hasUnsavedChanges) return;
     if (dynamicVariablesError) return;
-    const videoStream = vision ? await camera.start() : null;
     await connect({
       visionEnabled: vision,
-      videoStream,
       dynamicVariables,
     });
   }, [
     assistantId,
-    camera,
     connect,
     dynamicVariables,
     dynamicVariablesError,
@@ -2695,7 +2681,7 @@ function DebugVoicePanel({
             connect={startConversation}
           />
         ) : (
-          <DebugVideoPanel camera={camera} />
+          <DebugVideoPanel camera={camera} streamOverride={preview.videoStream} />
         )
       ) : !SHOW_VOICE_VIZ || showTranscript ? (
         showIdleHub ? (
@@ -2955,19 +2941,27 @@ function DebugIdleHub({
   );
 }
 
-function DebugVideoPanel({ camera }: { camera: CameraPreview }) {
+function DebugVideoPanel({
+  camera,
+  streamOverride,
+}: {
+  camera: CameraPreview;
+  streamOverride?: MediaStream | null;
+}) {
   const { stream, error, starting, active } = camera;
+  const effectiveStream = streamOverride ?? stream;
+  const effectiveActive = Boolean(effectiveStream) || active;
   const videoRef = useRef<HTMLVideoElement>(null);
 
   useEffect(() => {
-    if (videoRef.current) videoRef.current.srcObject = stream;
-  }, [stream]);
+    if (videoRef.current) videoRef.current.srcObject = effectiveStream;
+  }, [effectiveStream]);
 
   return (
     <div
       className={[
         "relative flex min-h-0 flex-1 items-center justify-center overflow-hidden",
-        active ? "bg-black" : "bg-canvas-soft",
+        effectiveActive ? "bg-black" : "bg-canvas-soft",
       ].join(" ")}
     >
       <video
@@ -2977,10 +2971,10 @@ function DebugVideoPanel({ camera }: { camera: CameraPreview }) {
         muted
         className={[
           "h-full w-full -scale-x-100 object-contain",
-          active ? "" : "hidden",
+          effectiveActive ? "" : "hidden",
         ].join(" ")}
       />
-      {active ? (
+      {effectiveActive ? (
         <Badge
           variant="secondary"
           className="absolute left-3 top-3 gap-1.5 rounded-full border border-hairline bg-card/80 px-2.5 py-1 text-[11px] font-medium text-muted-foreground shadow-none backdrop-blur"
diff --git a/frontend/src/components/pages/MobileCallPage.tsx b/frontend/src/components/pages/MobileCallPage.tsx
index 7769b95..a81ef20 100644
--- a/frontend/src/components/pages/MobileCallPage.tsx
+++ b/frontend/src/components/pages/MobileCallPage.tsx
@@ -251,18 +251,16 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
     callEnded,
     networkQuality,
     connect,
-    replaceVideoStream,
+    videoStream,
+    selectCamera: selectTransportCamera,
     disconnect,
     audioRef,
   } = preview;
   const {
-    stream: cameraStream,
     error: cameraError,
     starting: cameraStarting,
     devices: cameraDevices,
     deviceId: cameraDeviceId,
-    start: startCamera,
-    stop: stopCamera,
     selectCamera: changeCamera,
   } = camera;
   const videoRef = useRef<HTMLVideoElement>(null);
@@ -271,29 +269,26 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
   const inCall = !callEnded && (connecting || status === "connected");
 
   useEffect(() => {
-    if (videoRef.current) videoRef.current.srcObject = cameraStream;
-  }, [cameraStream]);
+    if (videoRef.current) videoRef.current.srcObject = videoStream;
+  }, [videoStream]);
 
   const startCall = useCallback(async () => {
-    const videoStream = await startCamera();
     await connect({
       visionEnabled: true,
-      videoStream,
     });
-  }, [connect, startCamera]);
+  }, [connect]);
 
   const endCall = useCallback(() => {
     setView("camera");
     disconnect();
-    stopCamera();
-  }, [disconnect, stopCamera]);
+  }, [disconnect]);
 
   const selectCamera = useCallback(
     async (deviceId: string) => {
-      const nextStream = await changeCamera(deviceId);
-      await replaceVideoStream(nextStream);
+      await changeCamera(deviceId);
+      selectTransportCamera(deviceId);
     },
-    [changeCamera, replaceVideoStream],
+    [changeCamera, selectTransportCamera],
   );
 
   useEffect(() => {
@@ -301,12 +296,6 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
     // 首次打开页面时自动发起通话；hooks 会在卸载时各自释放媒体资源。
   }, [startCall]);
 
-  useEffect(() => {
-    if (callEnded || status === "idle" || status === "failed") {
-      stopCamera();
-    }
-  }, [callEnded, status, stopCamera]);
-
   const error = previewError || cameraError;
 
   return (
@@ -323,7 +312,7 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
         muted
         className={[
           "absolute inset-0 h-full w-full -scale-x-100 object-cover transition-opacity duration-300 lg:object-contain",
-          cameraStream && view === "camera" ? "opacity-100" : "opacity-0",
+          videoStream && view === "camera" ? "opacity-100" : "opacity-0",
         ].join(" ")}
       />
 
@@ -376,7 +365,7 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
         </button>
       )}
 
-      {!cameraStream && inCall && view === "camera" && (
+      {!videoStream && inCall && view === "camera" && (
         <div className="absolute inset-0 z-[2] flex items-center justify-center px-8 text-center">
           <div className="flex max-w-sm flex-col items-center gap-3">
             {cameraStarting ? (
diff --git a/frontend/src/hooks/use-voice-preview.ts b/frontend/src/hooks/use-voice-preview.ts
index fafecc0..2d4e761 100644
--- a/frontend/src/hooks/use-voice-preview.ts
+++ b/frontend/src/hooks/use-voice-preview.ts
@@ -89,6 +89,7 @@ export function useVoicePreview(
   const [error, setError] = useState<string | null>(null);
   const [micWarning, setMicWarning] = useState<string | null>(null);
   const [localStream, setLocalStream] = useState<MediaStream | null>(null);
+  const [videoStream, setVideoStream] = useState<MediaStream | null>(null);
   const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
   const [messages, setMessages] = useState<ChatMessage[]>([]);
   const [callEnded, setCallEnded] = useState(false);
@@ -161,6 +162,7 @@ export function useVoicePreview(
   const disconnect = useCallback(() => {
     releaseResources();
     setLocalStream(null);
+    setVideoStream(null);
     setRemoteStream(null);
     setMessages([]);
     messageSeqRef.current = 0;
@@ -173,6 +175,7 @@ export function useVoicePreview(
   const fail = useCallback((message: string) => {
     releaseResources();
     setLocalStream(null);
+    setVideoStream(null);
     setRemoteStream(null);
     setError(message);
     setNetworkQuality("unknown");
@@ -306,6 +309,10 @@ export function useVoicePreview(
       onDeviceError: (deviceError) => {
         setMicWarning(deviceError.message || "无法访问麦克风。已尝试继续连接。");
       },
+      onCamUpdated: () => {
+        const track = transport.tracks().local.video;
+        setVideoStream(track ? new MediaStream([track]) : null);
+      },
       onTransportStateChanged: (nextState) => {
         if (nextState === "connected" || nextState === "ready") {
           setStatus("connected");
@@ -331,7 +338,9 @@ export function useVoicePreview(
         await transport.updateMic(selectedDeviceIdRef.current);
       }
       const localAudio = transport.tracks().local.audio;
+      const localVideo = transport.tracks().local.video;
       setLocalStream(localAudio ? new MediaStream([localAudio]) : null);
+      setVideoStream(localVideo ? new MediaStream([localVideo]) : null);
       void refreshDevices();
 
       const request = new Request(`${API_BASE}/api/webrtc/offer`, {
@@ -377,6 +386,10 @@ export function useVoicePreview(
     if (transport && deviceId) await transport.updateCam(deviceId);
   }, []);
 
+  const selectCamera = useCallback((deviceId: string) => {
+    transportRef.current?.updateCam(deviceId);
+  }, []);
+
   const selectDevice = useCallback(async (deviceId: string) => {
     setSelectedDeviceId(deviceId);
     selectedDeviceIdRef.current = deviceId;
@@ -413,6 +426,7 @@ export function useVoicePreview(
     error,
     micWarning,
     localStream,
+    videoStream,
     remoteStream,
     messages,
     callEnded,
@@ -428,6 +442,7 @@ export function useVoicePreview(
     sendText,
     connect,
     replaceVideoStream,
+    selectCamera,
     disconnect,
     audioRef,
   };