Refactor camera handling and video stream management in Assistant and MobileCall pages

- Remove direct camera start/stop calls and replace them with streamlined video stream handling. - Update DebugVideoPanel to accept a streamOverride for better flexibility in video source management. - Adjust MobileCallPage to utilize the new videoStream state, enhancing the call experience. - Introduce videoStream state management in useVoicePreview for improved media handling during voice interactions.
2026-07-13 07:05:06 +08:00
parent ede5e2334b
commit fd57f34a55
3 changed files with 43 additions and 45 deletions
--- a/frontend/src/components/pages/AssistantPage.tsx
+++ b/frontend/src/components/pages/AssistantPage.tsx
@@ -2155,11 +2155,8 @@ function DebugDrawer({
  const [dynamicVariableValues, setDynamicVariableValues] = useState<
    Record<string, string | number | boolean>
  >({});
-  const stopCamera = camera.stop;
-  const startCamera = camera.start;
  const recording =
    preview.status === "connecting" || preview.status === "connected";
-  const shouldKeepCamera = vision && recording;
  const dynamicVariableEntries = Object.entries(dynamicVariableDefinitions);
  const resolvedDynamicVariables: Record<string, string | number | boolean> = {};
  let dynamicVariablesError = "";
@@ -2174,18 +2171,10 @@ function DebugDrawer({
    resolvedDynamicVariables[name] = value;
  }

-  useEffect(() => {
-    if (shouldKeepCamera) {
-      void startCamera();
-    } else {
-      stopCamera();
-    }
-  }, [shouldKeepCamera, startCamera, stopCamera]);
-
  const selectCamera = useCallback(
    async (deviceId: string) => {
-      const nextStream = await camera.selectCamera(deviceId);
-      await preview.replaceVideoStream(nextStream);
+      await camera.selectCamera(deviceId);
+      preview.selectCamera(deviceId);
    },
    [camera, preview],
  );
@@ -2666,15 +2655,12 @@ function DebugVoicePanel({
  const startConversation = useCallback(async () => {
    if (!assistantId || hasUnsavedChanges) return;
    if (dynamicVariablesError) return;
-    const videoStream = vision ? await camera.start() : null;
    await connect({
      visionEnabled: vision,
-      videoStream,
      dynamicVariables,
    });
  }, [
    assistantId,
-    camera,
    connect,
    dynamicVariables,
    dynamicVariablesError,
@@ -2695,7 +2681,7 @@ function DebugVoicePanel({
            connect={startConversation}
          />
        ) : (
-          <DebugVideoPanel camera={camera} />
+          <DebugVideoPanel camera={camera} streamOverride={preview.videoStream} />
        )
      ) : !SHOW_VOICE_VIZ || showTranscript ? (
        showIdleHub ? (
@@ -2955,19 +2941,27 @@ function DebugIdleHub({
  );
 }

-function DebugVideoPanel({ camera }: { camera: CameraPreview }) {
+function DebugVideoPanel({
+  camera,
+  streamOverride,
+}: {
+  camera: CameraPreview;
+  streamOverride?: MediaStream | null;
+}) {
  const { stream, error, starting, active } = camera;
+  const effectiveStream = streamOverride ?? stream;
+  const effectiveActive = Boolean(effectiveStream) || active;
  const videoRef = useRef<HTMLVideoElement>(null);

  useEffect(() => {
-    if (videoRef.current) videoRef.current.srcObject = stream;
-  }, [stream]);
+    if (videoRef.current) videoRef.current.srcObject = effectiveStream;
+  }, [effectiveStream]);

  return (
    <div
      className={[
        "relative flex min-h-0 flex-1 items-center justify-center overflow-hidden",
-        active ? "bg-black" : "bg-canvas-soft",
+        effectiveActive ? "bg-black" : "bg-canvas-soft",
      ].join(" ")}
    >
      <video
@@ -2977,10 +2971,10 @@ function DebugVideoPanel({ camera }: { camera: CameraPreview }) {
        muted
        className={[
          "h-full w-full -scale-x-100 object-contain",
-          active ? "" : "hidden",
+          effectiveActive ? "" : "hidden",
        ].join(" ")}
      />
-      {active ? (
+      {effectiveActive ? (
        <Badge
          variant="secondary"
          className="absolute left-3 top-3 gap-1.5 rounded-full border border-hairline bg-card/80 px-2.5 py-1 text-[11px] font-medium text-muted-foreground shadow-none backdrop-blur"
--- a/frontend/src/components/pages/MobileCallPage.tsx
+++ b/frontend/src/components/pages/MobileCallPage.tsx
@@ -251,18 +251,16 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
    callEnded,
    networkQuality,
    connect,
-    replaceVideoStream,
+    videoStream,
+    selectCamera: selectTransportCamera,
    disconnect,
    audioRef,
  } = preview;
  const {
-    stream: cameraStream,
    error: cameraError,
    starting: cameraStarting,
    devices: cameraDevices,
    deviceId: cameraDeviceId,
-    start: startCamera,
-    stop: stopCamera,
    selectCamera: changeCamera,
  } = camera;
  const videoRef = useRef<HTMLVideoElement>(null);
@@ -271,29 +269,26 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
  const inCall = !callEnded && (connecting || status === "connected");

  useEffect(() => {
-    if (videoRef.current) videoRef.current.srcObject = cameraStream;
-  }, [cameraStream]);
+    if (videoRef.current) videoRef.current.srcObject = videoStream;
+  }, [videoStream]);

  const startCall = useCallback(async () => {
-    const videoStream = await startCamera();
    await connect({
      visionEnabled: true,
-      videoStream,
    });
-  }, [connect, startCamera]);
+  }, [connect]);

  const endCall = useCallback(() => {
    setView("camera");
    disconnect();
-    stopCamera();
-  }, [disconnect, stopCamera]);
+  }, [disconnect]);

  const selectCamera = useCallback(
    async (deviceId: string) => {
-      const nextStream = await changeCamera(deviceId);
-      await replaceVideoStream(nextStream);
+      await changeCamera(deviceId);
+      selectTransportCamera(deviceId);
    },
-    [changeCamera, replaceVideoStream],
+    [changeCamera, selectTransportCamera],
  );

  useEffect(() => {
@@ -301,12 +296,6 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
    // 首次打开页面时自动发起通话；hooks 会在卸载时各自释放媒体资源。
  }, [startCall]);

-  useEffect(() => {
-    if (callEnded || status === "idle" || status === "failed") {
-      stopCamera();
-    }
-  }, [callEnded, status, stopCamera]);
-
  const error = previewError || cameraError;

  return (
@@ -323,7 +312,7 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
        muted
        className={[
          "absolute inset-0 h-full w-full -scale-x-100 object-cover transition-opacity duration-300 lg:object-contain",
-          cameraStream && view === "camera" ? "opacity-100" : "opacity-0",
+          videoStream && view === "camera" ? "opacity-100" : "opacity-0",
        ].join(" ")}
      />

@@ -376,7 +365,7 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
        </button>
      )}

-      {!cameraStream && inCall && view === "camera" && (
+      {!videoStream && inCall && view === "camera" && (
        <div className="absolute inset-0 z-[2] flex items-center justify-center px-8 text-center">
          <div className="flex max-w-sm flex-col items-center gap-3">
            {cameraStarting ? (
--- a/frontend/src/hooks/use-voice-preview.ts
+++ b/frontend/src/hooks/use-voice-preview.ts
@@ -89,6 +89,7 @@ export function useVoicePreview(
  const [error, setError] = useState<string | null>(null);
  const [micWarning, setMicWarning] = useState<string | null>(null);
  const [localStream, setLocalStream] = useState<MediaStream | null>(null);
+  const [videoStream, setVideoStream] = useState<MediaStream | null>(null);
  const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
  const [messages, setMessages] = useState<ChatMessage[]>([]);
  const [callEnded, setCallEnded] = useState(false);
@@ -161,6 +162,7 @@ export function useVoicePreview(
  const disconnect = useCallback(() => {
    releaseResources();
    setLocalStream(null);
+    setVideoStream(null);
    setRemoteStream(null);
    setMessages([]);
    messageSeqRef.current = 0;
@@ -173,6 +175,7 @@ export function useVoicePreview(
  const fail = useCallback((message: string) => {
    releaseResources();
    setLocalStream(null);
+    setVideoStream(null);
    setRemoteStream(null);
    setError(message);
    setNetworkQuality("unknown");
@@ -306,6 +309,10 @@ export function useVoicePreview(
      onDeviceError: (deviceError) => {
        setMicWarning(deviceError.message || "无法访问麦克风。已尝试继续连接。");
      },
+      onCamUpdated: () => {
+        const track = transport.tracks().local.video;
+        setVideoStream(track ? new MediaStream([track]) : null);
+      },
      onTransportStateChanged: (nextState) => {
        if (nextState === "connected" || nextState === "ready") {
          setStatus("connected");
@@ -331,7 +338,9 @@ export function useVoicePreview(
        await transport.updateMic(selectedDeviceIdRef.current);
      }
      const localAudio = transport.tracks().local.audio;
+      const localVideo = transport.tracks().local.video;
      setLocalStream(localAudio ? new MediaStream([localAudio]) : null);
+      setVideoStream(localVideo ? new MediaStream([localVideo]) : null);
      void refreshDevices();

      const request = new Request(`${API_BASE}/api/webrtc/offer`, {
@@ -377,6 +386,10 @@ export function useVoicePreview(
    if (transport && deviceId) await transport.updateCam(deviceId);
  }, []);

+  const selectCamera = useCallback((deviceId: string) => {
+    transportRef.current?.updateCam(deviceId);
+  }, []);
+
  const selectDevice = useCallback(async (deviceId: string) => {
    setSelectedDeviceId(deviceId);
    selectedDeviceIdRef.current = deviceId;
@@ -413,6 +426,7 @@ export function useVoicePreview(
    error,
    micWarning,
    localStream,
+    videoStream,
    remoteStream,
    messages,
    callEnded,
@@ -428,6 +442,7 @@ export function useVoicePreview(
    sendText,
    connect,
    replaceVideoStream,
+    selectCamera,
    disconnect,
    audioRef,
  };