}
title="视觉理解"
- description="开启后,右侧调试面板可切换到视频流预览,并可同时选择麦克风与摄像头"
+ hint="开启后,右侧调试面板可切换到视频流预览,并可同时选择麦克风与摄像头。"
checked={visionEnabled}
onChange={setVisionEnabled}
/>
@@ -1841,6 +1841,7 @@ type VizStyle = "aura" | "nebula" | "bars" | "wave";
// 调试面板顶部主视图:聊天记录 / 视频流(仅视觉理解开启时可选)
type DebugView = "chat" | "video";
+type DebugInputMode = "mic" | "text";
const VIZ_OPTIONS: { style: VizStyle; label: string; icon: React.ReactNode }[] =
[
@@ -1917,38 +1918,29 @@ function DebugDrawer({
const camera = useCameraPreview();
const [showTranscript, setShowTranscript] = useState(false);
const [vizStyle, setVizStyle] = useState("aura");
- const [view, setView] = useState(vision ? "video" : "chat");
+ const [view, setView] = useState("chat");
const effectiveView: DebugView = vision ? view : "chat";
const stopCamera = camera.stop;
+ const startCamera = camera.start;
+ const cameraShouldRun =
+ vision &&
+ (preview.status === "connecting" || preview.status === "connected");
useEffect(() => {
- // eslint-disable-next-line react-hooks/set-state-in-effect
- setView(vision ? "video" : "chat");
- if (!vision) stopCamera();
+ if (!vision) {
+ // eslint-disable-next-line react-hooks/set-state-in-effect
+ setView("chat");
+ stopCamera();
+ }
}, [vision, stopCamera]);
- const { start: startCamera, active: cameraActive, starting: cameraStarting } =
- camera;
useEffect(() => {
- if (
- vision &&
- effectiveView === "video" &&
- !cameraActive &&
- !cameraStarting
- ) {
+ if (cameraShouldRun) {
void startCamera();
+ } else {
+ stopCamera();
}
- }, [
- vision,
- effectiveView,
- cameraActive,
- cameraStarting,
- startCamera,
- ]);
-
- useEffect(() => {
- if (effectiveView !== "video") stopCamera();
- }, [effectiveView, stopCamera]);
+ }, [cameraShouldRun, camera.deviceId, startCamera, stopCamera]);
const containerClass = asSheet
? "flex h-full min-w-0 flex-1 flex-col overflow-hidden"
@@ -2001,7 +1993,6 @@ function DebugDrawer({
>
)}
-
-
+ {vision && (
+
+ )}
);
}
-function DeviceMenu({
- preview,
- camera,
- vision,
-}: {
- preview: VoicePreview;
- camera: CameraPreview;
- vision: boolean;
-}) {
- return (
-
-
-
-
-
-
-
输入设备
-
- {vision
- ? "选择麦克风与摄像头,分别用于语音与视觉预览。"
- : "选择麦克风用于语音对话。"}
-
-
-
-
}
- label="麦克风"
- placeholder="默认麦克风"
- fallbackLabel="麦克风"
- value={preview.selectedDeviceId}
- devices={preview.audioInputs}
- onSelect={(id) => preview.selectDevice(id)}
- />
- {vision && (
- <>
-
-
}
- label="摄像头"
- placeholder="默认摄像头"
- fallbackLabel="摄像头"
- value={camera.deviceId}
- devices={camera.devices}
- onSelect={(id) => void camera.selectCamera(id)}
- />
- >
- )}
-
-
-
- );
-}
-
-function DeviceSelectRow({
+function DeviceSelectField({
icon,
- label,
+ ariaLabel,
placeholder,
fallbackLabel,
value,
@@ -2106,7 +2043,7 @@ function DeviceSelectRow({
onSelect,
}: {
icon: React.ReactNode;
- label: string;
+ ariaLabel: string;
placeholder: string;
fallbackLabel: string;
value: string;
@@ -2114,30 +2051,96 @@ function DeviceSelectRow({
onSelect: (deviceId: string) => void;
}) {
return (
-
-
{label}
-
+
+
+
+
+
+
+ {placeholder}
+ {devices.map((device, index) => (
+
+ {device.label || `${fallbackLabel} ${index + 1}`}
+
+ ))}
+
+
+ );
+}
+
+function MicrophoneDeviceField({ preview }: { preview: VoicePreview }) {
+ return (
+
}
+ ariaLabel="选择麦克风"
+ placeholder="默认麦克风"
+ fallbackLabel="麦克风"
+ value={preview.selectedDeviceId}
+ devices={preview.audioInputs}
+ onSelect={(deviceId) => preview.selectDevice(deviceId)}
+ />
+ );
+}
+
+function CameraDeviceField({ camera }: { camera: CameraPreview }) {
+ return (
+
}
+ ariaLabel="选择摄像头"
+ placeholder="默认摄像头"
+ fallbackLabel="摄像头"
+ value={camera.deviceId}
+ devices={camera.devices}
+ onSelect={(deviceId) => void camera.selectCamera(deviceId)}
+ />
+ );
+}
+
+function DebugInputModeButton({
+ selected,
+ label,
+ onClick,
+ children,
+}: {
+ selected: boolean;
+ label: string;
+ onClick: () => void;
+ children: React.ReactNode;
+}) {
+ return (
+
);
}
@@ -2148,6 +2151,7 @@ function DebugVoicePanel({
assistantId,
preview,
camera,
+ vision,
}: {
view: DebugView;
showTranscript: boolean;
@@ -2155,6 +2159,7 @@ function DebugVoicePanel({
assistantId: string | null;
preview: VoicePreview;
camera: CameraPreview;
+ vision: boolean;
}) {
const {
status,
@@ -2170,11 +2175,13 @@ function DebugVoicePanel({
} = preview;
const recording = status === "connecting" || status === "connected";
const [textDraft, setTextDraft] = useState("");
+ const [inputMode, setInputMode] = useState
("mic");
const inChatView = view === "chat" && (!SHOW_VOICE_VIZ || showTranscript);
+ const idleOrFailed = status === "idle" || status === "failed";
const showIdleHub =
- inChatView &&
messages.length === 0 &&
- (status === "idle" || status === "failed");
+ idleOrFailed &&
+ (inChatView || (vision && view === "video"));
function handleSendText() {
if (sendText(textDraft)) {
@@ -2186,7 +2193,14 @@ function DebugVoicePanel({
{/* 后端 TTS 音频经 WebRTC 媒体流过来,挂这里播放 */}
- {view === "video" ? (
+ {view === "video" && showIdleHub ? (
+
+ ) : view === "video" ? (
) : !SHOW_VOICE_VIZ || showTranscript ? (
showIdleHub ? (
@@ -2200,7 +2214,7 @@ function DebugVoicePanel({
)
) : (
-