diff --git a/frontend/src/components/pages/AssistantPage.tsx b/frontend/src/components/pages/AssistantPage.tsx index 8522aac..ff56d00 100644 --- a/frontend/src/components/pages/AssistantPage.tsx +++ b/frontend/src/components/pages/AssistantPage.tsx @@ -32,6 +32,8 @@ import { Orbit, Waves, Bug, + Video, + SlidersHorizontal, } from "lucide-react"; import { Button } from "@/components/ui/button"; @@ -95,8 +97,13 @@ import { import { useVoicePreview, type ChatMessage, + type VoicePreview, type VoicePreviewStatus, } from "@/hooks/use-voice-preview"; +import { + useCameraPreview, + type CameraPreview, +} from "@/hooks/use-camera-preview"; import { WorkflowEditor, type WorkflowSettings, @@ -325,6 +332,8 @@ export function AssistantPage(props: AssistantPageProps) { const editingId = props.mode === "edit" ? props.assistantId : null; const [form, setForm] = useState(() => blankPromptForm("")); + // 视觉理解(暂不持久化,不进 savedSnapshot,避免影响「未保存改动」判定) + const [visionEnabled, setVisionEnabled] = useState(false); const [fastGptForm, setFastGptForm] = useState(() => blankFastGptForm(""), ); @@ -1716,6 +1725,14 @@ export function AssistantPage(props: AssistantPageProps) { +
+ +
- + ); @@ -1822,6 +1839,9 @@ export function AssistantPage(props: AssistantPageProps) { type VizStyle = "aura" | "nebula" | "bars" | "wave"; +// 调试面板顶部主视图:聊天记录 / 视频流(仅视觉理解开启时可选) +type DebugView = "chat" | "video"; + const VIZ_OPTIONS: { style: VizStyle; label: string; icon: React.ReactNode }[] = [ { style: "aura", label: "光环", icon: }, @@ -1886,15 +1906,50 @@ function DebugDrawer({ assistantId, asSheet = false, onNodeActive, + vision = false, }: { assistantId: string | null; asSheet?: boolean; onNodeActive?: (nodeId: string | null) => void; + vision?: boolean; }) { + const preview = useVoicePreview(assistantId, onNodeActive); + const camera = useCameraPreview(); const [showTranscript, setShowTranscript] = useState(false); const [vizStyle, setVizStyle] = useState("aura"); + const [view, setView] = useState(vision ? "video" : "chat"); + const effectiveView: DebugView = vision ? view : "chat"; + const stopCamera = camera.stop; + + useEffect(() => { + // eslint-disable-next-line react-hooks/set-state-in-effect + setView(vision ? "video" : "chat"); + if (!vision) stopCamera(); + }, [vision, stopCamera]); + + const { start: startCamera, active: cameraActive, starting: cameraStarting } = + camera; + useEffect(() => { + if ( + vision && + effectiveView === "video" && + !cameraActive && + !cameraStarting + ) { + void startCamera(); + } + }, [ + vision, + effectiveView, + cameraActive, + cameraStarting, + startCamera, + ]); + + useEffect(() => { + if (effectiveView !== "video") stopCamera(); + }, [effectiveView, stopCamera]); - // 内联(prompt 编辑器右栏)用 aside + 圆角边框;抽屉模式占满 Sheet。 const containerClass = asSheet ? "flex h-full min-w-0 flex-1 flex-col overflow-hidden" : "hidden min-w-0 flex-1 flex-col overflow-hidden rounded-2xl border border-hairline bg-card shadow-sm lg:flex"; @@ -1902,63 +1957,204 @@ function DebugDrawer({ return ( ); } +function DeviceMenu({ + preview, + camera, + vision, +}: { + preview: VoicePreview; + camera: CameraPreview; + vision: boolean; +}) { + return ( + + + + + +
+
输入设备
+

+ {vision + ? "选择麦克风与摄像头,分别用于语音与视觉预览。" + : "选择麦克风用于语音对话。"} +

+
+
+ } + label="麦克风" + placeholder="默认麦克风" + fallbackLabel="麦克风" + value={preview.selectedDeviceId} + devices={preview.audioInputs} + onSelect={(id) => preview.selectDevice(id)} + /> + {vision && ( + <> +
+ } + label="摄像头" + placeholder="默认摄像头" + fallbackLabel="摄像头" + value={camera.deviceId} + devices={camera.devices} + onSelect={(id) => void camera.selectCamera(id)} + /> + + )} +
+ + + ); +} + +function DeviceSelectRow({ + icon, + label, + placeholder, + fallbackLabel, + value, + devices, + onSelect, +}: { + icon: React.ReactNode; + label: string; + placeholder: string; + fallbackLabel: string; + value: string; + devices: MediaDeviceInfo[]; + onSelect: (deviceId: string) => void; +}) { + return ( +
+
{label}
+ +
+ ); +} + function DebugVoicePanel({ + view, showTranscript, vizStyle, assistantId, - onNodeActive, + preview, + camera, }: { + view: DebugView; showTranscript: boolean; vizStyle: VizStyle; assistantId: string | null; - onNodeActive?: (nodeId: string | null) => void; + preview: VoicePreview; + camera: CameraPreview; }) { const { status, @@ -1967,17 +2163,18 @@ function DebugVoicePanel({ localStream, remoteStream, messages, - audioInputs, - selectedDeviceId, - selectDevice, sendText, connect, disconnect, audioRef, - } = useVoicePreview(assistantId, onNodeActive); - // 连接中或已连通都视作"会话进行中" + } = preview; const recording = status === "connecting" || status === "connected"; const [textDraft, setTextDraft] = useState(""); + const inChatView = view === "chat" && (!SHOW_VOICE_VIZ || showTranscript); + const showIdleHub = + inChatView && + messages.length === 0 && + (status === "idle" || status === "failed"); function handleSendText() { if (sendText(textDraft)) { @@ -1989,21 +2186,19 @@ function DebugVoicePanel({
{/* 后端 TTS 音频经 WebRTC 媒体流过来,挂这里播放 */}
- {/* 底部双轨波形:用户麦克风 + 助手音频,可折叠 */} + {/* 底部双轨波形:会话连通后默认展开,空闲时收起 */} void; connect: () => Promise; - disconnect: () => void; }) { - const recording = status === "connecting" || status === "connected"; const hint = status === "failed" ? error || "连接失败，请确认后端已启动且助手已保存后重试。" : !assistantId ? "请先保存助手，再开始语音预览。" - : micWarning - ? `${micWarning} 可接收助手播报，但无法发送语音。` - : null; + : "测试语音识别、响应速度与助手的播报效果。"; return ( -
-
- - - {status === "connecting" - ? "连接中…" - : status === "connected" - ? micWarning - ? "仅收听" - : "进行中" - : status === "failed" - ? "连接失败" - : "准备开始"} - - - - +
+
+
+ +
+
+ {status === "failed" ? "连接失败" : "开始一次语音对话"} +
+

{hint}

+

+ 在右上角「设备」中选择麦克风 +

+
+ ); +} - {hint && ( -

{hint}

+function DebugVideoPanel({ camera }: { camera: CameraPreview }) { + const { stream, error, starting, active, start } = camera; + const videoRef = useRef(null); + + useEffect(() => { + if (videoRef.current) videoRef.current.srcObject = stream; + }, [stream]); + + return ( +
+
); } +function DebugConnectionStatus({ + status, + micWarning, +}: { + status: VoicePreviewStatus; + micWarning: string | null; +}) { + const recording = status === "connecting" || status === "connected"; + const label = + status === "connecting" + ? "连接中…" + : status === "connected" + ? micWarning + ? "仅收听" + : "进行中" + : status === "failed" + ? "连接失败" + : "准备开始"; + + return ( + + + {label} + + ); +} + // ISO 时间戳 → HH:MM(本地时区),解析失败返回空串 function formatMessageTime(iso: string): string { const d = new Date(iso); @@ -2291,10 +2519,10 @@ function formatMessageTime(iso: string): string { function DebugTranscriptPanel({ messages, - recording, + recording = false, }: { messages: ChatMessage[]; - recording: boolean; + recording?: boolean; }) { const scrollRef = useRef(null); @@ -2306,16 +2534,23 @@ function DebugTranscriptPanel({ if (messages.length === 0) { return ( -
- -
- {recording ? "暂无聊天记录" : "尚未开始对话"} +
+
+ +
+ {recording ? "暂无聊天记录" : "尚未开始对话"} +
+

+ {recording + ? "开口说话或在下方输入文字，对话内容会实时显示在这里。" + : "点击「开始对话」后，语音与文字消息会实时显示在这里。"} +

-

- {recording - ? "开口说话或在下方输入文字，对话内容会实时显示在这里。" - : "点击「开始对话」后，语音与文字消息会实时显示在这里。"} -

); } diff --git a/frontend/src/hooks/use-camera-preview.ts b/frontend/src/hooks/use-camera-preview.ts new file mode 100644 index 0000000..0123a56 --- /dev/null +++ b/frontend/src/hooks/use-camera-preview.ts @@ -0,0 +1,113 @@ +"use client"; + +/** + * 摄像头预览:纯前端 getUserMedia 视频流,供调试抽屉的「视频流」视图使用。 + * + * 与 use-voice-preview 分离:麦克风/语音会话走 WebRTC 到后端,摄像头目前只在 + * 本地预览。后续接入视觉理解管线时,把这条 video track 加进现有 PeerConnection + * (pc.addTrack)并在后端 transport 打开 video_in_enabled 即可。 + */ + +import { useCallback, useEffect, useRef, useState } from "react"; + +function cameraErrorMessage(error: unknown): string { + if (error instanceof DOMException) { + if (error.name === "NotAllowedError") + return "摄像头权限被拒绝，请在浏览器网站设置中允许使用摄像头。"; + if (error.name === "NotFoundError") + return "未检测到可用摄像头，请连接或启用摄像头后重试。"; + if (error.name === "NotReadableError") + return "摄像头正被其他应用占用，或系统未允许浏览器访问摄像头。"; + } + if (error instanceof Error && error.message) return error.message; + return "无法访问摄像头。"; +} + +export function useCameraPreview() { + const [stream, setStream] = useState(null); + const [error, setError] = useState(null); + const [starting, setStarting] = useState(false); + const [devices, setDevices] = useState([]); + const [deviceId, setDeviceId] = useState(""); + const streamRef = useRef(null); + const deviceIdRef = useRef(""); + + const active = Boolean(stream); + + const refreshCameras = useCallback(async () => { + if (!navigator.mediaDevices?.enumerateDevices) return; + try { + const list = await navigator.mediaDevices.enumerateDevices(); + setDevices(list.filter((d) => d.kind === "videoinput" && d.deviceId)); + } catch { + /* 忽略枚举失败 */ + } + }, []); + + const stop = useCallback(() => { + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + setStream(null); + }, []); + + const start = useCallback(async () => { + if (streamRef.current) return; + setError(null); + if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) { + setError("当前页面无法访问摄像头，请通过 https 或 localhost 访问。"); + return; + } + setStarting(true); + try { + const id = deviceIdRef.current; + const media = await navigator.mediaDevices.getUserMedia({ + video: id ? { deviceId: { exact: id } } : true, + }); + streamRef.current = media; + setStream(media); + void refreshCameras(); + } catch (mediaError) { + setError(cameraErrorMessage(mediaError)); + } finally { + setStarting(false); + } + }, [refreshCameras]); + + const selectCamera = useCallback( + async (id: string) => { + deviceIdRef.current = id; + setDeviceId(id); + if (streamRef.current) { + stop(); + await start(); + } + }, + [start, stop], + ); + + useEffect(() => { + if (!navigator.mediaDevices?.enumerateDevices) return; + // eslint-disable-next-line react-hooks/set-state-in-effect + void refreshCameras(); + navigator.mediaDevices.addEventListener("devicechange", refreshCameras); + return () => { + navigator.mediaDevices.removeEventListener("devicechange", refreshCameras); + }; + }, [refreshCameras]); + + useEffect(() => stop, [stop]); + + return { + stream, + error, + starting, + devices, + deviceId, + active, + start, + stop, + selectCamera, + }; +} + +export type CameraPreview = ReturnType; diff --git a/frontend/src/hooks/use-voice-preview.ts b/frontend/src/hooks/use-voice-preview.ts index 93b3f0e..f1059f4 100644 --- a/frontend/src/hooks/use-voice-preview.ts +++ b/frontend/src/hooks/use-voice-preview.ts @@ -558,3 +558,5 @@ export function useVoicePreview( audioRef, }; } + +export type VoicePreview = ReturnType;