Implement camera preview functionality and enhance AssistantPage
- Add a new `useCameraPreview` hook for managing camera access and video stream handling. - Integrate camera preview capabilities into the AssistantPage, allowing users to toggle video stream visibility. - Introduce a new toggle for enabling visual understanding, which activates the video stream preview alongside voice functionalities. - Update the DebugDrawer component to support switching between chat and video views based on the visual understanding setting. - Refactor related components to accommodate the new camera features and improve user interaction during debugging.
This commit is contained in:
@@ -32,6 +32,8 @@ import {
|
||||
Orbit,
|
||||
Waves,
|
||||
Bug,
|
||||
Video,
|
||||
SlidersHorizontal,
|
||||
} from "lucide-react";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
@@ -95,8 +97,13 @@ import {
|
||||
import {
|
||||
useVoicePreview,
|
||||
type ChatMessage,
|
||||
type VoicePreview,
|
||||
type VoicePreviewStatus,
|
||||
} from "@/hooks/use-voice-preview";
|
||||
import {
|
||||
useCameraPreview,
|
||||
type CameraPreview,
|
||||
} from "@/hooks/use-camera-preview";
|
||||
import {
|
||||
WorkflowEditor,
|
||||
type WorkflowSettings,
|
||||
@@ -325,6 +332,8 @@ export function AssistantPage(props: AssistantPageProps) {
|
||||
const editingId = props.mode === "edit" ? props.assistantId : null;
|
||||
|
||||
const [form, setForm] = useState<AssistantForm>(() => blankPromptForm(""));
|
||||
// 视觉理解(暂不持久化,不进 savedSnapshot,避免影响「未保存改动」判定)
|
||||
const [visionEnabled, setVisionEnabled] = useState(false);
|
||||
const [fastGptForm, setFastGptForm] = useState<FastGptForm>(() =>
|
||||
blankFastGptForm(""),
|
||||
);
|
||||
@@ -1716,6 +1725,14 @@ export function AssistantPage(props: AssistantPageProps) {
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="mt-4 border-t border-hairline pt-4">
|
||||
<ToggleRow
|
||||
title="视觉理解"
|
||||
description="开启后,右侧调试面板可切换到视频流预览,并可同时选择麦克风与摄像头"
|
||||
checked={visionEnabled}
|
||||
onChange={setVisionEnabled}
|
||||
/>
|
||||
</div>
|
||||
</SectionCard>
|
||||
|
||||
<SectionCard
|
||||
@@ -1814,7 +1831,7 @@ export function AssistantPage(props: AssistantPageProps) {
|
||||
</SectionCard>
|
||||
</div>
|
||||
|
||||
<DebugDrawer assistantId={editingId} />
|
||||
<DebugDrawer assistantId={editingId} vision={visionEnabled} />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
@@ -1822,6 +1839,9 @@ export function AssistantPage(props: AssistantPageProps) {
|
||||
|
||||
type VizStyle = "aura" | "nebula" | "bars" | "wave";
|
||||
|
||||
// 调试面板顶部主视图:聊天记录 / 视频流(仅视觉理解开启时可选)
|
||||
type DebugView = "chat" | "video";
|
||||
|
||||
const VIZ_OPTIONS: { style: VizStyle; label: string; icon: React.ReactNode }[] =
|
||||
[
|
||||
{ style: "aura", label: "光环", icon: <Orbit size={14} /> },
|
||||
@@ -1886,15 +1906,50 @@ function DebugDrawer({
|
||||
assistantId,
|
||||
asSheet = false,
|
||||
onNodeActive,
|
||||
vision = false,
|
||||
}: {
|
||||
assistantId: string | null;
|
||||
asSheet?: boolean;
|
||||
onNodeActive?: (nodeId: string | null) => void;
|
||||
vision?: boolean;
|
||||
}) {
|
||||
const preview = useVoicePreview(assistantId, onNodeActive);
|
||||
const camera = useCameraPreview();
|
||||
const [showTranscript, setShowTranscript] = useState(false);
|
||||
const [vizStyle, setVizStyle] = useState<VizStyle>("aura");
|
||||
const [view, setView] = useState<DebugView>(vision ? "video" : "chat");
|
||||
const effectiveView: DebugView = vision ? view : "chat";
|
||||
const stopCamera = camera.stop;
|
||||
|
||||
useEffect(() => {
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||||
setView(vision ? "video" : "chat");
|
||||
if (!vision) stopCamera();
|
||||
}, [vision, stopCamera]);
|
||||
|
||||
const { start: startCamera, active: cameraActive, starting: cameraStarting } =
|
||||
camera;
|
||||
useEffect(() => {
|
||||
if (
|
||||
vision &&
|
||||
effectiveView === "video" &&
|
||||
!cameraActive &&
|
||||
!cameraStarting
|
||||
) {
|
||||
void startCamera();
|
||||
}
|
||||
}, [
|
||||
vision,
|
||||
effectiveView,
|
||||
cameraActive,
|
||||
cameraStarting,
|
||||
startCamera,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
if (effectiveView !== "video") stopCamera();
|
||||
}, [effectiveView, stopCamera]);
|
||||
|
||||
// 内联(prompt 编辑器右栏)用 aside + 圆角边框;抽屉模式占满 Sheet。
|
||||
const containerClass = asSheet
|
||||
? "flex h-full min-w-0 flex-1 flex-col overflow-hidden"
|
||||
: "hidden min-w-0 flex-1 flex-col overflow-hidden rounded-2xl border border-hairline bg-card shadow-sm lg:flex";
|
||||
@@ -1902,63 +1957,204 @@ function DebugDrawer({
|
||||
return (
|
||||
<aside className={containerClass}>
|
||||
<div className="flex shrink-0 items-center justify-between gap-3 border-b border-hairline px-5 py-3">
|
||||
<div className="text-sm font-medium text-foreground">调试与预览</div>
|
||||
{SHOW_VOICE_VIZ && (
|
||||
<div className="flex items-center gap-2">
|
||||
{!showTranscript && (
|
||||
<SegmentedIconGroup label="可视化样式">
|
||||
{VIZ_OPTIONS.map((option) => (
|
||||
<SegmentedIconButton
|
||||
key={option.style}
|
||||
selected={vizStyle === option.style}
|
||||
label={`可视化样式:${option.label}`}
|
||||
onClick={() => setVizStyle(option.style)}
|
||||
>
|
||||
{option.icon}
|
||||
</SegmentedIconButton>
|
||||
))}
|
||||
</SegmentedIconGroup>
|
||||
)}
|
||||
<SegmentedIconGroup label="预览视图">
|
||||
<SegmentedIconButton
|
||||
selected={!showTranscript}
|
||||
label="语音可视化视图"
|
||||
onClick={() => setShowTranscript(false)}
|
||||
>
|
||||
<Mic size={14} />
|
||||
</SegmentedIconButton>
|
||||
<SegmentedIconButton
|
||||
selected={showTranscript}
|
||||
label="文字聊天记录视图"
|
||||
onClick={() => setShowTranscript(true)}
|
||||
>
|
||||
<MessageSquareText size={14} />
|
||||
</SegmentedIconButton>
|
||||
</SegmentedIconGroup>
|
||||
<div className="flex min-w-0 items-center gap-2.5">
|
||||
<div className="shrink-0 text-sm font-medium text-foreground">
|
||||
调试与预览
|
||||
</div>
|
||||
)}
|
||||
<DebugConnectionStatus
|
||||
status={preview.status}
|
||||
micWarning={preview.micWarning}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-2">
|
||||
{SHOW_VOICE_VIZ && effectiveView === "chat" && (
|
||||
<>
|
||||
{!showTranscript && (
|
||||
<SegmentedIconGroup label="可视化样式">
|
||||
{VIZ_OPTIONS.map((option) => (
|
||||
<SegmentedIconButton
|
||||
key={option.style}
|
||||
selected={vizStyle === option.style}
|
||||
label={`可视化样式:${option.label}`}
|
||||
onClick={() => setVizStyle(option.style)}
|
||||
>
|
||||
{option.icon}
|
||||
</SegmentedIconButton>
|
||||
))}
|
||||
</SegmentedIconGroup>
|
||||
)}
|
||||
<SegmentedIconGroup label="语音视图">
|
||||
<SegmentedIconButton
|
||||
selected={!showTranscript}
|
||||
label="语音可视化视图"
|
||||
onClick={() => setShowTranscript(false)}
|
||||
>
|
||||
<Mic size={14} />
|
||||
</SegmentedIconButton>
|
||||
<SegmentedIconButton
|
||||
selected={showTranscript}
|
||||
label="文字聊天记录视图"
|
||||
onClick={() => setShowTranscript(true)}
|
||||
>
|
||||
<MessageSquareText size={14} />
|
||||
</SegmentedIconButton>
|
||||
</SegmentedIconGroup>
|
||||
</>
|
||||
)}
|
||||
<DeviceMenu preview={preview} camera={camera} vision={vision} />
|
||||
<SegmentedIconGroup label="预览视图">
|
||||
<SegmentedIconButton
|
||||
selected={effectiveView === "chat"}
|
||||
label="聊天记录"
|
||||
onClick={() => setView("chat")}
|
||||
>
|
||||
<MessageSquareText size={14} />
|
||||
</SegmentedIconButton>
|
||||
{vision && (
|
||||
<SegmentedIconButton
|
||||
selected={effectiveView === "video"}
|
||||
label="视频流"
|
||||
onClick={() => setView("video")}
|
||||
>
|
||||
<Video size={14} />
|
||||
</SegmentedIconButton>
|
||||
)}
|
||||
</SegmentedIconGroup>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<DebugVoicePanel
|
||||
view={effectiveView}
|
||||
showTranscript={showTranscript}
|
||||
vizStyle={vizStyle}
|
||||
assistantId={assistantId}
|
||||
onNodeActive={onNodeActive}
|
||||
preview={preview}
|
||||
camera={camera}
|
||||
/>
|
||||
</aside>
|
||||
);
|
||||
}
|
||||
|
||||
function DeviceMenu({
|
||||
preview,
|
||||
camera,
|
||||
vision,
|
||||
}: {
|
||||
preview: VoicePreview;
|
||||
camera: CameraPreview;
|
||||
vision: boolean;
|
||||
}) {
|
||||
return (
|
||||
<Popover>
|
||||
<PopoverTrigger asChild>
|
||||
<button
|
||||
type="button"
|
||||
aria-label="设备设置"
|
||||
title="设备设置"
|
||||
className="flex h-8 items-center gap-1.5 rounded-full border border-hairline bg-canvas-soft px-2.5 text-xs text-muted-soft transition-colors hover:text-foreground"
|
||||
>
|
||||
<SlidersHorizontal size={14} />
|
||||
<span className="hidden sm:inline">设备</span>
|
||||
</button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent align="end" className="w-72 p-0">
|
||||
<div className="border-b border-hairline px-3 py-2.5">
|
||||
<div className="text-xs font-medium text-foreground">输入设备</div>
|
||||
<p className="mt-0.5 text-[11px] leading-4 text-muted-foreground">
|
||||
{vision
|
||||
? "选择麦克风与摄像头,分别用于语音与视觉预览。"
|
||||
: "选择麦克风用于语音对话。"}
|
||||
</p>
|
||||
</div>
|
||||
<div className="space-y-3 p-3">
|
||||
<DeviceSelectRow
|
||||
icon={<Mic size={13} className="shrink-0 text-muted-soft" />}
|
||||
label="麦克风"
|
||||
placeholder="默认麦克风"
|
||||
fallbackLabel="麦克风"
|
||||
value={preview.selectedDeviceId}
|
||||
devices={preview.audioInputs}
|
||||
onSelect={(id) => preview.selectDevice(id)}
|
||||
/>
|
||||
{vision && (
|
||||
<>
|
||||
<div className="border-t border-hairline" />
|
||||
<DeviceSelectRow
|
||||
icon={<Video size={13} className="shrink-0 text-muted-soft" />}
|
||||
label="摄像头"
|
||||
placeholder="默认摄像头"
|
||||
fallbackLabel="摄像头"
|
||||
value={camera.deviceId}
|
||||
devices={camera.devices}
|
||||
onSelect={(id) => void camera.selectCamera(id)}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
);
|
||||
}
|
||||
|
||||
function DeviceSelectRow({
|
||||
icon,
|
||||
label,
|
||||
placeholder,
|
||||
fallbackLabel,
|
||||
value,
|
||||
devices,
|
||||
onSelect,
|
||||
}: {
|
||||
icon: React.ReactNode;
|
||||
label: string;
|
||||
placeholder: string;
|
||||
fallbackLabel: string;
|
||||
value: string;
|
||||
devices: MediaDeviceInfo[];
|
||||
onSelect: (deviceId: string) => void;
|
||||
}) {
|
||||
return (
|
||||
<div className="space-y-1.5">
|
||||
<div className="text-xs font-medium text-muted-foreground">{label}</div>
|
||||
<Select
|
||||
value={value || "default"}
|
||||
onValueChange={(v) => onSelect(v === "default" ? "" : v)}
|
||||
>
|
||||
<SelectTrigger
|
||||
size="sm"
|
||||
className="w-full gap-2 rounded-full border-hairline bg-canvas-soft text-xs text-muted-foreground"
|
||||
aria-label={`选择${label}`}
|
||||
>
|
||||
{icon}
|
||||
<SelectValue placeholder={placeholder} />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="default">{placeholder}</SelectItem>
|
||||
{devices.map((device, index) => (
|
||||
<SelectItem key={device.deviceId} value={device.deviceId}>
|
||||
{device.label || `${fallbackLabel} ${index + 1}`}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function DebugVoicePanel({
|
||||
view,
|
||||
showTranscript,
|
||||
vizStyle,
|
||||
assistantId,
|
||||
onNodeActive,
|
||||
preview,
|
||||
camera,
|
||||
}: {
|
||||
view: DebugView;
|
||||
showTranscript: boolean;
|
||||
vizStyle: VizStyle;
|
||||
assistantId: string | null;
|
||||
onNodeActive?: (nodeId: string | null) => void;
|
||||
preview: VoicePreview;
|
||||
camera: CameraPreview;
|
||||
}) {
|
||||
const {
|
||||
status,
|
||||
@@ -1967,17 +2163,18 @@ function DebugVoicePanel({
|
||||
localStream,
|
||||
remoteStream,
|
||||
messages,
|
||||
audioInputs,
|
||||
selectedDeviceId,
|
||||
selectDevice,
|
||||
sendText,
|
||||
connect,
|
||||
disconnect,
|
||||
audioRef,
|
||||
} = useVoicePreview(assistantId, onNodeActive);
|
||||
// 连接中或已连通都视作"会话进行中"
|
||||
} = preview;
|
||||
const recording = status === "connecting" || status === "connected";
|
||||
const [textDraft, setTextDraft] = useState("");
|
||||
const inChatView = view === "chat" && (!SHOW_VOICE_VIZ || showTranscript);
|
||||
const showIdleHub =
|
||||
inChatView &&
|
||||
messages.length === 0 &&
|
||||
(status === "idle" || status === "failed");
|
||||
|
||||
function handleSendText() {
|
||||
if (sendText(textDraft)) {
|
||||
@@ -1989,21 +2186,19 @@ function DebugVoicePanel({
|
||||
<div className="flex min-h-0 flex-1 flex-col">
|
||||
{/* 后端 TTS 音频经 WebRTC 媒体流过来,挂这里播放 */}
|
||||
<audio ref={audioRef} autoPlay playsInline className="hidden" />
|
||||
{!SHOW_VOICE_VIZ || showTranscript ? (
|
||||
<>
|
||||
<DebugTranscriptPanel messages={messages} recording={recording} />
|
||||
<VoiceSessionControls
|
||||
{view === "video" ? (
|
||||
<DebugVideoPanel camera={camera} />
|
||||
) : !SHOW_VOICE_VIZ || showTranscript ? (
|
||||
showIdleHub ? (
|
||||
<DebugIdleHub
|
||||
status={status}
|
||||
error={error}
|
||||
micWarning={micWarning}
|
||||
assistantId={assistantId}
|
||||
audioInputs={audioInputs}
|
||||
selectedDeviceId={selectedDeviceId}
|
||||
selectDevice={selectDevice}
|
||||
connect={connect}
|
||||
disconnect={disconnect}
|
||||
/>
|
||||
</>
|
||||
) : (
|
||||
<DebugTranscriptPanel messages={messages} recording={recording} />
|
||||
)
|
||||
) : (
|
||||
<div className="relative flex min-h-0 flex-1 flex-col items-center justify-center gap-3 overflow-y-auto px-6 py-3 text-center">
|
||||
<div
|
||||
@@ -2068,32 +2263,6 @@ function DebugVoicePanel({
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="relative w-full max-w-xs">
|
||||
<Select
|
||||
value={selectedDeviceId || "default"}
|
||||
onValueChange={(value) =>
|
||||
selectDevice(value === "default" ? "" : value)
|
||||
}
|
||||
>
|
||||
<SelectTrigger
|
||||
size="sm"
|
||||
className="w-full justify-center gap-2 rounded-full border-hairline bg-canvas-soft text-xs text-muted-foreground"
|
||||
aria-label="选择麦克风"
|
||||
>
|
||||
<Mic size={13} className="shrink-0 text-muted-soft" />
|
||||
<SelectValue placeholder="默认麦克风" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="default">默认麦克风</SelectItem>
|
||||
{audioInputs.map((device, index) => (
|
||||
<SelectItem key={device.deviceId} value={device.deviceId}>
|
||||
{device.label || `麦克风 ${index + 1}`}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
|
||||
<Button
|
||||
disabled={!assistantId || status === "connecting"}
|
||||
onClick={() => {
|
||||
@@ -2152,11 +2321,42 @@ function DebugVoicePanel({
|
||||
>
|
||||
<Send size={16} />
|
||||
</Button>
|
||||
{!showIdleHub && (
|
||||
<Button
|
||||
size="sm"
|
||||
disabled={!assistantId || status === "connecting"}
|
||||
onClick={() => {
|
||||
if (recording) {
|
||||
disconnect();
|
||||
} else {
|
||||
void connect();
|
||||
}
|
||||
}}
|
||||
className={[
|
||||
"shrink-0 gap-1.5 rounded-full",
|
||||
recording
|
||||
? "bg-destructive text-white hover:bg-destructive/90"
|
||||
: "",
|
||||
].join(" ")}
|
||||
aria-label={recording ? "结束语音测试" : "开始语音测试"}
|
||||
>
|
||||
{status === "connecting" ? (
|
||||
<Loader2 size={14} className="animate-spin" />
|
||||
) : recording ? (
|
||||
<PhoneOff size={14} />
|
||||
) : (
|
||||
<Mic size={14} />
|
||||
)}
|
||||
{recording ? "结束对话" : "开始对话"}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* 底部双轨波形:用户麦克风 + 助手音频,可折叠 */}
|
||||
{/* 底部双轨波形:会话连通后默认展开,空闲时收起 */}
|
||||
<WaveformTimelinePanel
|
||||
key={status === "connected" ? "active" : "idle"}
|
||||
defaultOpen={status === "connected"}
|
||||
userStream={localStream}
|
||||
agentStream={remoteStream}
|
||||
active={status === "connected"}
|
||||
@@ -2165,122 +2365,150 @@ function DebugVoicePanel({
|
||||
);
|
||||
}
|
||||
|
||||
// 会话控制条:状态 + 麦克风选择 + 开始/结束按钮。
|
||||
// 原本这些控件在中央可视化视图里,可视化隐藏后(SHOW_VOICE_VIZ=false)集中到这一条。
|
||||
function VoiceSessionControls({
|
||||
// 空闲态:图标 + 说明 + 开始按钮合一,替代「撑满高度的空 transcript + 底部细控制条」
|
||||
function DebugIdleHub({
|
||||
status,
|
||||
error,
|
||||
micWarning,
|
||||
assistantId,
|
||||
audioInputs,
|
||||
selectedDeviceId,
|
||||
selectDevice,
|
||||
connect,
|
||||
disconnect,
|
||||
}: {
|
||||
status: VoicePreviewStatus;
|
||||
error: string | null;
|
||||
micWarning: string | null;
|
||||
assistantId: string | null;
|
||||
audioInputs: MediaDeviceInfo[];
|
||||
selectedDeviceId: string;
|
||||
selectDevice: (deviceId: string) => void;
|
||||
connect: () => Promise<void>;
|
||||
disconnect: () => void;
|
||||
}) {
|
||||
const recording = status === "connecting" || status === "connected";
|
||||
const hint =
|
||||
status === "failed"
|
||||
? error || "连接失败,请确认后端已启动且助手已保存后重试。"
|
||||
: !assistantId
|
||||
? "请先保存助手,再开始语音预览。"
|
||||
: micWarning
|
||||
? `${micWarning} 可接收助手播报,但无法发送语音。`
|
||||
: null;
|
||||
: "测试语音识别、响应速度与助手的播报效果。";
|
||||
|
||||
return (
|
||||
<div className="shrink-0 border-t border-hairline px-3 py-2.5">
|
||||
<div className="flex items-center gap-2">
|
||||
<span
|
||||
className={[
|
||||
"h-1.5 w-1.5 shrink-0 rounded-full",
|
||||
recording
|
||||
? "animate-pulse bg-success"
|
||||
: status === "failed"
|
||||
? "bg-destructive"
|
||||
: "bg-muted-soft",
|
||||
].join(" ")}
|
||||
/>
|
||||
<span className="shrink-0 text-xs text-muted-foreground">
|
||||
{status === "connecting"
|
||||
? "连接中…"
|
||||
: status === "connected"
|
||||
? micWarning
|
||||
? "仅收听"
|
||||
: "进行中"
|
||||
: status === "failed"
|
||||
? "连接失败"
|
||||
: "准备开始"}
|
||||
</span>
|
||||
|
||||
<Select
|
||||
value={selectedDeviceId || "default"}
|
||||
onValueChange={(value) =>
|
||||
selectDevice(value === "default" ? "" : value)
|
||||
}
|
||||
>
|
||||
<SelectTrigger
|
||||
size="sm"
|
||||
className="min-w-0 flex-1 gap-2 rounded-full border-hairline bg-canvas-soft text-xs text-muted-foreground"
|
||||
aria-label="选择麦克风"
|
||||
>
|
||||
<Mic size={13} className="shrink-0 text-muted-soft" />
|
||||
<SelectValue placeholder="默认麦克风" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="default">默认麦克风</SelectItem>
|
||||
{audioInputs.map((device, index) => (
|
||||
<SelectItem key={device.deviceId} value={device.deviceId}>
|
||||
{device.label || `麦克风 ${index + 1}`}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
|
||||
<div className="flex min-h-0 flex-1 items-center justify-center px-6 py-8">
|
||||
<div className="flex max-w-xs flex-col items-center gap-3 text-center">
|
||||
<div className="flex h-12 w-12 items-center justify-center rounded-full bg-canvas-soft">
|
||||
<Mic size={22} className="text-muted-soft" />
|
||||
</div>
|
||||
<div className="font-display display-sm text-foreground">
|
||||
{status === "failed" ? "连接失败" : "开始一次语音对话"}
|
||||
</div>
|
||||
<p className="text-xs leading-5 text-muted-foreground">{hint}</p>
|
||||
<Button
|
||||
size="sm"
|
||||
disabled={!assistantId || status === "connecting"}
|
||||
onClick={() => {
|
||||
if (recording) {
|
||||
disconnect();
|
||||
} else {
|
||||
void connect();
|
||||
}
|
||||
}}
|
||||
className={[
|
||||
"shrink-0 gap-1.5 rounded-full",
|
||||
recording ? "bg-destructive text-white hover:bg-destructive/90" : "",
|
||||
].join(" ")}
|
||||
aria-label={recording ? "结束语音测试" : "开始语音测试"}
|
||||
disabled={!assistantId}
|
||||
onClick={() => void connect()}
|
||||
className="h-11 gap-2 rounded-full px-6 text-sm font-medium shadow-sm"
|
||||
aria-label={status === "failed" ? "重新连接" : "开始语音测试"}
|
||||
>
|
||||
{status === "connecting" ? (
|
||||
<Loader2 size={14} className="animate-spin" />
|
||||
) : recording ? (
|
||||
<PhoneOff size={14} />
|
||||
) : (
|
||||
<Mic size={14} />
|
||||
)}
|
||||
{recording ? "结束对话" : "开始对话"}
|
||||
<Mic size={18} />
|
||||
{status === "failed" ? "重新连接" : "开始对话"}
|
||||
</Button>
|
||||
<p className="text-[11px] leading-4 text-muted-soft">
|
||||
在右上角「设备」中选择麦克风
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
{hint && (
|
||||
<p className="mt-1.5 text-xs leading-5 text-muted-foreground">{hint}</p>
|
||||
function DebugVideoPanel({ camera }: { camera: CameraPreview }) {
|
||||
const { stream, error, starting, active, start } = camera;
|
||||
const videoRef = useRef<HTMLVideoElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (videoRef.current) videoRef.current.srcObject = stream;
|
||||
}, [stream]);
|
||||
|
||||
return (
|
||||
<div
|
||||
className={[
|
||||
"relative flex min-h-0 flex-1 items-center justify-center overflow-hidden",
|
||||
active ? "bg-black" : "bg-canvas-soft",
|
||||
].join(" ")}
|
||||
>
|
||||
<video
|
||||
ref={videoRef}
|
||||
autoPlay
|
||||
playsInline
|
||||
muted
|
||||
className={[
|
||||
"h-full w-full -scale-x-100 object-contain",
|
||||
active ? "" : "hidden",
|
||||
].join(" ")}
|
||||
/>
|
||||
{active ? (
|
||||
<Badge
|
||||
variant="secondary"
|
||||
className="absolute left-3 top-3 gap-1.5 rounded-full border border-hairline bg-card/80 px-2.5 py-1 text-[11px] font-medium text-muted-foreground shadow-none backdrop-blur"
|
||||
>
|
||||
<span className="h-1.5 w-1.5 animate-pulse rounded-full bg-success" />
|
||||
预览中
|
||||
</Badge>
|
||||
) : (
|
||||
<div className="flex max-w-xs flex-col items-center gap-3 px-6 text-center">
|
||||
<Video size={28} className="text-muted-soft" />
|
||||
<div className="text-sm font-medium text-foreground">
|
||||
{starting ? "正在开启摄像头…" : "摄像头未开启"}
|
||||
</div>
|
||||
<p className="text-xs leading-5 text-muted-foreground">
|
||||
{error ?? "开启摄像头预览画面。后续可将视频流接入后端视觉理解管线。"}
|
||||
</p>
|
||||
<Button
|
||||
size="sm"
|
||||
disabled={starting}
|
||||
onClick={() => void start()}
|
||||
className="gap-1.5 rounded-full"
|
||||
aria-label="开启摄像头"
|
||||
>
|
||||
{starting ? (
|
||||
<Loader2 size={14} className="animate-spin" />
|
||||
) : (
|
||||
<Video size={14} />
|
||||
)}
|
||||
开启摄像头
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function DebugConnectionStatus({
|
||||
status,
|
||||
micWarning,
|
||||
}: {
|
||||
status: VoicePreviewStatus;
|
||||
micWarning: string | null;
|
||||
}) {
|
||||
const recording = status === "connecting" || status === "connected";
|
||||
const label =
|
||||
status === "connecting"
|
||||
? "连接中…"
|
||||
: status === "connected"
|
||||
? micWarning
|
||||
? "仅收听"
|
||||
: "进行中"
|
||||
: status === "failed"
|
||||
? "连接失败"
|
||||
: "准备开始";
|
||||
|
||||
return (
|
||||
<span className="flex min-w-0 items-center gap-1.5 truncate text-xs text-muted-foreground">
|
||||
<span
|
||||
className={[
|
||||
"h-1.5 w-1.5 shrink-0 rounded-full",
|
||||
recording
|
||||
? "animate-pulse bg-success"
|
||||
: status === "failed"
|
||||
? "bg-destructive"
|
||||
: "bg-muted-soft",
|
||||
].join(" ")}
|
||||
/>
|
||||
{label}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
// ISO 时间戳 → HH:MM(本地时区),解析失败返回空串
|
||||
function formatMessageTime(iso: string): string {
|
||||
const d = new Date(iso);
|
||||
@@ -2291,10 +2519,10 @@ function formatMessageTime(iso: string): string {
|
||||
|
||||
function DebugTranscriptPanel({
|
||||
messages,
|
||||
recording,
|
||||
recording = false,
|
||||
}: {
|
||||
messages: ChatMessage[];
|
||||
recording: boolean;
|
||||
recording?: boolean;
|
||||
}) {
|
||||
const scrollRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
@@ -2306,16 +2534,23 @@ function DebugTranscriptPanel({
|
||||
|
||||
if (messages.length === 0) {
|
||||
return (
|
||||
<div className="flex min-h-0 flex-1 flex-col items-center justify-center gap-2 px-6 text-center">
|
||||
<MessageSquareText size={28} className="text-muted-soft" />
|
||||
<div className="text-sm font-medium text-foreground">
|
||||
{recording ? "暂无聊天记录" : "尚未开始对话"}
|
||||
<div
|
||||
className={[
|
||||
"flex min-h-0 flex-1 flex-col overflow-y-auto px-5 py-3",
|
||||
recording ? "items-center justify-center" : "pt-5",
|
||||
].join(" ")}
|
||||
>
|
||||
<div className="text-center">
|
||||
<MessageSquareText size={22} className="mx-auto text-muted-soft" />
|
||||
<div className="mt-2 text-sm font-medium text-foreground">
|
||||
{recording ? "暂无聊天记录" : "尚未开始对话"}
|
||||
</div>
|
||||
<p className="mx-auto mt-1 max-w-xs text-xs leading-5 text-muted-foreground">
|
||||
{recording
|
||||
? "开口说话或在下方输入文字,对话内容会实时显示在这里。"
|
||||
: "点击「开始对话」后,语音与文字消息会实时显示在这里。"}
|
||||
</p>
|
||||
</div>
|
||||
<p className="max-w-xs text-xs leading-5 text-muted-foreground">
|
||||
{recording
|
||||
? "开口说话或在下方输入文字,对话内容会实时显示在这里。"
|
||||
: "点击「开始对话」后,语音与文字消息会实时显示在这里。"}
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
113
frontend/src/hooks/use-camera-preview.ts
Normal file
113
frontend/src/hooks/use-camera-preview.ts
Normal file
@@ -0,0 +1,113 @@
|
||||
"use client";
|
||||
|
||||
/**
|
||||
* 摄像头预览:纯前端 getUserMedia 视频流,供调试抽屉的「视频流」视图使用。
|
||||
*
|
||||
* 与 use-voice-preview 分离:麦克风/语音会话走 WebRTC 到后端,摄像头目前只在
|
||||
* 本地预览。后续接入视觉理解管线时,把这条 video track 加进现有 PeerConnection
|
||||
* (pc.addTrack)并在后端 transport 打开 video_in_enabled 即可。
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
|
||||
function cameraErrorMessage(error: unknown): string {
|
||||
if (error instanceof DOMException) {
|
||||
if (error.name === "NotAllowedError")
|
||||
return "摄像头权限被拒绝,请在浏览器网站设置中允许使用摄像头。";
|
||||
if (error.name === "NotFoundError")
|
||||
return "未检测到可用摄像头,请连接或启用摄像头后重试。";
|
||||
if (error.name === "NotReadableError")
|
||||
return "摄像头正被其他应用占用,或系统未允许浏览器访问摄像头。";
|
||||
}
|
||||
if (error instanceof Error && error.message) return error.message;
|
||||
return "无法访问摄像头。";
|
||||
}
|
||||
|
||||
export function useCameraPreview() {
|
||||
const [stream, setStream] = useState<MediaStream | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [starting, setStarting] = useState(false);
|
||||
const [devices, setDevices] = useState<MediaDeviceInfo[]>([]);
|
||||
const [deviceId, setDeviceId] = useState("");
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const deviceIdRef = useRef("");
|
||||
|
||||
const active = Boolean(stream);
|
||||
|
||||
const refreshCameras = useCallback(async () => {
|
||||
if (!navigator.mediaDevices?.enumerateDevices) return;
|
||||
try {
|
||||
const list = await navigator.mediaDevices.enumerateDevices();
|
||||
setDevices(list.filter((d) => d.kind === "videoinput" && d.deviceId));
|
||||
} catch {
|
||||
/* 忽略枚举失败 */
|
||||
}
|
||||
}, []);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
streamRef.current?.getTracks().forEach((track) => track.stop());
|
||||
streamRef.current = null;
|
||||
setStream(null);
|
||||
}, []);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (streamRef.current) return;
|
||||
setError(null);
|
||||
if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
|
||||
setError("当前页面无法访问摄像头,请通过 https 或 localhost 访问。");
|
||||
return;
|
||||
}
|
||||
setStarting(true);
|
||||
try {
|
||||
const id = deviceIdRef.current;
|
||||
const media = await navigator.mediaDevices.getUserMedia({
|
||||
video: id ? { deviceId: { exact: id } } : true,
|
||||
});
|
||||
streamRef.current = media;
|
||||
setStream(media);
|
||||
void refreshCameras();
|
||||
} catch (mediaError) {
|
||||
setError(cameraErrorMessage(mediaError));
|
||||
} finally {
|
||||
setStarting(false);
|
||||
}
|
||||
}, [refreshCameras]);
|
||||
|
||||
const selectCamera = useCallback(
|
||||
async (id: string) => {
|
||||
deviceIdRef.current = id;
|
||||
setDeviceId(id);
|
||||
if (streamRef.current) {
|
||||
stop();
|
||||
await start();
|
||||
}
|
||||
},
|
||||
[start, stop],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!navigator.mediaDevices?.enumerateDevices) return;
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||||
void refreshCameras();
|
||||
navigator.mediaDevices.addEventListener("devicechange", refreshCameras);
|
||||
return () => {
|
||||
navigator.mediaDevices.removeEventListener("devicechange", refreshCameras);
|
||||
};
|
||||
}, [refreshCameras]);
|
||||
|
||||
useEffect(() => stop, [stop]);
|
||||
|
||||
return {
|
||||
stream,
|
||||
error,
|
||||
starting,
|
||||
devices,
|
||||
deviceId,
|
||||
active,
|
||||
start,
|
||||
stop,
|
||||
selectCamera,
|
||||
};
|
||||
}
|
||||
|
||||
export type CameraPreview = ReturnType<typeof useCameraPreview>;
|
||||
@@ -558,3 +558,5 @@ export function useVoicePreview(
|
||||
audioRef,
|
||||
};
|
||||
}
|
||||
|
||||
export type VoicePreview = ReturnType<typeof useVoicePreview>;
|
||||
|
||||
Reference in New Issue
Block a user