Implement camera preview functionality and enhance AssistantPage
- Add a new `useCameraPreview` hook for managing camera access and video stream handling. - Integrate camera preview capabilities into the AssistantPage, allowing users to toggle video stream visibility. - Introduce a new toggle for enabling visual understanding, which activates the video stream preview alongside voice functionalities. - Update the DebugDrawer component to support switching between chat and video views based on the visual understanding setting. - Refactor related components to accommodate the new camera features and improve user interaction during debugging.
This commit is contained in:
@@ -32,6 +32,8 @@ import {
|
|||||||
Orbit,
|
Orbit,
|
||||||
Waves,
|
Waves,
|
||||||
Bug,
|
Bug,
|
||||||
|
Video,
|
||||||
|
SlidersHorizontal,
|
||||||
} from "lucide-react";
|
} from "lucide-react";
|
||||||
|
|
||||||
import { Button } from "@/components/ui/button";
|
import { Button } from "@/components/ui/button";
|
||||||
@@ -95,8 +97,13 @@ import {
|
|||||||
import {
|
import {
|
||||||
useVoicePreview,
|
useVoicePreview,
|
||||||
type ChatMessage,
|
type ChatMessage,
|
||||||
|
type VoicePreview,
|
||||||
type VoicePreviewStatus,
|
type VoicePreviewStatus,
|
||||||
} from "@/hooks/use-voice-preview";
|
} from "@/hooks/use-voice-preview";
|
||||||
|
import {
|
||||||
|
useCameraPreview,
|
||||||
|
type CameraPreview,
|
||||||
|
} from "@/hooks/use-camera-preview";
|
||||||
import {
|
import {
|
||||||
WorkflowEditor,
|
WorkflowEditor,
|
||||||
type WorkflowSettings,
|
type WorkflowSettings,
|
||||||
@@ -325,6 +332,8 @@ export function AssistantPage(props: AssistantPageProps) {
|
|||||||
const editingId = props.mode === "edit" ? props.assistantId : null;
|
const editingId = props.mode === "edit" ? props.assistantId : null;
|
||||||
|
|
||||||
const [form, setForm] = useState<AssistantForm>(() => blankPromptForm(""));
|
const [form, setForm] = useState<AssistantForm>(() => blankPromptForm(""));
|
||||||
|
// 视觉理解(暂不持久化,不进 savedSnapshot,避免影响「未保存改动」判定)
|
||||||
|
const [visionEnabled, setVisionEnabled] = useState(false);
|
||||||
const [fastGptForm, setFastGptForm] = useState<FastGptForm>(() =>
|
const [fastGptForm, setFastGptForm] = useState<FastGptForm>(() =>
|
||||||
blankFastGptForm(""),
|
blankFastGptForm(""),
|
||||||
);
|
);
|
||||||
@@ -1716,6 +1725,14 @@ export function AssistantPage(props: AssistantPageProps) {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<div className="mt-4 border-t border-hairline pt-4">
|
||||||
|
<ToggleRow
|
||||||
|
title="视觉理解"
|
||||||
|
description="开启后,右侧调试面板可切换到视频流预览,并可同时选择麦克风与摄像头"
|
||||||
|
checked={visionEnabled}
|
||||||
|
onChange={setVisionEnabled}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
</SectionCard>
|
</SectionCard>
|
||||||
|
|
||||||
<SectionCard
|
<SectionCard
|
||||||
@@ -1814,7 +1831,7 @@ export function AssistantPage(props: AssistantPageProps) {
|
|||||||
</SectionCard>
|
</SectionCard>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<DebugDrawer assistantId={editingId} />
|
<DebugDrawer assistantId={editingId} vision={visionEnabled} />
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
@@ -1822,6 +1839,9 @@ export function AssistantPage(props: AssistantPageProps) {
|
|||||||
|
|
||||||
type VizStyle = "aura" | "nebula" | "bars" | "wave";
|
type VizStyle = "aura" | "nebula" | "bars" | "wave";
|
||||||
|
|
||||||
|
// 调试面板顶部主视图:聊天记录 / 视频流(仅视觉理解开启时可选)
|
||||||
|
type DebugView = "chat" | "video";
|
||||||
|
|
||||||
const VIZ_OPTIONS: { style: VizStyle; label: string; icon: React.ReactNode }[] =
|
const VIZ_OPTIONS: { style: VizStyle; label: string; icon: React.ReactNode }[] =
|
||||||
[
|
[
|
||||||
{ style: "aura", label: "光环", icon: <Orbit size={14} /> },
|
{ style: "aura", label: "光环", icon: <Orbit size={14} /> },
|
||||||
@@ -1886,15 +1906,50 @@ function DebugDrawer({
|
|||||||
assistantId,
|
assistantId,
|
||||||
asSheet = false,
|
asSheet = false,
|
||||||
onNodeActive,
|
onNodeActive,
|
||||||
|
vision = false,
|
||||||
}: {
|
}: {
|
||||||
assistantId: string | null;
|
assistantId: string | null;
|
||||||
asSheet?: boolean;
|
asSheet?: boolean;
|
||||||
onNodeActive?: (nodeId: string | null) => void;
|
onNodeActive?: (nodeId: string | null) => void;
|
||||||
|
vision?: boolean;
|
||||||
}) {
|
}) {
|
||||||
|
const preview = useVoicePreview(assistantId, onNodeActive);
|
||||||
|
const camera = useCameraPreview();
|
||||||
const [showTranscript, setShowTranscript] = useState(false);
|
const [showTranscript, setShowTranscript] = useState(false);
|
||||||
const [vizStyle, setVizStyle] = useState<VizStyle>("aura");
|
const [vizStyle, setVizStyle] = useState<VizStyle>("aura");
|
||||||
|
const [view, setView] = useState<DebugView>(vision ? "video" : "chat");
|
||||||
|
const effectiveView: DebugView = vision ? view : "chat";
|
||||||
|
const stopCamera = camera.stop;
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||||||
|
setView(vision ? "video" : "chat");
|
||||||
|
if (!vision) stopCamera();
|
||||||
|
}, [vision, stopCamera]);
|
||||||
|
|
||||||
|
const { start: startCamera, active: cameraActive, starting: cameraStarting } =
|
||||||
|
camera;
|
||||||
|
useEffect(() => {
|
||||||
|
if (
|
||||||
|
vision &&
|
||||||
|
effectiveView === "video" &&
|
||||||
|
!cameraActive &&
|
||||||
|
!cameraStarting
|
||||||
|
) {
|
||||||
|
void startCamera();
|
||||||
|
}
|
||||||
|
}, [
|
||||||
|
vision,
|
||||||
|
effectiveView,
|
||||||
|
cameraActive,
|
||||||
|
cameraStarting,
|
||||||
|
startCamera,
|
||||||
|
]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (effectiveView !== "video") stopCamera();
|
||||||
|
}, [effectiveView, stopCamera]);
|
||||||
|
|
||||||
// 内联(prompt 编辑器右栏)用 aside + 圆角边框;抽屉模式占满 Sheet。
|
|
||||||
const containerClass = asSheet
|
const containerClass = asSheet
|
||||||
? "flex h-full min-w-0 flex-1 flex-col overflow-hidden"
|
? "flex h-full min-w-0 flex-1 flex-col overflow-hidden"
|
||||||
: "hidden min-w-0 flex-1 flex-col overflow-hidden rounded-2xl border border-hairline bg-card shadow-sm lg:flex";
|
: "hidden min-w-0 flex-1 flex-col overflow-hidden rounded-2xl border border-hairline bg-card shadow-sm lg:flex";
|
||||||
@@ -1902,63 +1957,204 @@ function DebugDrawer({
|
|||||||
return (
|
return (
|
||||||
<aside className={containerClass}>
|
<aside className={containerClass}>
|
||||||
<div className="flex shrink-0 items-center justify-between gap-3 border-b border-hairline px-5 py-3">
|
<div className="flex shrink-0 items-center justify-between gap-3 border-b border-hairline px-5 py-3">
|
||||||
<div className="text-sm font-medium text-foreground">调试与预览</div>
|
<div className="flex min-w-0 items-center gap-2.5">
|
||||||
{SHOW_VOICE_VIZ && (
|
<div className="shrink-0 text-sm font-medium text-foreground">
|
||||||
<div className="flex items-center gap-2">
|
调试与预览
|
||||||
{!showTranscript && (
|
|
||||||
<SegmentedIconGroup label="可视化样式">
|
|
||||||
{VIZ_OPTIONS.map((option) => (
|
|
||||||
<SegmentedIconButton
|
|
||||||
key={option.style}
|
|
||||||
selected={vizStyle === option.style}
|
|
||||||
label={`可视化样式:${option.label}`}
|
|
||||||
onClick={() => setVizStyle(option.style)}
|
|
||||||
>
|
|
||||||
{option.icon}
|
|
||||||
</SegmentedIconButton>
|
|
||||||
))}
|
|
||||||
</SegmentedIconGroup>
|
|
||||||
)}
|
|
||||||
<SegmentedIconGroup label="预览视图">
|
|
||||||
<SegmentedIconButton
|
|
||||||
selected={!showTranscript}
|
|
||||||
label="语音可视化视图"
|
|
||||||
onClick={() => setShowTranscript(false)}
|
|
||||||
>
|
|
||||||
<Mic size={14} />
|
|
||||||
</SegmentedIconButton>
|
|
||||||
<SegmentedIconButton
|
|
||||||
selected={showTranscript}
|
|
||||||
label="文字聊天记录视图"
|
|
||||||
onClick={() => setShowTranscript(true)}
|
|
||||||
>
|
|
||||||
<MessageSquareText size={14} />
|
|
||||||
</SegmentedIconButton>
|
|
||||||
</SegmentedIconGroup>
|
|
||||||
</div>
|
</div>
|
||||||
)}
|
<DebugConnectionStatus
|
||||||
|
status={preview.status}
|
||||||
|
micWarning={preview.micWarning}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div className="flex shrink-0 items-center gap-2">
|
||||||
|
{SHOW_VOICE_VIZ && effectiveView === "chat" && (
|
||||||
|
<>
|
||||||
|
{!showTranscript && (
|
||||||
|
<SegmentedIconGroup label="可视化样式">
|
||||||
|
{VIZ_OPTIONS.map((option) => (
|
||||||
|
<SegmentedIconButton
|
||||||
|
key={option.style}
|
||||||
|
selected={vizStyle === option.style}
|
||||||
|
label={`可视化样式:${option.label}`}
|
||||||
|
onClick={() => setVizStyle(option.style)}
|
||||||
|
>
|
||||||
|
{option.icon}
|
||||||
|
</SegmentedIconButton>
|
||||||
|
))}
|
||||||
|
</SegmentedIconGroup>
|
||||||
|
)}
|
||||||
|
<SegmentedIconGroup label="语音视图">
|
||||||
|
<SegmentedIconButton
|
||||||
|
selected={!showTranscript}
|
||||||
|
label="语音可视化视图"
|
||||||
|
onClick={() => setShowTranscript(false)}
|
||||||
|
>
|
||||||
|
<Mic size={14} />
|
||||||
|
</SegmentedIconButton>
|
||||||
|
<SegmentedIconButton
|
||||||
|
selected={showTranscript}
|
||||||
|
label="文字聊天记录视图"
|
||||||
|
onClick={() => setShowTranscript(true)}
|
||||||
|
>
|
||||||
|
<MessageSquareText size={14} />
|
||||||
|
</SegmentedIconButton>
|
||||||
|
</SegmentedIconGroup>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
<DeviceMenu preview={preview} camera={camera} vision={vision} />
|
||||||
|
<SegmentedIconGroup label="预览视图">
|
||||||
|
<SegmentedIconButton
|
||||||
|
selected={effectiveView === "chat"}
|
||||||
|
label="聊天记录"
|
||||||
|
onClick={() => setView("chat")}
|
||||||
|
>
|
||||||
|
<MessageSquareText size={14} />
|
||||||
|
</SegmentedIconButton>
|
||||||
|
{vision && (
|
||||||
|
<SegmentedIconButton
|
||||||
|
selected={effectiveView === "video"}
|
||||||
|
label="视频流"
|
||||||
|
onClick={() => setView("video")}
|
||||||
|
>
|
||||||
|
<Video size={14} />
|
||||||
|
</SegmentedIconButton>
|
||||||
|
)}
|
||||||
|
</SegmentedIconGroup>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<DebugVoicePanel
|
<DebugVoicePanel
|
||||||
|
view={effectiveView}
|
||||||
showTranscript={showTranscript}
|
showTranscript={showTranscript}
|
||||||
vizStyle={vizStyle}
|
vizStyle={vizStyle}
|
||||||
assistantId={assistantId}
|
assistantId={assistantId}
|
||||||
onNodeActive={onNodeActive}
|
preview={preview}
|
||||||
|
camera={camera}
|
||||||
/>
|
/>
|
||||||
</aside>
|
</aside>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function DeviceMenu({
|
||||||
|
preview,
|
||||||
|
camera,
|
||||||
|
vision,
|
||||||
|
}: {
|
||||||
|
preview: VoicePreview;
|
||||||
|
camera: CameraPreview;
|
||||||
|
vision: boolean;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<Popover>
|
||||||
|
<PopoverTrigger asChild>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
aria-label="设备设置"
|
||||||
|
title="设备设置"
|
||||||
|
className="flex h-8 items-center gap-1.5 rounded-full border border-hairline bg-canvas-soft px-2.5 text-xs text-muted-soft transition-colors hover:text-foreground"
|
||||||
|
>
|
||||||
|
<SlidersHorizontal size={14} />
|
||||||
|
<span className="hidden sm:inline">设备</span>
|
||||||
|
</button>
|
||||||
|
</PopoverTrigger>
|
||||||
|
<PopoverContent align="end" className="w-72 p-0">
|
||||||
|
<div className="border-b border-hairline px-3 py-2.5">
|
||||||
|
<div className="text-xs font-medium text-foreground">输入设备</div>
|
||||||
|
<p className="mt-0.5 text-[11px] leading-4 text-muted-foreground">
|
||||||
|
{vision
|
||||||
|
? "选择麦克风与摄像头,分别用于语音与视觉预览。"
|
||||||
|
: "选择麦克风用于语音对话。"}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div className="space-y-3 p-3">
|
||||||
|
<DeviceSelectRow
|
||||||
|
icon={<Mic size={13} className="shrink-0 text-muted-soft" />}
|
||||||
|
label="麦克风"
|
||||||
|
placeholder="默认麦克风"
|
||||||
|
fallbackLabel="麦克风"
|
||||||
|
value={preview.selectedDeviceId}
|
||||||
|
devices={preview.audioInputs}
|
||||||
|
onSelect={(id) => preview.selectDevice(id)}
|
||||||
|
/>
|
||||||
|
{vision && (
|
||||||
|
<>
|
||||||
|
<div className="border-t border-hairline" />
|
||||||
|
<DeviceSelectRow
|
||||||
|
icon={<Video size={13} className="shrink-0 text-muted-soft" />}
|
||||||
|
label="摄像头"
|
||||||
|
placeholder="默认摄像头"
|
||||||
|
fallbackLabel="摄像头"
|
||||||
|
value={camera.deviceId}
|
||||||
|
devices={camera.devices}
|
||||||
|
onSelect={(id) => void camera.selectCamera(id)}
|
||||||
|
/>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</PopoverContent>
|
||||||
|
</Popover>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function DeviceSelectRow({
|
||||||
|
icon,
|
||||||
|
label,
|
||||||
|
placeholder,
|
||||||
|
fallbackLabel,
|
||||||
|
value,
|
||||||
|
devices,
|
||||||
|
onSelect,
|
||||||
|
}: {
|
||||||
|
icon: React.ReactNode;
|
||||||
|
label: string;
|
||||||
|
placeholder: string;
|
||||||
|
fallbackLabel: string;
|
||||||
|
value: string;
|
||||||
|
devices: MediaDeviceInfo[];
|
||||||
|
onSelect: (deviceId: string) => void;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<div className="space-y-1.5">
|
||||||
|
<div className="text-xs font-medium text-muted-foreground">{label}</div>
|
||||||
|
<Select
|
||||||
|
value={value || "default"}
|
||||||
|
onValueChange={(v) => onSelect(v === "default" ? "" : v)}
|
||||||
|
>
|
||||||
|
<SelectTrigger
|
||||||
|
size="sm"
|
||||||
|
className="w-full gap-2 rounded-full border-hairline bg-canvas-soft text-xs text-muted-foreground"
|
||||||
|
aria-label={`选择${label}`}
|
||||||
|
>
|
||||||
|
{icon}
|
||||||
|
<SelectValue placeholder={placeholder} />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
<SelectItem value="default">{placeholder}</SelectItem>
|
||||||
|
{devices.map((device, index) => (
|
||||||
|
<SelectItem key={device.deviceId} value={device.deviceId}>
|
||||||
|
{device.label || `${fallbackLabel} ${index + 1}`}
|
||||||
|
</SelectItem>
|
||||||
|
))}
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function DebugVoicePanel({
|
function DebugVoicePanel({
|
||||||
|
view,
|
||||||
showTranscript,
|
showTranscript,
|
||||||
vizStyle,
|
vizStyle,
|
||||||
assistantId,
|
assistantId,
|
||||||
onNodeActive,
|
preview,
|
||||||
|
camera,
|
||||||
}: {
|
}: {
|
||||||
|
view: DebugView;
|
||||||
showTranscript: boolean;
|
showTranscript: boolean;
|
||||||
vizStyle: VizStyle;
|
vizStyle: VizStyle;
|
||||||
assistantId: string | null;
|
assistantId: string | null;
|
||||||
onNodeActive?: (nodeId: string | null) => void;
|
preview: VoicePreview;
|
||||||
|
camera: CameraPreview;
|
||||||
}) {
|
}) {
|
||||||
const {
|
const {
|
||||||
status,
|
status,
|
||||||
@@ -1967,17 +2163,18 @@ function DebugVoicePanel({
|
|||||||
localStream,
|
localStream,
|
||||||
remoteStream,
|
remoteStream,
|
||||||
messages,
|
messages,
|
||||||
audioInputs,
|
|
||||||
selectedDeviceId,
|
|
||||||
selectDevice,
|
|
||||||
sendText,
|
sendText,
|
||||||
connect,
|
connect,
|
||||||
disconnect,
|
disconnect,
|
||||||
audioRef,
|
audioRef,
|
||||||
} = useVoicePreview(assistantId, onNodeActive);
|
} = preview;
|
||||||
// 连接中或已连通都视作"会话进行中"
|
|
||||||
const recording = status === "connecting" || status === "connected";
|
const recording = status === "connecting" || status === "connected";
|
||||||
const [textDraft, setTextDraft] = useState("");
|
const [textDraft, setTextDraft] = useState("");
|
||||||
|
const inChatView = view === "chat" && (!SHOW_VOICE_VIZ || showTranscript);
|
||||||
|
const showIdleHub =
|
||||||
|
inChatView &&
|
||||||
|
messages.length === 0 &&
|
||||||
|
(status === "idle" || status === "failed");
|
||||||
|
|
||||||
function handleSendText() {
|
function handleSendText() {
|
||||||
if (sendText(textDraft)) {
|
if (sendText(textDraft)) {
|
||||||
@@ -1989,21 +2186,19 @@ function DebugVoicePanel({
|
|||||||
<div className="flex min-h-0 flex-1 flex-col">
|
<div className="flex min-h-0 flex-1 flex-col">
|
||||||
{/* 后端 TTS 音频经 WebRTC 媒体流过来,挂这里播放 */}
|
{/* 后端 TTS 音频经 WebRTC 媒体流过来,挂这里播放 */}
|
||||||
<audio ref={audioRef} autoPlay playsInline className="hidden" />
|
<audio ref={audioRef} autoPlay playsInline className="hidden" />
|
||||||
{!SHOW_VOICE_VIZ || showTranscript ? (
|
{view === "video" ? (
|
||||||
<>
|
<DebugVideoPanel camera={camera} />
|
||||||
<DebugTranscriptPanel messages={messages} recording={recording} />
|
) : !SHOW_VOICE_VIZ || showTranscript ? (
|
||||||
<VoiceSessionControls
|
showIdleHub ? (
|
||||||
|
<DebugIdleHub
|
||||||
status={status}
|
status={status}
|
||||||
error={error}
|
error={error}
|
||||||
micWarning={micWarning}
|
|
||||||
assistantId={assistantId}
|
assistantId={assistantId}
|
||||||
audioInputs={audioInputs}
|
|
||||||
selectedDeviceId={selectedDeviceId}
|
|
||||||
selectDevice={selectDevice}
|
|
||||||
connect={connect}
|
connect={connect}
|
||||||
disconnect={disconnect}
|
|
||||||
/>
|
/>
|
||||||
</>
|
) : (
|
||||||
|
<DebugTranscriptPanel messages={messages} recording={recording} />
|
||||||
|
)
|
||||||
) : (
|
) : (
|
||||||
<div className="relative flex min-h-0 flex-1 flex-col items-center justify-center gap-3 overflow-y-auto px-6 py-3 text-center">
|
<div className="relative flex min-h-0 flex-1 flex-col items-center justify-center gap-3 overflow-y-auto px-6 py-3 text-center">
|
||||||
<div
|
<div
|
||||||
@@ -2068,32 +2263,6 @@ function DebugVoicePanel({
|
|||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="relative w-full max-w-xs">
|
|
||||||
<Select
|
|
||||||
value={selectedDeviceId || "default"}
|
|
||||||
onValueChange={(value) =>
|
|
||||||
selectDevice(value === "default" ? "" : value)
|
|
||||||
}
|
|
||||||
>
|
|
||||||
<SelectTrigger
|
|
||||||
size="sm"
|
|
||||||
className="w-full justify-center gap-2 rounded-full border-hairline bg-canvas-soft text-xs text-muted-foreground"
|
|
||||||
aria-label="选择麦克风"
|
|
||||||
>
|
|
||||||
<Mic size={13} className="shrink-0 text-muted-soft" />
|
|
||||||
<SelectValue placeholder="默认麦克风" />
|
|
||||||
</SelectTrigger>
|
|
||||||
<SelectContent>
|
|
||||||
<SelectItem value="default">默认麦克风</SelectItem>
|
|
||||||
{audioInputs.map((device, index) => (
|
|
||||||
<SelectItem key={device.deviceId} value={device.deviceId}>
|
|
||||||
{device.label || `麦克风 ${index + 1}`}
|
|
||||||
</SelectItem>
|
|
||||||
))}
|
|
||||||
</SelectContent>
|
|
||||||
</Select>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<Button
|
<Button
|
||||||
disabled={!assistantId || status === "connecting"}
|
disabled={!assistantId || status === "connecting"}
|
||||||
onClick={() => {
|
onClick={() => {
|
||||||
@@ -2152,11 +2321,42 @@ function DebugVoicePanel({
|
|||||||
>
|
>
|
||||||
<Send size={16} />
|
<Send size={16} />
|
||||||
</Button>
|
</Button>
|
||||||
|
{!showIdleHub && (
|
||||||
|
<Button
|
||||||
|
size="sm"
|
||||||
|
disabled={!assistantId || status === "connecting"}
|
||||||
|
onClick={() => {
|
||||||
|
if (recording) {
|
||||||
|
disconnect();
|
||||||
|
} else {
|
||||||
|
void connect();
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
className={[
|
||||||
|
"shrink-0 gap-1.5 rounded-full",
|
||||||
|
recording
|
||||||
|
? "bg-destructive text-white hover:bg-destructive/90"
|
||||||
|
: "",
|
||||||
|
].join(" ")}
|
||||||
|
aria-label={recording ? "结束语音测试" : "开始语音测试"}
|
||||||
|
>
|
||||||
|
{status === "connecting" ? (
|
||||||
|
<Loader2 size={14} className="animate-spin" />
|
||||||
|
) : recording ? (
|
||||||
|
<PhoneOff size={14} />
|
||||||
|
) : (
|
||||||
|
<Mic size={14} />
|
||||||
|
)}
|
||||||
|
{recording ? "结束对话" : "开始对话"}
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* 底部双轨波形:用户麦克风 + 助手音频,可折叠 */}
|
{/* 底部双轨波形:会话连通后默认展开,空闲时收起 */}
|
||||||
<WaveformTimelinePanel
|
<WaveformTimelinePanel
|
||||||
|
key={status === "connected" ? "active" : "idle"}
|
||||||
|
defaultOpen={status === "connected"}
|
||||||
userStream={localStream}
|
userStream={localStream}
|
||||||
agentStream={remoteStream}
|
agentStream={remoteStream}
|
||||||
active={status === "connected"}
|
active={status === "connected"}
|
||||||
@@ -2165,122 +2365,150 @@ function DebugVoicePanel({
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 会话控制条:状态 + 麦克风选择 + 开始/结束按钮。
|
// 空闲态:图标 + 说明 + 开始按钮合一,替代「撑满高度的空 transcript + 底部细控制条」
|
||||||
// 原本这些控件在中央可视化视图里,可视化隐藏后(SHOW_VOICE_VIZ=false)集中到这一条。
|
function DebugIdleHub({
|
||||||
function VoiceSessionControls({
|
|
||||||
status,
|
status,
|
||||||
error,
|
error,
|
||||||
micWarning,
|
|
||||||
assistantId,
|
assistantId,
|
||||||
audioInputs,
|
|
||||||
selectedDeviceId,
|
|
||||||
selectDevice,
|
|
||||||
connect,
|
connect,
|
||||||
disconnect,
|
|
||||||
}: {
|
}: {
|
||||||
status: VoicePreviewStatus;
|
status: VoicePreviewStatus;
|
||||||
error: string | null;
|
error: string | null;
|
||||||
micWarning: string | null;
|
|
||||||
assistantId: string | null;
|
assistantId: string | null;
|
||||||
audioInputs: MediaDeviceInfo[];
|
|
||||||
selectedDeviceId: string;
|
|
||||||
selectDevice: (deviceId: string) => void;
|
|
||||||
connect: () => Promise<void>;
|
connect: () => Promise<void>;
|
||||||
disconnect: () => void;
|
|
||||||
}) {
|
}) {
|
||||||
const recording = status === "connecting" || status === "connected";
|
|
||||||
const hint =
|
const hint =
|
||||||
status === "failed"
|
status === "failed"
|
||||||
? error || "连接失败,请确认后端已启动且助手已保存后重试。"
|
? error || "连接失败,请确认后端已启动且助手已保存后重试。"
|
||||||
: !assistantId
|
: !assistantId
|
||||||
? "请先保存助手,再开始语音预览。"
|
? "请先保存助手,再开始语音预览。"
|
||||||
: micWarning
|
: "测试语音识别、响应速度与助手的播报效果。";
|
||||||
? `${micWarning} 可接收助手播报,但无法发送语音。`
|
|
||||||
: null;
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="shrink-0 border-t border-hairline px-3 py-2.5">
|
<div className="flex min-h-0 flex-1 items-center justify-center px-6 py-8">
|
||||||
<div className="flex items-center gap-2">
|
<div className="flex max-w-xs flex-col items-center gap-3 text-center">
|
||||||
<span
|
<div className="flex h-12 w-12 items-center justify-center rounded-full bg-canvas-soft">
|
||||||
className={[
|
<Mic size={22} className="text-muted-soft" />
|
||||||
"h-1.5 w-1.5 shrink-0 rounded-full",
|
</div>
|
||||||
recording
|
<div className="font-display display-sm text-foreground">
|
||||||
? "animate-pulse bg-success"
|
{status === "failed" ? "连接失败" : "开始一次语音对话"}
|
||||||
: status === "failed"
|
</div>
|
||||||
? "bg-destructive"
|
<p className="text-xs leading-5 text-muted-foreground">{hint}</p>
|
||||||
: "bg-muted-soft",
|
|
||||||
].join(" ")}
|
|
||||||
/>
|
|
||||||
<span className="shrink-0 text-xs text-muted-foreground">
|
|
||||||
{status === "connecting"
|
|
||||||
? "连接中…"
|
|
||||||
: status === "connected"
|
|
||||||
? micWarning
|
|
||||||
? "仅收听"
|
|
||||||
: "进行中"
|
|
||||||
: status === "failed"
|
|
||||||
? "连接失败"
|
|
||||||
: "准备开始"}
|
|
||||||
</span>
|
|
||||||
|
|
||||||
<Select
|
|
||||||
value={selectedDeviceId || "default"}
|
|
||||||
onValueChange={(value) =>
|
|
||||||
selectDevice(value === "default" ? "" : value)
|
|
||||||
}
|
|
||||||
>
|
|
||||||
<SelectTrigger
|
|
||||||
size="sm"
|
|
||||||
className="min-w-0 flex-1 gap-2 rounded-full border-hairline bg-canvas-soft text-xs text-muted-foreground"
|
|
||||||
aria-label="选择麦克风"
|
|
||||||
>
|
|
||||||
<Mic size={13} className="shrink-0 text-muted-soft" />
|
|
||||||
<SelectValue placeholder="默认麦克风" />
|
|
||||||
</SelectTrigger>
|
|
||||||
<SelectContent>
|
|
||||||
<SelectItem value="default">默认麦克风</SelectItem>
|
|
||||||
{audioInputs.map((device, index) => (
|
|
||||||
<SelectItem key={device.deviceId} value={device.deviceId}>
|
|
||||||
{device.label || `麦克风 ${index + 1}`}
|
|
||||||
</SelectItem>
|
|
||||||
))}
|
|
||||||
</SelectContent>
|
|
||||||
</Select>
|
|
||||||
|
|
||||||
<Button
|
<Button
|
||||||
size="sm"
|
disabled={!assistantId}
|
||||||
disabled={!assistantId || status === "connecting"}
|
onClick={() => void connect()}
|
||||||
onClick={() => {
|
className="h-11 gap-2 rounded-full px-6 text-sm font-medium shadow-sm"
|
||||||
if (recording) {
|
aria-label={status === "failed" ? "重新连接" : "开始语音测试"}
|
||||||
disconnect();
|
|
||||||
} else {
|
|
||||||
void connect();
|
|
||||||
}
|
|
||||||
}}
|
|
||||||
className={[
|
|
||||||
"shrink-0 gap-1.5 rounded-full",
|
|
||||||
recording ? "bg-destructive text-white hover:bg-destructive/90" : "",
|
|
||||||
].join(" ")}
|
|
||||||
aria-label={recording ? "结束语音测试" : "开始语音测试"}
|
|
||||||
>
|
>
|
||||||
{status === "connecting" ? (
|
<Mic size={18} />
|
||||||
<Loader2 size={14} className="animate-spin" />
|
{status === "failed" ? "重新连接" : "开始对话"}
|
||||||
) : recording ? (
|
|
||||||
<PhoneOff size={14} />
|
|
||||||
) : (
|
|
||||||
<Mic size={14} />
|
|
||||||
)}
|
|
||||||
{recording ? "结束对话" : "开始对话"}
|
|
||||||
</Button>
|
</Button>
|
||||||
|
<p className="text-[11px] leading-4 text-muted-soft">
|
||||||
|
在右上角「设备」中选择麦克风
|
||||||
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
{hint && (
|
function DebugVideoPanel({ camera }: { camera: CameraPreview }) {
|
||||||
<p className="mt-1.5 text-xs leading-5 text-muted-foreground">{hint}</p>
|
const { stream, error, starting, active, start } = camera;
|
||||||
|
const videoRef = useRef<HTMLVideoElement>(null);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (videoRef.current) videoRef.current.srcObject = stream;
|
||||||
|
}, [stream]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className={[
|
||||||
|
"relative flex min-h-0 flex-1 items-center justify-center overflow-hidden",
|
||||||
|
active ? "bg-black" : "bg-canvas-soft",
|
||||||
|
].join(" ")}
|
||||||
|
>
|
||||||
|
<video
|
||||||
|
ref={videoRef}
|
||||||
|
autoPlay
|
||||||
|
playsInline
|
||||||
|
muted
|
||||||
|
className={[
|
||||||
|
"h-full w-full -scale-x-100 object-contain",
|
||||||
|
active ? "" : "hidden",
|
||||||
|
].join(" ")}
|
||||||
|
/>
|
||||||
|
{active ? (
|
||||||
|
<Badge
|
||||||
|
variant="secondary"
|
||||||
|
className="absolute left-3 top-3 gap-1.5 rounded-full border border-hairline bg-card/80 px-2.5 py-1 text-[11px] font-medium text-muted-foreground shadow-none backdrop-blur"
|
||||||
|
>
|
||||||
|
<span className="h-1.5 w-1.5 animate-pulse rounded-full bg-success" />
|
||||||
|
预览中
|
||||||
|
</Badge>
|
||||||
|
) : (
|
||||||
|
<div className="flex max-w-xs flex-col items-center gap-3 px-6 text-center">
|
||||||
|
<Video size={28} className="text-muted-soft" />
|
||||||
|
<div className="text-sm font-medium text-foreground">
|
||||||
|
{starting ? "正在开启摄像头…" : "摄像头未开启"}
|
||||||
|
</div>
|
||||||
|
<p className="text-xs leading-5 text-muted-foreground">
|
||||||
|
{error ?? "开启摄像头预览画面。后续可将视频流接入后端视觉理解管线。"}
|
||||||
|
</p>
|
||||||
|
<Button
|
||||||
|
size="sm"
|
||||||
|
disabled={starting}
|
||||||
|
onClick={() => void start()}
|
||||||
|
className="gap-1.5 rounded-full"
|
||||||
|
aria-label="开启摄像头"
|
||||||
|
>
|
||||||
|
{starting ? (
|
||||||
|
<Loader2 size={14} className="animate-spin" />
|
||||||
|
) : (
|
||||||
|
<Video size={14} />
|
||||||
|
)}
|
||||||
|
开启摄像头
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function DebugConnectionStatus({
|
||||||
|
status,
|
||||||
|
micWarning,
|
||||||
|
}: {
|
||||||
|
status: VoicePreviewStatus;
|
||||||
|
micWarning: string | null;
|
||||||
|
}) {
|
||||||
|
const recording = status === "connecting" || status === "connected";
|
||||||
|
const label =
|
||||||
|
status === "connecting"
|
||||||
|
? "连接中…"
|
||||||
|
: status === "connected"
|
||||||
|
? micWarning
|
||||||
|
? "仅收听"
|
||||||
|
: "进行中"
|
||||||
|
: status === "failed"
|
||||||
|
? "连接失败"
|
||||||
|
: "准备开始";
|
||||||
|
|
||||||
|
return (
|
||||||
|
<span className="flex min-w-0 items-center gap-1.5 truncate text-xs text-muted-foreground">
|
||||||
|
<span
|
||||||
|
className={[
|
||||||
|
"h-1.5 w-1.5 shrink-0 rounded-full",
|
||||||
|
recording
|
||||||
|
? "animate-pulse bg-success"
|
||||||
|
: status === "failed"
|
||||||
|
? "bg-destructive"
|
||||||
|
: "bg-muted-soft",
|
||||||
|
].join(" ")}
|
||||||
|
/>
|
||||||
|
{label}
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// ISO 时间戳 → HH:MM(本地时区),解析失败返回空串
|
// ISO 时间戳 → HH:MM(本地时区),解析失败返回空串
|
||||||
function formatMessageTime(iso: string): string {
|
function formatMessageTime(iso: string): string {
|
||||||
const d = new Date(iso);
|
const d = new Date(iso);
|
||||||
@@ -2291,10 +2519,10 @@ function formatMessageTime(iso: string): string {
|
|||||||
|
|
||||||
function DebugTranscriptPanel({
|
function DebugTranscriptPanel({
|
||||||
messages,
|
messages,
|
||||||
recording,
|
recording = false,
|
||||||
}: {
|
}: {
|
||||||
messages: ChatMessage[];
|
messages: ChatMessage[];
|
||||||
recording: boolean;
|
recording?: boolean;
|
||||||
}) {
|
}) {
|
||||||
const scrollRef = useRef<HTMLDivElement>(null);
|
const scrollRef = useRef<HTMLDivElement>(null);
|
||||||
|
|
||||||
@@ -2306,16 +2534,23 @@ function DebugTranscriptPanel({
|
|||||||
|
|
||||||
if (messages.length === 0) {
|
if (messages.length === 0) {
|
||||||
return (
|
return (
|
||||||
<div className="flex min-h-0 flex-1 flex-col items-center justify-center gap-2 px-6 text-center">
|
<div
|
||||||
<MessageSquareText size={28} className="text-muted-soft" />
|
className={[
|
||||||
<div className="text-sm font-medium text-foreground">
|
"flex min-h-0 flex-1 flex-col overflow-y-auto px-5 py-3",
|
||||||
{recording ? "暂无聊天记录" : "尚未开始对话"}
|
recording ? "items-center justify-center" : "pt-5",
|
||||||
|
].join(" ")}
|
||||||
|
>
|
||||||
|
<div className="text-center">
|
||||||
|
<MessageSquareText size={22} className="mx-auto text-muted-soft" />
|
||||||
|
<div className="mt-2 text-sm font-medium text-foreground">
|
||||||
|
{recording ? "暂无聊天记录" : "尚未开始对话"}
|
||||||
|
</div>
|
||||||
|
<p className="mx-auto mt-1 max-w-xs text-xs leading-5 text-muted-foreground">
|
||||||
|
{recording
|
||||||
|
? "开口说话或在下方输入文字,对话内容会实时显示在这里。"
|
||||||
|
: "点击「开始对话」后,语音与文字消息会实时显示在这里。"}
|
||||||
|
</p>
|
||||||
</div>
|
</div>
|
||||||
<p className="max-w-xs text-xs leading-5 text-muted-foreground">
|
|
||||||
{recording
|
|
||||||
? "开口说话或在下方输入文字,对话内容会实时显示在这里。"
|
|
||||||
: "点击「开始对话」后,语音与文字消息会实时显示在这里。"}
|
|
||||||
</p>
|
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
113
frontend/src/hooks/use-camera-preview.ts
Normal file
113
frontend/src/hooks/use-camera-preview.ts
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 摄像头预览:纯前端 getUserMedia 视频流,供调试抽屉的「视频流」视图使用。
|
||||||
|
*
|
||||||
|
* 与 use-voice-preview 分离:麦克风/语音会话走 WebRTC 到后端,摄像头目前只在
|
||||||
|
* 本地预览。后续接入视觉理解管线时,把这条 video track 加进现有 PeerConnection
|
||||||
|
* (pc.addTrack)并在后端 transport 打开 video_in_enabled 即可。
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { useCallback, useEffect, useRef, useState } from "react";
|
||||||
|
|
||||||
|
function cameraErrorMessage(error: unknown): string {
|
||||||
|
if (error instanceof DOMException) {
|
||||||
|
if (error.name === "NotAllowedError")
|
||||||
|
return "摄像头权限被拒绝,请在浏览器网站设置中允许使用摄像头。";
|
||||||
|
if (error.name === "NotFoundError")
|
||||||
|
return "未检测到可用摄像头,请连接或启用摄像头后重试。";
|
||||||
|
if (error.name === "NotReadableError")
|
||||||
|
return "摄像头正被其他应用占用,或系统未允许浏览器访问摄像头。";
|
||||||
|
}
|
||||||
|
if (error instanceof Error && error.message) return error.message;
|
||||||
|
return "无法访问摄像头。";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useCameraPreview() {
|
||||||
|
const [stream, setStream] = useState<MediaStream | null>(null);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const [starting, setStarting] = useState(false);
|
||||||
|
const [devices, setDevices] = useState<MediaDeviceInfo[]>([]);
|
||||||
|
const [deviceId, setDeviceId] = useState("");
|
||||||
|
const streamRef = useRef<MediaStream | null>(null);
|
||||||
|
const deviceIdRef = useRef("");
|
||||||
|
|
||||||
|
const active = Boolean(stream);
|
||||||
|
|
||||||
|
const refreshCameras = useCallback(async () => {
|
||||||
|
if (!navigator.mediaDevices?.enumerateDevices) return;
|
||||||
|
try {
|
||||||
|
const list = await navigator.mediaDevices.enumerateDevices();
|
||||||
|
setDevices(list.filter((d) => d.kind === "videoinput" && d.deviceId));
|
||||||
|
} catch {
|
||||||
|
/* 忽略枚举失败 */
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const stop = useCallback(() => {
|
||||||
|
streamRef.current?.getTracks().forEach((track) => track.stop());
|
||||||
|
streamRef.current = null;
|
||||||
|
setStream(null);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const start = useCallback(async () => {
|
||||||
|
if (streamRef.current) return;
|
||||||
|
setError(null);
|
||||||
|
if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
|
||||||
|
setError("当前页面无法访问摄像头,请通过 https 或 localhost 访问。");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setStarting(true);
|
||||||
|
try {
|
||||||
|
const id = deviceIdRef.current;
|
||||||
|
const media = await navigator.mediaDevices.getUserMedia({
|
||||||
|
video: id ? { deviceId: { exact: id } } : true,
|
||||||
|
});
|
||||||
|
streamRef.current = media;
|
||||||
|
setStream(media);
|
||||||
|
void refreshCameras();
|
||||||
|
} catch (mediaError) {
|
||||||
|
setError(cameraErrorMessage(mediaError));
|
||||||
|
} finally {
|
||||||
|
setStarting(false);
|
||||||
|
}
|
||||||
|
}, [refreshCameras]);
|
||||||
|
|
||||||
|
const selectCamera = useCallback(
|
||||||
|
async (id: string) => {
|
||||||
|
deviceIdRef.current = id;
|
||||||
|
setDeviceId(id);
|
||||||
|
if (streamRef.current) {
|
||||||
|
stop();
|
||||||
|
await start();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[start, stop],
|
||||||
|
);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!navigator.mediaDevices?.enumerateDevices) return;
|
||||||
|
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||||||
|
void refreshCameras();
|
||||||
|
navigator.mediaDevices.addEventListener("devicechange", refreshCameras);
|
||||||
|
return () => {
|
||||||
|
navigator.mediaDevices.removeEventListener("devicechange", refreshCameras);
|
||||||
|
};
|
||||||
|
}, [refreshCameras]);
|
||||||
|
|
||||||
|
useEffect(() => stop, [stop]);
|
||||||
|
|
||||||
|
return {
|
||||||
|
stream,
|
||||||
|
error,
|
||||||
|
starting,
|
||||||
|
devices,
|
||||||
|
deviceId,
|
||||||
|
active,
|
||||||
|
start,
|
||||||
|
stop,
|
||||||
|
selectCamera,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export type CameraPreview = ReturnType<typeof useCameraPreview>;
|
||||||
@@ -558,3 +558,5 @@ export function useVoicePreview(
|
|||||||
audioRef,
|
audioRef,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type VoicePreview = ReturnType<typeof useVoicePreview>;
|
||||||
|
|||||||
Reference in New Issue
Block a user