Enhance voice interaction and transcript handling in the assistant

- Add a new Docker configuration for the UI in launch.json to facilitate development.
- Refactor pipeline.py to integrate a TranscriptProcessor for managing user and assistant transcripts, including event handlers for real-time updates and message handling.
- Update useVoicePreview.ts to establish a data channel for sending and receiving text messages, improving interaction flow.
- Modify AssistantPage.tsx to support displaying chat messages and sending user input, enhancing the user experience during voice interactions.
- Revise DebugTranscriptPanel to dynamically render chat messages with timestamps, improving the visual representation of conversation history.
This commit is contained in:
Xin Wang
2026-06-10 15:11:34 +08:00
parent b711350c0c
commit 2c2af1f2cd
4 changed files with 223 additions and 28 deletions

View File

@@ -78,7 +78,7 @@ import {
type Credential,
type KnowledgeBase,
} from "@/lib/api";
import { useVoicePreview } from "@/hooks/use-voice-preview";
import { useVoicePreview, type ChatMessage } from "@/hooks/use-voice-preview";
type RuntimeMode = "pipeline" | "realtime";
@@ -1856,19 +1856,28 @@ function DebugVoicePanel({
error,
micWarning,
localStream,
messages,
sendText,
connect,
disconnect,
audioRef,
} = useVoicePreview(assistantId);
// 连接中或已连通都视作"会话进行中"
const recording = status === "connecting" || status === "connected";
const [textDraft, setTextDraft] = useState("");
function handleSendText() {
if (sendText(textDraft)) {
setTextDraft("");
}
}
return (
<div className="flex min-h-0 flex-1 flex-col">
{/* 后端 TTS 音频经 WebRTC 媒体流过来,挂这里播放 */}
<audio ref={audioRef} autoPlay playsInline className="hidden" />
{showTranscript ? (
<DebugTranscriptPanel />
<DebugTranscriptPanel messages={messages} recording={recording} />
) : (
<div className="relative flex min-h-0 flex-1 flex-col items-center justify-center gap-3 overflow-y-auto px-6 py-3 text-center">
<div
@@ -1966,10 +1975,29 @@ function DebugVoicePanel({
<div className="flex items-end gap-2">
<Textarea
rows={1}
placeholder="输入文字以模拟用户消息…"
value={textDraft}
disabled={status !== "connected"}
onChange={(event) => setTextDraft(event.target.value)}
onKeyDown={(event) => {
if (event.key === "Enter" && !event.shiftKey && !event.nativeEvent.isComposing) {
event.preventDefault();
handleSendText();
}
}}
placeholder={
status === "connected"
? "输入文字发送给助手,将打断当前播报…"
: "开始对话后可输入文字…"
}
className="max-h-24 min-h-10 flex-1 resize-none border-hairline-strong bg-background text-sm text-foreground placeholder:text-muted-soft"
/>
<Button size="icon" className="shrink-0" aria-label="发送调试消息">
<Button
size="icon"
className="shrink-0"
aria-label="发送调试消息"
disabled={status !== "connected" || !textDraft.trim()}
onClick={handleSendText}
>
<Send size={16} />
</Button>
</div>
@@ -1978,31 +2006,79 @@ function DebugVoicePanel({
);
}
function DebugTranscriptPanel() {
// ISO 时间戳 → HH:MM(本地时区),解析失败返回空串
function formatMessageTime(iso: string): string {
const d = new Date(iso);
if (Number.isNaN(d.getTime())) return "";
const pad = (n: number) => String(n).padStart(2, "0");
return `${pad(d.getHours())}:${pad(d.getMinutes())}`;
}
function DebugTranscriptPanel({
messages,
recording,
}: {
messages: ChatMessage[];
recording: boolean;
}) {
const scrollRef = useRef<HTMLDivElement>(null);
// 新消息时滚到底部
useEffect(() => {
const el = scrollRef.current;
if (el) el.scrollTop = el.scrollHeight;
}, [messages]);
if (messages.length === 0) {
return (
<div className="flex min-h-0 flex-1 flex-col items-center justify-center gap-2 px-6 text-center">
<MessageSquareText size={28} className="text-muted-soft" />
<div className="text-sm font-medium text-foreground">
{recording ? "暂无聊天记录" : "尚未开始对话"}
</div>
<p className="max-w-xs text-xs leading-5 text-muted-foreground">
{recording
? "开口说话或在下方输入文字,对话内容会实时显示在这里。"
: "点击「开始对话」后,语音与文字消息会实时显示在这里。"}
</p>
</div>
);
}
return (
<div className="flex min-h-0 flex-1 flex-col overflow-y-auto px-5 py-4">
<div
ref={scrollRef}
className="flex min-h-0 flex-1 flex-col overflow-y-auto px-5 py-4"
>
<div className="flex flex-col gap-4">
<div className="flex max-w-[88%] flex-col gap-1 self-start">
<span className="px-1 text-[11px] text-muted-soft"> · 10:24</span>
<div className="rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
AI
</div>
</div>
<div className="flex max-w-[88%] flex-col items-end gap-1 self-end">
<span className="px-1 text-[11px] text-muted-soft"> · 10:25</span>
<div className="rounded-2xl rounded-tr-sm bg-primary px-4 py-2.5 text-sm leading-6 text-primary-foreground">
</div>
</div>
<div className="flex max-w-[88%] flex-col gap-1 self-start">
<span className="px-1 text-[11px] text-muted-soft"> · 10:25</span>
<div className="rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
线线线 App
线
</div>
</div>
{messages.map((message) => {
const time = formatMessageTime(message.timestamp);
return message.role === "assistant" ? (
<div
key={message.id}
className="flex max-w-[88%] flex-col gap-1 self-start"
>
<span className="px-1 text-[11px] text-muted-soft">
{time ? ` · ${time}` : ""}
</span>
<div className="whitespace-pre-wrap rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
{message.content}
</div>
</div>
) : (
<div
key={message.id}
className="flex max-w-[88%] flex-col items-end gap-1 self-end"
>
<span className="px-1 text-[11px] text-muted-soft">
{time ? ` · ${time}` : ""}
</span>
<div className="whitespace-pre-wrap rounded-2xl rounded-tr-sm bg-primary px-4 py-2.5 text-sm leading-6 text-primary-foreground">
{message.content}
</div>
</div>
);
})}
</div>
</div>
);