Enhance voice interaction and transcript handling in the assistant
- Add a new Docker configuration for the UI in launch.json to facilitate development. - Refactor pipeline.py to integrate a TranscriptProcessor for managing user and assistant transcripts, including event handlers for real-time updates and message handling. - Update useVoicePreview.ts to establish a data channel for sending and receiving text messages, improving interaction flow. - Modify AssistantPage.tsx to support displaying chat messages and sending user input, enhancing the user experience during voice interactions. - Revise DebugTranscriptPanel to dynamically render chat messages with timestamps, improving the visual representation of conversation history.
This commit is contained in:
@@ -78,7 +78,7 @@ import {
|
||||
type Credential,
|
||||
type KnowledgeBase,
|
||||
} from "@/lib/api";
|
||||
import { useVoicePreview } from "@/hooks/use-voice-preview";
|
||||
import { useVoicePreview, type ChatMessage } from "@/hooks/use-voice-preview";
|
||||
|
||||
type RuntimeMode = "pipeline" | "realtime";
|
||||
|
||||
@@ -1856,19 +1856,28 @@ function DebugVoicePanel({
|
||||
error,
|
||||
micWarning,
|
||||
localStream,
|
||||
messages,
|
||||
sendText,
|
||||
connect,
|
||||
disconnect,
|
||||
audioRef,
|
||||
} = useVoicePreview(assistantId);
|
||||
// 连接中或已连通都视作"会话进行中"
|
||||
const recording = status === "connecting" || status === "connected";
|
||||
const [textDraft, setTextDraft] = useState("");
|
||||
|
||||
function handleSendText() {
|
||||
if (sendText(textDraft)) {
|
||||
setTextDraft("");
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex min-h-0 flex-1 flex-col">
|
||||
{/* 后端 TTS 音频经 WebRTC 媒体流过来,挂这里播放 */}
|
||||
<audio ref={audioRef} autoPlay playsInline className="hidden" />
|
||||
{showTranscript ? (
|
||||
<DebugTranscriptPanel />
|
||||
<DebugTranscriptPanel messages={messages} recording={recording} />
|
||||
) : (
|
||||
<div className="relative flex min-h-0 flex-1 flex-col items-center justify-center gap-3 overflow-y-auto px-6 py-3 text-center">
|
||||
<div
|
||||
@@ -1966,10 +1975,29 @@ function DebugVoicePanel({
|
||||
<div className="flex items-end gap-2">
|
||||
<Textarea
|
||||
rows={1}
|
||||
placeholder="输入文字以模拟用户消息…"
|
||||
value={textDraft}
|
||||
disabled={status !== "connected"}
|
||||
onChange={(event) => setTextDraft(event.target.value)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter" && !event.shiftKey && !event.nativeEvent.isComposing) {
|
||||
event.preventDefault();
|
||||
handleSendText();
|
||||
}
|
||||
}}
|
||||
placeholder={
|
||||
status === "connected"
|
||||
? "输入文字发送给助手,将打断当前播报…"
|
||||
: "开始对话后可输入文字…"
|
||||
}
|
||||
className="max-h-24 min-h-10 flex-1 resize-none border-hairline-strong bg-background text-sm text-foreground placeholder:text-muted-soft"
|
||||
/>
|
||||
<Button size="icon" className="shrink-0" aria-label="发送调试消息">
|
||||
<Button
|
||||
size="icon"
|
||||
className="shrink-0"
|
||||
aria-label="发送调试消息"
|
||||
disabled={status !== "connected" || !textDraft.trim()}
|
||||
onClick={handleSendText}
|
||||
>
|
||||
<Send size={16} />
|
||||
</Button>
|
||||
</div>
|
||||
@@ -1978,31 +2006,79 @@ function DebugVoicePanel({
|
||||
);
|
||||
}
|
||||
|
||||
function DebugTranscriptPanel() {
|
||||
// ISO 时间戳 → HH:MM(本地时区),解析失败返回空串
|
||||
function formatMessageTime(iso: string): string {
|
||||
const d = new Date(iso);
|
||||
if (Number.isNaN(d.getTime())) return "";
|
||||
const pad = (n: number) => String(n).padStart(2, "0");
|
||||
return `${pad(d.getHours())}:${pad(d.getMinutes())}`;
|
||||
}
|
||||
|
||||
function DebugTranscriptPanel({
|
||||
messages,
|
||||
recording,
|
||||
}: {
|
||||
messages: ChatMessage[];
|
||||
recording: boolean;
|
||||
}) {
|
||||
const scrollRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
// 新消息时滚到底部
|
||||
useEffect(() => {
|
||||
const el = scrollRef.current;
|
||||
if (el) el.scrollTop = el.scrollHeight;
|
||||
}, [messages]);
|
||||
|
||||
if (messages.length === 0) {
|
||||
return (
|
||||
<div className="flex min-h-0 flex-1 flex-col items-center justify-center gap-2 px-6 text-center">
|
||||
<MessageSquareText size={28} className="text-muted-soft" />
|
||||
<div className="text-sm font-medium text-foreground">
|
||||
{recording ? "暂无聊天记录" : "尚未开始对话"}
|
||||
</div>
|
||||
<p className="max-w-xs text-xs leading-5 text-muted-foreground">
|
||||
{recording
|
||||
? "开口说话或在下方输入文字,对话内容会实时显示在这里。"
|
||||
: "点击「开始对话」后,语音与文字消息会实时显示在这里。"}
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex min-h-0 flex-1 flex-col overflow-y-auto px-5 py-4">
|
||||
<div
|
||||
ref={scrollRef}
|
||||
className="flex min-h-0 flex-1 flex-col overflow-y-auto px-5 py-4"
|
||||
>
|
||||
<div className="flex flex-col gap-4">
|
||||
<div className="flex max-w-[88%] flex-col gap-1 self-start">
|
||||
<span className="px-1 text-[11px] text-muted-soft">助手 · 10:24</span>
|
||||
<div className="rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
|
||||
您好,我是 AI 视频助手,请问有什么可以帮您?
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex max-w-[88%] flex-col items-end gap-1 self-end">
|
||||
<span className="px-1 text-[11px] text-muted-soft">我 · 10:25</span>
|
||||
<div className="rounded-2xl rounded-tr-sm bg-primary px-4 py-2.5 text-sm leading-6 text-primary-foreground">
|
||||
我想了解一下社保卡的办理流程。
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex max-w-[88%] flex-col gap-1 self-start">
|
||||
<span className="px-1 text-[11px] text-muted-soft">助手 · 10:25</span>
|
||||
<div className="rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
|
||||
社保卡可通过线上或线下渠道办理。线上可在政务服务 App
|
||||
提交申请,线下可前往社保经办网点。
|
||||
</div>
|
||||
</div>
|
||||
{messages.map((message) => {
|
||||
const time = formatMessageTime(message.timestamp);
|
||||
return message.role === "assistant" ? (
|
||||
<div
|
||||
key={message.id}
|
||||
className="flex max-w-[88%] flex-col gap-1 self-start"
|
||||
>
|
||||
<span className="px-1 text-[11px] text-muted-soft">
|
||||
助手{time ? ` · ${time}` : ""}
|
||||
</span>
|
||||
<div className="whitespace-pre-wrap rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
|
||||
{message.content}
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div
|
||||
key={message.id}
|
||||
className="flex max-w-[88%] flex-col items-end gap-1 self-end"
|
||||
>
|
||||
<span className="px-1 text-[11px] text-muted-soft">
|
||||
我{time ? ` · ${time}` : ""}
|
||||
</span>
|
||||
<div className="whitespace-pre-wrap rounded-2xl rounded-tr-sm bg-primary px-4 py-2.5 text-sm leading-6 text-primary-foreground">
|
||||
{message.content}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user