Enhance text input processing and LLM interaction in the backend

- Refactor TextInputProcessor to handle immediate and silent text inputs, improving user experience during voice interactions.
- Introduce PassthroughLLMAssistantAggregator to manage LLM responses while preserving context for downstream TTS processing.
- Update event handling for text input and client readiness, ensuring timely updates to the conversation context.
- Modify run_pipeline to integrate new aggregators and streamline message handling, enhancing overall pipeline efficiency.
- Improve message ordering in useVoicePreview to ensure accurate display of chat messages based on timestamps.
This commit is contained in:
Xin Wang
2026-06-14 22:12:56 +08:00
parent 86d9acce78
commit b749d2e075
2 changed files with 107 additions and 17 deletions

View File

@@ -54,6 +54,11 @@ function errorMessage(error: unknown, fallback: string): string {
return fallback;
}
function messageOrder(message: ChatMessage): number {
const timestamp = Date.parse(message.timestamp);
return Number.isNaN(timestamp) ? Number.MAX_SAFE_INTEGER : timestamp;
}
function microphoneErrorMessage(error: unknown): string {
if (error instanceof DOMException) {
if (error.name === "NotAllowedError") {
@@ -137,6 +142,7 @@ export function useVoicePreview(assistantId: string | null) {
const channel = dataChannelRef.current;
dataChannelRef.current = null;
if (channel) {
channel.onopen = null;
channel.onmessage = null;
channel.close();
}
@@ -290,6 +296,9 @@ export function useVoicePreview(assistantId: string | null) {
// 由浏览器侧主动创建,后端 SmallWebRTCConnection 的 on("datachannel") 会接住。
const channel = pc.createDataChannel("chat");
dataChannelRef.current = channel;
channel.onopen = () => {
channel.send(JSON.stringify({ type: "client-ready" }));
};
channel.onmessage = (event) => {
try {
const msg = JSON.parse(event.data);
@@ -309,7 +318,14 @@ export function useVoicePreview(assistantId: string | null) {
? msg.timestamp
: new Date().toISOString(),
};
setMessages((prev) => [...prev, next]);
setMessages((prev) =>
[...prev, next].sort(
(a, b) =>
messageOrder(a) - messageOrder(b) ||
Number(a.id.replace("msg-", "")) -
Number(b.id.replace("msg-", "")),
),
);
}
} catch {
/* 非 JSON / 未知消息,忽略 */