diff --git a/frontend/src/components/assistant-editor/debug-preview.tsx b/frontend/src/components/assistant-editor/debug-preview.tsx index 7f6bbc4..dd02552 100644 --- a/frontend/src/components/assistant-editor/debug-preview.tsx +++ b/frontend/src/components/assistant-editor/debug-preview.tsx @@ -49,6 +49,7 @@ import { WaveformTimelinePanel } from "@/components/ui/waveform-timeline"; import { useCameraPreview, type CameraPreview } from "@/hooks/use-camera-preview"; import { usePhotoCaptureTool } from "@/hooks/use-photo-capture-tool"; import { + createUserInputId, useVoicePreview, type ChatMessage, type ClientToolDefinition, @@ -799,6 +800,7 @@ function DebugVoicePanel({ sendText, sendUserInput, appendUserImage, + removeUserImage, connect, disconnect, audioRef, @@ -866,6 +868,7 @@ function DebugVoicePanel({ setSendingInput(true); setInputError(""); let assetToken = ""; + let inputId = ""; try { const imageUrl = await fileToDataUrl(pendingImage.file); const asset = await inputAssetsApi.uploadImage(pendingImage.file); @@ -878,11 +881,13 @@ function DebugVoicePanel({ }); const timestamp = new Date().toISOString(); - const result = await sendUserInput(parts); - appendUserImage(result.inputId, imageUrl, timestamp, text); + inputId = createUserInputId(); + appendUserImage(inputId, imageUrl, timestamp, text); + await sendUserInput(parts, { inputId }); setTextDraft(""); setPendingImage(null); } catch (sendError) { + if (inputId) removeUserImage(inputId); if (assetToken) { void inputAssetsApi.remove(assetToken).catch(() => {}); } diff --git a/frontend/src/hooks/use-photo-capture-tool.ts b/frontend/src/hooks/use-photo-capture-tool.ts index c4f9476..955ffc8 100644 --- a/frontend/src/hooks/use-photo-capture-tool.ts +++ b/frontend/src/hooks/use-photo-capture-tool.ts @@ -2,7 +2,10 @@ import { useCallback, useEffect, useState } from "react"; -import type { VoicePreview } from "@/hooks/use-voice-preview"; +import { + createUserInputId, + type VoicePreview, +} from "@/hooks/use-voice-preview"; const PHOTO_BUTTON_TOOL = "set_photo_button_visible"; const PHOTO_BUTTON_DEFINITION = { @@ -72,6 +75,7 @@ async function capturePreviewImage(stream: MediaStream | null): Promise export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) { const { appendUserImage, + removeUserImage, registerClientTool, sendUserInput, status, @@ -107,27 +111,37 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) { if (capturing || status !== "connected") return; setCapturing(true); setError(null); + let inputId = ""; try { const timestamp = new Date().toISOString(); const imageUrl = await capturePreviewImage(videoStream); - const result = await sendUserInput( + inputId = createUserInputId(); + appendUserImage(inputId, imageUrl, timestamp); + await sendUserInput( [ { type: "input_image", source: { type: "camera_frame", frame: "current" }, }, ], - { runImmediately: true, interrupt: true }, + { runImmediately: true, interrupt: true, inputId }, ); - appendUserImage(result.inputId, imageUrl, timestamp); } catch (captureError) { + if (inputId) removeUserImage(inputId); setError( captureError instanceof Error ? captureError.message : "拍照提交失败", ); } finally { setCapturing(false); } - }, [appendUserImage, capturing, sendUserInput, status, videoStream]); + }, [ + appendUserImage, + capturing, + removeUserImage, + sendUserInput, + status, + videoStream, + ]); return { visible, diff --git a/frontend/src/hooks/use-voice-preview.ts b/frontend/src/hooks/use-voice-preview.ts index 20852e2..88db2d3 100644 --- a/frontend/src/hooks/use-voice-preview.ts +++ b/frontend/src/hooks/use-voice-preview.ts @@ -174,15 +174,11 @@ function errorMessage(error: unknown, fallback: string): string { return fallback; } -function messageOrder(message: ChatMessage): number { - const timestamp = Date.parse(message.timestamp); - return Number.isNaN(timestamp) ? Number.MAX_SAFE_INTEGER : timestamp; -} - function sortMessages(messages: ChatMessage[]): ChatMessage[] { - return messages.sort( - (a, b) => messageOrder(a) - messageOrder(b) || a.sequence - b.sequence, - ); + // Timestamps come from both the browser and the backend server. Remote + // deployments can have clock skew, so causal UI order must use the single + // client-owned sequence instead of comparing clocks from different hosts. + return messages.sort((a, b) => a.sequence - b.sequence); } function encodeHeaderJson(value: unknown): string { @@ -194,7 +190,7 @@ function encodeHeaderJson(value: unknown): string { return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); } -function newInputId(): string { +export function createUserInputId(): string { if (typeof crypto !== "undefined" && "randomUUID" in crypto) { return `input_${crypto.randomUUID()}`; } @@ -831,7 +827,7 @@ export function useVoicePreview( const trimmed = text.trim(); const transport = transportRef.current; if (!trimmed || !transport || transport.state !== "connected") return false; - const inputId = newInputId(); + const inputId = createUserInputId(); transport.sendAppMessage({ type: "user-input", schema_version: 1, @@ -845,13 +841,17 @@ export function useVoicePreview( const sendUserInput = useCallback( ( parts: UserInputPart[], - options: { runImmediately?: boolean; interrupt?: boolean } = {}, + options: { + runImmediately?: boolean; + interrupt?: boolean; + inputId?: string; + } = {}, ): Promise => { const transport = transportRef.current; if (!transport || transport.state !== "connected") { return Promise.reject(new Error("当前未连接语音服务")); } - const inputId = newInputId(); + const inputId = options.inputId ?? createUserInputId(); return new Promise((resolve, reject) => { const timeout = window.setTimeout(() => { pendingUserInputsRef.current.delete(inputId); @@ -914,6 +914,12 @@ export function useVoicePreview( [], ); + const removeUserImage = useCallback((inputId: string) => { + setMessages((previous) => + previous.filter((message) => message.id !== `user-image-${inputId}`), + ); + }, []); + const updateSession = useCallback( ({ dynamicVariables, @@ -1006,6 +1012,7 @@ export function useVoicePreview( sendText, sendUserInput, appendUserImage, + removeUserImage, updateSession, registerClientTool, connect,