fix(chat): preserve user image message order

This commit is contained in:
Xin Wang
2026-08-06 13:59:59 +08:00
parent 4a07abd7fc
commit 4d00e719c4
3 changed files with 45 additions and 19 deletions

View File

@@ -49,6 +49,7 @@ import { WaveformTimelinePanel } from "@/components/ui/waveform-timeline";
import { useCameraPreview, type CameraPreview } from "@/hooks/use-camera-preview";
import { usePhotoCaptureTool } from "@/hooks/use-photo-capture-tool";
import {
createUserInputId,
useVoicePreview,
type ChatMessage,
type ClientToolDefinition,
@@ -799,6 +800,7 @@ function DebugVoicePanel({
sendText,
sendUserInput,
appendUserImage,
removeUserImage,
connect,
disconnect,
audioRef,
@@ -866,6 +868,7 @@ function DebugVoicePanel({
setSendingInput(true);
setInputError("");
let assetToken = "";
let inputId = "";
try {
const imageUrl = await fileToDataUrl(pendingImage.file);
const asset = await inputAssetsApi.uploadImage(pendingImage.file);
@@ -878,11 +881,13 @@ function DebugVoicePanel({
});
const timestamp = new Date().toISOString();
const result = await sendUserInput(parts);
appendUserImage(result.inputId, imageUrl, timestamp, text);
inputId = createUserInputId();
appendUserImage(inputId, imageUrl, timestamp, text);
await sendUserInput(parts, { inputId });
setTextDraft("");
setPendingImage(null);
} catch (sendError) {
if (inputId) removeUserImage(inputId);
if (assetToken) {
void inputAssetsApi.remove(assetToken).catch(() => {});
}

View File

@@ -2,7 +2,10 @@
import { useCallback, useEffect, useState } from "react";
import type { VoicePreview } from "@/hooks/use-voice-preview";
import {
createUserInputId,
type VoicePreview,
} from "@/hooks/use-voice-preview";
const PHOTO_BUTTON_TOOL = "set_photo_button_visible";
const PHOTO_BUTTON_DEFINITION = {
@@ -72,6 +75,7 @@ async function capturePreviewImage(stream: MediaStream | null): Promise<string>
export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
const {
appendUserImage,
removeUserImage,
registerClientTool,
sendUserInput,
status,
@@ -107,27 +111,37 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
if (capturing || status !== "connected") return;
setCapturing(true);
setError(null);
let inputId = "";
try {
const timestamp = new Date().toISOString();
const imageUrl = await capturePreviewImage(videoStream);
const result = await sendUserInput(
inputId = createUserInputId();
appendUserImage(inputId, imageUrl, timestamp);
await sendUserInput(
[
{
type: "input_image",
source: { type: "camera_frame", frame: "current" },
},
],
{ runImmediately: true, interrupt: true },
{ runImmediately: true, interrupt: true, inputId },
);
appendUserImage(result.inputId, imageUrl, timestamp);
} catch (captureError) {
if (inputId) removeUserImage(inputId);
setError(
captureError instanceof Error ? captureError.message : "拍照提交失败",
);
} finally {
setCapturing(false);
}
}, [appendUserImage, capturing, sendUserInput, status, videoStream]);
}, [
appendUserImage,
capturing,
removeUserImage,
sendUserInput,
status,
videoStream,
]);
return {
visible,

View File

@@ -174,15 +174,11 @@ function errorMessage(error: unknown, fallback: string): string {
return fallback;
}
function messageOrder(message: ChatMessage): number {
const timestamp = Date.parse(message.timestamp);
return Number.isNaN(timestamp) ? Number.MAX_SAFE_INTEGER : timestamp;
}
function sortMessages(messages: ChatMessage[]): ChatMessage[] {
return messages.sort(
(a, b) => messageOrder(a) - messageOrder(b) || a.sequence - b.sequence,
);
// Timestamps come from both the browser and the backend server. Remote
// deployments can have clock skew, so causal UI order must use the single
// client-owned sequence instead of comparing clocks from different hosts.
return messages.sort((a, b) => a.sequence - b.sequence);
}
function encodeHeaderJson(value: unknown): string {
@@ -194,7 +190,7 @@ function encodeHeaderJson(value: unknown): string {
return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
}
function newInputId(): string {
export function createUserInputId(): string {
if (typeof crypto !== "undefined" && "randomUUID" in crypto) {
return `input_${crypto.randomUUID()}`;
}
@@ -831,7 +827,7 @@ export function useVoicePreview(
const trimmed = text.trim();
const transport = transportRef.current;
if (!trimmed || !transport || transport.state !== "connected") return false;
const inputId = newInputId();
const inputId = createUserInputId();
transport.sendAppMessage({
type: "user-input",
schema_version: 1,
@@ -845,13 +841,17 @@ export function useVoicePreview(
const sendUserInput = useCallback(
(
parts: UserInputPart[],
options: { runImmediately?: boolean; interrupt?: boolean } = {},
options: {
runImmediately?: boolean;
interrupt?: boolean;
inputId?: string;
} = {},
): Promise<UserInputResult> => {
const transport = transportRef.current;
if (!transport || transport.state !== "connected") {
return Promise.reject(new Error("当前未连接语音服务"));
}
const inputId = newInputId();
const inputId = options.inputId ?? createUserInputId();
return new Promise<UserInputResult>((resolve, reject) => {
const timeout = window.setTimeout(() => {
pendingUserInputsRef.current.delete(inputId);
@@ -914,6 +914,12 @@ export function useVoicePreview(
[],
);
const removeUserImage = useCallback((inputId: string) => {
setMessages((previous) =>
previous.filter((message) => message.id !== `user-image-${inputId}`),
);
}, []);
const updateSession = useCallback(
({
dynamicVariables,
@@ -1006,6 +1012,7 @@ export function useVoicePreview(
sendText,
sendUserInput,
appendUserImage,
removeUserImage,
updateSession,
registerClientTool,
connect,