fix(chat): preserve user image message order
This commit is contained in:
@@ -49,6 +49,7 @@ import { WaveformTimelinePanel } from "@/components/ui/waveform-timeline";
|
||||
import { useCameraPreview, type CameraPreview } from "@/hooks/use-camera-preview";
|
||||
import { usePhotoCaptureTool } from "@/hooks/use-photo-capture-tool";
|
||||
import {
|
||||
createUserInputId,
|
||||
useVoicePreview,
|
||||
type ChatMessage,
|
||||
type ClientToolDefinition,
|
||||
@@ -799,6 +800,7 @@ function DebugVoicePanel({
|
||||
sendText,
|
||||
sendUserInput,
|
||||
appendUserImage,
|
||||
removeUserImage,
|
||||
connect,
|
||||
disconnect,
|
||||
audioRef,
|
||||
@@ -866,6 +868,7 @@ function DebugVoicePanel({
|
||||
setSendingInput(true);
|
||||
setInputError("");
|
||||
let assetToken = "";
|
||||
let inputId = "";
|
||||
try {
|
||||
const imageUrl = await fileToDataUrl(pendingImage.file);
|
||||
const asset = await inputAssetsApi.uploadImage(pendingImage.file);
|
||||
@@ -878,11 +881,13 @@ function DebugVoicePanel({
|
||||
});
|
||||
|
||||
const timestamp = new Date().toISOString();
|
||||
const result = await sendUserInput(parts);
|
||||
appendUserImage(result.inputId, imageUrl, timestamp, text);
|
||||
inputId = createUserInputId();
|
||||
appendUserImage(inputId, imageUrl, timestamp, text);
|
||||
await sendUserInput(parts, { inputId });
|
||||
setTextDraft("");
|
||||
setPendingImage(null);
|
||||
} catch (sendError) {
|
||||
if (inputId) removeUserImage(inputId);
|
||||
if (assetToken) {
|
||||
void inputAssetsApi.remove(assetToken).catch(() => {});
|
||||
}
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
|
||||
import type { VoicePreview } from "@/hooks/use-voice-preview";
|
||||
import {
|
||||
createUserInputId,
|
||||
type VoicePreview,
|
||||
} from "@/hooks/use-voice-preview";
|
||||
|
||||
const PHOTO_BUTTON_TOOL = "set_photo_button_visible";
|
||||
const PHOTO_BUTTON_DEFINITION = {
|
||||
@@ -72,6 +75,7 @@ async function capturePreviewImage(stream: MediaStream | null): Promise<string>
|
||||
export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
|
||||
const {
|
||||
appendUserImage,
|
||||
removeUserImage,
|
||||
registerClientTool,
|
||||
sendUserInput,
|
||||
status,
|
||||
@@ -107,27 +111,37 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
|
||||
if (capturing || status !== "connected") return;
|
||||
setCapturing(true);
|
||||
setError(null);
|
||||
let inputId = "";
|
||||
try {
|
||||
const timestamp = new Date().toISOString();
|
||||
const imageUrl = await capturePreviewImage(videoStream);
|
||||
const result = await sendUserInput(
|
||||
inputId = createUserInputId();
|
||||
appendUserImage(inputId, imageUrl, timestamp);
|
||||
await sendUserInput(
|
||||
[
|
||||
{
|
||||
type: "input_image",
|
||||
source: { type: "camera_frame", frame: "current" },
|
||||
},
|
||||
],
|
||||
{ runImmediately: true, interrupt: true },
|
||||
{ runImmediately: true, interrupt: true, inputId },
|
||||
);
|
||||
appendUserImage(result.inputId, imageUrl, timestamp);
|
||||
} catch (captureError) {
|
||||
if (inputId) removeUserImage(inputId);
|
||||
setError(
|
||||
captureError instanceof Error ? captureError.message : "拍照提交失败",
|
||||
);
|
||||
} finally {
|
||||
setCapturing(false);
|
||||
}
|
||||
}, [appendUserImage, capturing, sendUserInput, status, videoStream]);
|
||||
}, [
|
||||
appendUserImage,
|
||||
capturing,
|
||||
removeUserImage,
|
||||
sendUserInput,
|
||||
status,
|
||||
videoStream,
|
||||
]);
|
||||
|
||||
return {
|
||||
visible,
|
||||
|
||||
@@ -174,15 +174,11 @@ function errorMessage(error: unknown, fallback: string): string {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function messageOrder(message: ChatMessage): number {
|
||||
const timestamp = Date.parse(message.timestamp);
|
||||
return Number.isNaN(timestamp) ? Number.MAX_SAFE_INTEGER : timestamp;
|
||||
}
|
||||
|
||||
function sortMessages(messages: ChatMessage[]): ChatMessage[] {
|
||||
return messages.sort(
|
||||
(a, b) => messageOrder(a) - messageOrder(b) || a.sequence - b.sequence,
|
||||
);
|
||||
// Timestamps come from both the browser and the backend server. Remote
|
||||
// deployments can have clock skew, so causal UI order must use the single
|
||||
// client-owned sequence instead of comparing clocks from different hosts.
|
||||
return messages.sort((a, b) => a.sequence - b.sequence);
|
||||
}
|
||||
|
||||
function encodeHeaderJson(value: unknown): string {
|
||||
@@ -194,7 +190,7 @@ function encodeHeaderJson(value: unknown): string {
|
||||
return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
||||
}
|
||||
|
||||
function newInputId(): string {
|
||||
export function createUserInputId(): string {
|
||||
if (typeof crypto !== "undefined" && "randomUUID" in crypto) {
|
||||
return `input_${crypto.randomUUID()}`;
|
||||
}
|
||||
@@ -831,7 +827,7 @@ export function useVoicePreview(
|
||||
const trimmed = text.trim();
|
||||
const transport = transportRef.current;
|
||||
if (!trimmed || !transport || transport.state !== "connected") return false;
|
||||
const inputId = newInputId();
|
||||
const inputId = createUserInputId();
|
||||
transport.sendAppMessage({
|
||||
type: "user-input",
|
||||
schema_version: 1,
|
||||
@@ -845,13 +841,17 @@ export function useVoicePreview(
|
||||
const sendUserInput = useCallback(
|
||||
(
|
||||
parts: UserInputPart[],
|
||||
options: { runImmediately?: boolean; interrupt?: boolean } = {},
|
||||
options: {
|
||||
runImmediately?: boolean;
|
||||
interrupt?: boolean;
|
||||
inputId?: string;
|
||||
} = {},
|
||||
): Promise<UserInputResult> => {
|
||||
const transport = transportRef.current;
|
||||
if (!transport || transport.state !== "connected") {
|
||||
return Promise.reject(new Error("当前未连接语音服务"));
|
||||
}
|
||||
const inputId = newInputId();
|
||||
const inputId = options.inputId ?? createUserInputId();
|
||||
return new Promise<UserInputResult>((resolve, reject) => {
|
||||
const timeout = window.setTimeout(() => {
|
||||
pendingUserInputsRef.current.delete(inputId);
|
||||
@@ -914,6 +914,12 @@ export function useVoicePreview(
|
||||
[],
|
||||
);
|
||||
|
||||
const removeUserImage = useCallback((inputId: string) => {
|
||||
setMessages((previous) =>
|
||||
previous.filter((message) => message.id !== `user-image-${inputId}`),
|
||||
);
|
||||
}, []);
|
||||
|
||||
const updateSession = useCallback(
|
||||
({
|
||||
dynamicVariables,
|
||||
@@ -1006,6 +1012,7 @@ export function useVoicePreview(
|
||||
sendText,
|
||||
sendUserInput,
|
||||
appendUserImage,
|
||||
removeUserImage,
|
||||
updateSession,
|
||||
registerClientTool,
|
||||
connect,
|
||||
|
||||
Reference in New Issue
Block a user