feat: persist photos and enrich conversation history

This commit is contained in:
Xin Wang
2026-08-05 16:14:03 +08:00
parent 545536bdf3
commit 317a0600bc
19 changed files with 1146 additions and 119 deletions

View File

@@ -1154,7 +1154,10 @@ function DebugVisionWorkspace({
} | null>(null);
const latestMessage = [...messages]
.reverse()
.find((message) => message.content.trim());
.find(
(message) =>
message.content.trim() || (message.attachments?.length ?? 0) > 0,
);
useEffect(() => {
const container = containerRef.current;
@@ -1201,7 +1204,10 @@ function DebugVisionWorkspace({
{latestMessage?.role === "user" ? "我:" : "助手:"}
</span>
<span>
{latestMessage?.content || "暂无消息,点击返回聊天记录"}
{latestMessage?.content ||
(latestMessage?.attachments?.length
? "发送了一张照片"
: "暂无消息,点击返回聊天记录")}
</span>
</span>
</button>
@@ -1449,7 +1455,7 @@ function DebugTranscriptPanel({
{time ? ` · ${time}` : ""}
</span>
<div className="whitespace-pre-wrap rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
{message.content}
{message.content || (message.streaming ? "…" : "")}
</div>
</div>
) : (
@@ -1460,8 +1466,26 @@ function DebugTranscriptPanel({
<span className="px-1 text-[11px] text-muted-soft">
{time ? ` · ${time}` : ""}
</span>
<div className="whitespace-pre-wrap rounded-2xl rounded-tr-sm bg-primary px-4 py-2.5 text-sm leading-6 text-primary-foreground">
{message.content}
<div
className={[
"overflow-hidden whitespace-pre-wrap rounded-2xl rounded-tr-sm bg-primary text-sm leading-6 text-primary-foreground",
message.attachments?.length ? "p-1" : "px-4 py-2.5",
].join(" ")}
>
{message.attachments?.map((attachment) => (
// eslint-disable-next-line @next/next/no-img-element
<img
key={attachment.id}
src={attachment.url}
alt={attachment.alt}
className="max-h-72 w-full rounded-[0.8rem] object-cover"
/>
))}
{message.content && (
<div className={message.attachments?.length ? "px-3 py-2" : ""}>
{message.content}
</div>
)}
</div>
</div>
);

View File

@@ -1,13 +1,25 @@
"use client";
import {
Activity,
ArrowRight,
Camera,
CheckCircle2,
ChevronDown,
ChevronUp,
CircleDot,
Eye,
GitBranch,
ImageIcon,
Loader2,
MessageSquareText,
MonitorSmartphone,
MoreHorizontal,
Server,
Trash2,
Wrench,
XCircle,
type LucideIcon,
} from "lucide-react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
@@ -37,9 +49,11 @@ import {
} from "@/components/layout/list-page-layout";
import { SearchInput } from "@/components/ui/search-input";
import {
API_BASE,
conversationsApi,
type Conversation,
type ConversationDetail,
type ConversationMessage,
} from "@/lib/api";
import { cn } from "@/lib/utils";
@@ -93,6 +107,249 @@ function statusLabel(status: string): string {
return "已结束";
}
type TraceEvent = Record<string, unknown>;
type WorkflowNode = { label: string; type: string };
type TraceTone = "default" | "success" | "error" | "user";
type TracePresentation = {
title: string;
description: string;
icon: LucideIcon;
tone: TraceTone;
};
function recordValue(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
}
function textValue(value: unknown): string {
return typeof value === "string" ? value : "";
}
function numberValue(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) ? value : null;
}
function workflowNodes(detail: ConversationDetail): Map<string, WorkflowNode> {
const snapshot = recordValue(detail.extra.workflow?.snapshot);
const nodes = Array.isArray(snapshot.nodes) ? snapshot.nodes : [];
return new Map(
nodes.flatMap((rawNode) => {
const node = recordValue(rawNode);
const id = textValue(node.id);
if (!id) return [];
const data = recordValue(node.data);
const type = textValue(node.type) || "node";
return [
[
id,
{
label: textValue(data.name) || type,
type,
},
] as const,
];
}),
);
}
function nodeLabel(nodes: Map<string, WorkflowNode>, nodeId: unknown): string {
const id = textValue(nodeId);
const node = nodes.get(id);
return node ? `${node.label}` : id ? `${id}` : "当前节点";
}
function toolLabel(toolType: unknown): string {
if (toolType === "client") return "客户端工具";
if (toolType === "http") return "HTTP 服务端工具";
if (toolType === "mcp") return "MCP 服务端工具";
if (toolType === "system") return "系统工具";
return "工具";
}
function tracePresentation(
event: TraceEvent,
nodes: Map<string, WorkflowNode>,
startedEvent?: TraceEvent,
): TracePresentation {
const eventName = textValue(event.event);
const node = nodeLabel(nodes, event.nodeId);
const outcome = recordValue(event.outcome);
const duration = numberValue(outcome.durationMs);
const tool = toolLabel(event.toolType ?? startedEvent?.toolType);
switch (eventName) {
case "node_entered":
return {
title: `进入节点 ${node}`,
description: `节点类型:${textValue(event.nodeType) || nodes.get(textValue(event.nodeId))?.type || "未知"}`,
icon: CircleDot,
tone: "default",
};
case "node_exited":
return {
title: `离开节点 ${node}`,
description: "当前节点处理完成,准备选择下一步。",
icon: ArrowRight,
tone: "default",
};
case "edge_selected":
return {
title: `节点转移:${nodeLabel(nodes, event.sourceNodeId)}${nodeLabel(nodes, event.targetNodeId)}`,
description: `路由方式:${textValue(event.edgeMode) || "always"}`,
icon: GitBranch,
tone: "default",
};
case "action_started":
return {
title: `开始调用${tool}`,
description: `${node} · ${textValue(event.toolId) || "未命名工具"}`,
icon: tool === "客户端工具" ? MonitorSmartphone : Server,
tone: "default",
};
case "action_completed":
return {
title: `${tool}执行成功`,
description: `${node}${duration === null ? "" : ` · ${duration} ms`}`,
icon: CheckCircle2,
tone: "success",
};
case "action_failed":
case "action_cancelled":
return {
title: `${tool}${eventName === "action_failed" ? "执行失败" : "已取消"}`,
description: textValue(recordValue(outcome.error).message) || node,
icon: XCircle,
tone: "error",
};
case "tool_started":
return {
title: `开始调用${tool}`,
description: `${node} · ${textValue(event.toolName) || textValue(event.functionName) || textValue(event.toolId) || "未命名工具"}`,
icon: tool === "客户端工具" ? MonitorSmartphone : Server,
tone: "default",
};
case "tool_completed":
return {
title: `${tool}调用成功`,
description: `${textValue(event.toolName) || textValue(event.functionName) || node}${numberValue(event.durationMs) === null ? "" : ` · ${numberValue(event.durationMs)} ms`}`,
icon: CheckCircle2,
tone: "success",
};
case "tool_failed":
return {
title: `${tool}调用失败`,
description: textValue(event.error) || textValue(event.toolName) || node,
icon: XCircle,
tone: "error",
};
case "client_tool_started":
return {
title:
event.functionName === "show_message"
? "向客户端显示交互消息"
: "向客户端下发工具调用",
description: textValue(event.functionName) || "客户端工具",
icon: MonitorSmartphone,
tone: "default",
};
case "client_tool_completed": {
const userAction = textValue(event.userAction);
return {
title: userAction === "confirmed" ? "用户已确认" : "客户端交互已完成",
description: userAction
? `用户操作:${userAction}`
: textValue(event.functionName) || textValue(event.status) || "执行成功",
icon: CheckCircle2,
tone: userAction ? "user" : "success",
};
}
case "client_tool_failed":
return {
title: "客户端交互失败",
description: textValue(event.error) || textValue(event.functionName),
icon: XCircle,
tone: "error",
};
case "message_started":
return event.requiresConfirmation
? {
title: "等待用户确认",
description: `${node}向客户端显示了确认消息。`,
icon: MonitorSmartphone,
tone: "user",
}
: {
title: "开始播放固定消息",
description: node,
icon: Activity,
tone: "default",
};
case "message_completed": {
const action = textValue(event.action);
return {
title: action === "confirmed" ? "用户已确认" : "消息步骤已完成",
description: `${node}${action ? ` · 操作:${action}` : ""}`,
icon: CheckCircle2,
tone: action ? "user" : "success",
};
}
case "message_interrupted":
return {
title: "用户输入打断消息",
description: node,
icon: Activity,
tone: "user",
};
case "message_failed":
return {
title: "消息步骤失败",
description: textValue(event.error) || node,
icon: XCircle,
tone: "error",
};
case "variables_updated": {
const names = Array.isArray(event.variableNames)
? event.variableNames.filter((name): name is string => typeof name === "string")
: [];
return {
title: "会话变量已更新",
description: names.length ? names.join("、") : node,
icon: Wrench,
tone: "default",
};
}
case "variables_snapshot":
return {
title: "记录会话变量快照",
description: node,
icon: Wrench,
tone: "default",
};
case "workflow_ended":
return {
title: "工作流已结束",
description: `${node} · ${textValue(event.outcome) || "success"}`,
icon: CheckCircle2,
tone: "success",
};
default:
return {
title: eventName || "运行事件",
description: node,
icon: Activity,
tone: "default",
};
}
}
function artifactUrl(path: string): string {
if (/^(https?:|data:|blob:)/.test(path)) return path;
return `${API_BASE}${path.startsWith("/") ? path : `/${path}`}`;
}
export function HistoryPage() {
const [rows, setRows] = useState<Conversation[]>([]);
const [total, setTotal] = useState(0);
@@ -370,7 +627,7 @@ export function HistoryPage() {
<>
<ListPageLayout
title="历史记录"
description="查看每次语音或文字会话中最终确认的用户转写和助手回复。"
description="按会话查看对话、照片、工具调用、用户操作与工作流运行轨迹。"
>
<ListPageSection>
<ListToolbar
@@ -423,7 +680,7 @@ export function HistoryPage() {
</ListPageLayout>
<Dialog open={dialogOpen} onOpenChange={setDialogOpen}>
<DialogContent className="max-h-[calc(100vh-3rem)] grid-rows-[auto_minmax(0,1fr)_auto] overflow-hidden sm:max-w-4xl">
<DialogContent className="max-h-[calc(100vh-3rem)] grid-rows-[auto_minmax(0,1fr)_auto] overflow-hidden sm:max-w-5xl">
<DialogHeader>
<DialogTitle className="flex items-center gap-2">
<MessageSquareText size={18} />
@@ -432,7 +689,7 @@ export function HistoryPage() {
<DialogDescription>
{detail
? `${formatDate(detail.startedAt)} · ${channelLabel(detail.channel)} · ${detail.messageCount} 条消息`
: "查看本次会话中最终确认的文本消息。"}
: "查看本次会话的完整对话与运行记录。"}
</DialogDescription>
</DialogHeader>
@@ -464,56 +721,7 @@ export function HistoryPage() {
</div>
</section>
<section className="rounded-xl border border-hairline bg-surface-strong/20">
<div className="border-b border-hairline px-4 py-3 text-sm font-medium">
</div>
<div className="space-y-5 p-4 sm:p-5">
{detail.messages.map((message) => (
<div
key={message.id}
className={cn(
"flex",
message.role === "user"
? "justify-end"
: "justify-start",
)}
>
<div className="max-w-[85%]">
<div
className={cn(
"mb-1.5 text-xs text-muted-soft",
message.role === "user" && "text-right",
)}
>
{message.role === "user" ? "用户" : "助手"} ·{" "}
{formatDate(message.occurredAt)}
</div>
<div
className={cn(
"whitespace-pre-wrap rounded-2xl px-4 py-3 text-left leading-6",
message.role === "user"
? "rounded-br-md bg-primary text-primary-foreground"
: "rounded-bl-md border border-hairline bg-background text-foreground shadow-sm",
)}
>
{message.content}
</div>
{message.extra.interrupted && (
<div className="mt-1.5 text-xs text-muted-foreground">
</div>
)}
</div>
</div>
))}
{detail.messages.length === 0 && (
<div className="py-12 text-center text-muted-foreground">
</div>
)}
</div>
</section>
<ConversationTimeline detail={detail} />
</div>
)}
</div>
@@ -537,3 +745,237 @@ function Metadata({ label, value }: { label: string; value: string }) {
</div>
);
}
type TimelineEntry =
| {
kind: "message";
timestamp: string;
order: number;
message: ConversationMessage;
}
| {
kind: "trace";
timestamp: string;
order: number;
event: TraceEvent;
};
function timestampOrder(value: string): number {
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? Number.MAX_SAFE_INTEGER : parsed;
}
function ConversationTimeline({ detail }: { detail: ConversationDetail }) {
const nodes = workflowNodes(detail);
const trace = (detail.extra.workflowTrace ?? []).map(recordValue);
const startedByInvocation = new Map<string, TraceEvent>();
trace.forEach((event) => {
if (event.event === "action_started" || event.event === "tool_started") {
const invocationId = textValue(event.invocationId);
if (invocationId) startedByInvocation.set(invocationId, event);
}
});
const entries: TimelineEntry[] = [
...detail.messages.map((message) => ({
kind: "message" as const,
timestamp: message.occurredAt,
order: message.sequence,
message,
})),
...trace.map((event, index) => ({
kind: "trace" as const,
timestamp: textValue(event.timestamp),
order: numberValue(event.sequence) ?? index + 1,
event,
})),
].sort(
(a, b) =>
timestampOrder(a.timestamp) - timestampOrder(b.timestamp) ||
a.order - b.order,
);
const imageCount = detail.messages.reduce(
(count, message) =>
count + (message.artifacts ?? []).filter((item) => item.kind === "image").length,
0,
);
const toolCount = trace.filter(
(event) => event.event === "action_started" || event.event === "tool_started",
).length;
const transitionCount = trace.filter((event) => event.event === "edge_selected").length;
return (
<section className="overflow-hidden rounded-xl border border-hairline bg-surface-strong/20">
<div className="flex flex-wrap items-center justify-between gap-3 border-b border-hairline px-4 py-3">
<div className="text-sm font-medium">线</div>
<div className="flex flex-wrap gap-2">
<Badge variant="secondary" className="gap-1.5 bg-surface-strong text-muted-foreground">
<MessageSquareText size={12} />
{detail.messages.length}
</Badge>
{imageCount > 0 && (
<Badge variant="secondary" className="gap-1.5 bg-surface-strong text-muted-foreground">
<ImageIcon size={12} />
{imageCount}
</Badge>
)}
{toolCount > 0 && (
<Badge variant="secondary" className="gap-1.5 bg-surface-strong text-muted-foreground">
<Wrench size={12} />
{toolCount}
</Badge>
)}
{transitionCount > 0 && (
<Badge variant="secondary" className="gap-1.5 bg-surface-strong text-muted-foreground">
<GitBranch size={12} />
{transitionCount}
</Badge>
)}
</div>
</div>
<div className="space-y-4 p-4 sm:p-5">
{entries.map((entry, index) =>
entry.kind === "message" ? (
<TimelineMessage key={`message-${entry.message.id}`} message={entry.message} />
) : (
<TimelineTrace
key={`trace-${textValue(entry.event.eventId) || index}`}
event={entry.event}
nodes={nodes}
startedEvent={startedByInvocation.get(
textValue(recordValue(entry.event.outcome).invocationId) ||
textValue(entry.event.invocationId),
)}
/>
),
)}
{entries.length === 0 && (
<div className="py-12 text-center text-muted-foreground">
</div>
)}
</div>
</section>
);
}
function TimelineMessage({ message }: { message: ConversationMessage }) {
const isUser = message.role === "user";
const images = (message.artifacts ?? []).filter((item) => item.kind === "image");
const isImageMessage = message.contentType === "image" || images.length > 0;
return (
<div className={cn("flex", isUser ? "justify-end" : "justify-start")}>
<div className="max-w-[88%] sm:max-w-[78%]">
<div
className={cn(
"mb-1.5 flex items-center gap-1.5 text-xs text-muted-soft",
isUser && "justify-end",
)}
>
{isImageMessage && <Camera size={12} />}
{isUser ? "用户" : "助手"} · {formatDate(message.occurredAt)}
</div>
<div
className={cn(
"overflow-hidden rounded-2xl text-left leading-6",
isUser
? "rounded-br-md bg-primary text-primary-foreground"
: "rounded-bl-md border border-hairline bg-background text-foreground shadow-sm",
isImageMessage ? "p-1" : "px-4 py-3",
)}
>
{images.map((image) => {
const url = artifactUrl(image.contentUrl);
return (
<a
key={image.id}
href={url}
target="_blank"
rel="noreferrer"
className="block overflow-hidden rounded-[0.8rem] bg-black/10"
title="在新窗口查看原图"
>
{/* eslint-disable-next-line @next/next/no-img-element */}
<img
src={url}
alt="用户在通话中拍摄的照片"
className="max-h-[28rem] w-full object-contain"
/>
</a>
);
})}
{isImageMessage && images.length === 0 && (
<div className="flex min-h-40 min-w-56 flex-col items-center justify-center gap-2 rounded-[0.8rem] bg-background/10 px-6 text-center text-sm opacity-75">
<ImageIcon size={22} />
</div>
)}
{message.content && (
<div className={isImageMessage ? "px-3 py-2" : ""}>{message.content}</div>
)}
</div>
{(message.extra.source || message.extra.node_id) && (
<div className={cn("mt-1.5 text-[11px] text-muted-soft", isUser && "text-right")}>
{[message.extra.source, message.extra.node_id].filter(Boolean).join(" · ")}
</div>
)}
{message.extra.interrupted && (
<div className="mt-1.5 text-xs text-muted-foreground">
</div>
)}
</div>
</div>
);
}
function TimelineTrace({
event,
nodes,
startedEvent,
}: {
event: TraceEvent;
nodes: Map<string, WorkflowNode>;
startedEvent?: TraceEvent;
}) {
const presentation = tracePresentation(event, nodes, startedEvent);
const Icon = presentation.icon;
const iconClass = {
default: "bg-surface-strong text-muted-foreground",
success: "bg-success/10 text-success",
error: "bg-destructive/10 text-destructive",
user: "bg-primary/10 text-primary",
}[presentation.tone];
return (
<div className="flex gap-3 rounded-xl border border-hairline-soft bg-background/65 p-3">
<div className={cn("flex size-8 shrink-0 items-center justify-center rounded-full", iconClass)}>
<Icon size={15} />
</div>
<div className="min-w-0 flex-1">
<div className="flex flex-wrap items-start justify-between gap-x-4 gap-y-1">
<div>
<div className="text-sm font-medium text-foreground">{presentation.title}</div>
<div className="mt-0.5 text-xs leading-5 text-muted-foreground">
{presentation.description}
</div>
</div>
<time className="shrink-0 text-[11px] tabular-nums text-muted-soft">
{formatDate(textValue(event.timestamp))}
</time>
</div>
<details className="mt-2 text-xs text-muted-foreground">
<summary className="w-fit cursor-pointer select-none hover:text-foreground">
</summary>
<pre className="mt-2 max-h-64 overflow-auto rounded-lg bg-surface-strong p-3 font-mono text-[11px] leading-5 text-foreground">
{JSON.stringify(event, null, 2)}
</pre>
</details>
</div>
</div>
);
}

View File

@@ -91,13 +91,31 @@ function MobileCallTranscript({ messages }: { messages: ChatMessage[] }) {
</span>
<div
className={[
"whitespace-pre-wrap rounded-2xl px-3.5 py-2.5 text-sm leading-6 shadow-sm",
"overflow-hidden whitespace-pre-wrap rounded-2xl text-sm leading-6 shadow-sm",
isAssistant
? "rounded-tl-sm bg-white/10 text-white/90"
: "rounded-tr-sm bg-white text-[#07101a]",
? "rounded-tl-sm bg-white/10 px-3.5 py-2.5 text-white/90"
: `rounded-tr-sm bg-white text-[#07101a] ${
message.attachments?.length ? "p-1" : "px-3.5 py-2.5"
}`,
].join(" ")}
>
{message.content || (message.streaming ? "…" : "")}
{message.attachments?.map((attachment) => (
// eslint-disable-next-line @next/next/no-img-element
<img
key={attachment.id}
src={attachment.url}
alt={attachment.alt}
className="max-h-[52dvh] w-full rounded-[0.8rem] object-cover"
/>
))}
{message.content && (
<div className={message.attachments?.length ? "px-2.5 py-2" : ""}>
{message.content}
</div>
)}
{!message.content && !message.attachments?.length && message.streaming
? "…"
: null}
</div>
</div>
);
@@ -166,7 +184,10 @@ function MobileCallVisualWorkspace({
} | null>(null);
const latestMessage = [...messages]
.reverse()
.find((message) => message.content.trim());
.find(
(message) =>
message.content.trim() || (message.attachments?.length ?? 0) > 0,
);
useEffect(() => {
const container = containerRef.current;
@@ -283,7 +304,10 @@ function MobileCallVisualWorkspace({
<span className="mr-1.5 font-medium text-white/55">
{latestMessage?.role === "user" ? "我:" : "助手:"}
</span>
{latestMessage?.content || "暂无消息,点击返回聊天记录"}
{latestMessage?.content ||
(latestMessage?.attachments?.length
? "发送了一张照片"
: "暂无消息,点击返回聊天记录")}
</span>
</button>
</>

View File

@@ -19,8 +19,64 @@ const PHOTO_BUTTON_DEFINITION = {
],
} as const;
const MAX_PREVIEW_EDGE = 1280;
async function capturePreviewImage(stream: MediaStream | null): Promise<string> {
const track = stream?.getVideoTracks()[0];
if (!stream || !track || track.readyState !== "live") {
throw new Error("当前没有可用的摄像头画面");
}
const video = document.createElement("video");
video.muted = true;
video.playsInline = true;
video.srcObject = stream;
try {
if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
await new Promise<void>((resolve, reject) => {
const timeout = window.setTimeout(
() => reject(new Error("等待摄像头预览超时")),
2_000,
);
video.addEventListener(
"loadeddata",
() => {
window.clearTimeout(timeout);
resolve();
},
{ once: true },
);
});
}
await video.play();
const sourceWidth = video.videoWidth || track.getSettings().width || 0;
const sourceHeight = video.videoHeight || track.getSettings().height || 0;
if (!sourceWidth || !sourceHeight) {
throw new Error("摄像头画面尺寸不可用");
}
const scale = Math.min(1, MAX_PREVIEW_EDGE / Math.max(sourceWidth, sourceHeight));
const canvas = document.createElement("canvas");
canvas.width = Math.max(1, Math.round(sourceWidth * scale));
canvas.height = Math.max(1, Math.round(sourceHeight * scale));
const context = canvas.getContext("2d");
if (!context) throw new Error("浏览器无法生成照片预览");
context.drawImage(video, 0, 0, canvas.width, canvas.height);
return canvas.toDataURL("image/jpeg", 0.85);
} finally {
video.pause();
video.srcObject = null;
}
}
export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
const { registerClientTool, sendUserInput, status } = preview;
const {
appendUserImage,
registerClientTool,
sendUserInput,
status,
videoStream,
} = preview;
const [visible, setVisible] = useState(false);
const [capturing, setCapturing] = useState(false);
const [error, setError] = useState<string | null>(null);
@@ -52,7 +108,9 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
setCapturing(true);
setError(null);
try {
await sendUserInput(
const timestamp = new Date().toISOString();
const imageUrl = await capturePreviewImage(videoStream);
const result = await sendUserInput(
[
{
type: "input_image",
@@ -61,6 +119,7 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
],
{ runImmediately: true, interrupt: true },
);
appendUserImage(result.inputId, imageUrl, timestamp);
} catch (captureError) {
setError(
captureError instanceof Error ? captureError.message : "拍照提交失败",
@@ -68,7 +127,7 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
} finally {
setCapturing(false);
}
}, [capturing, sendUserInput, status]);
}, [appendUserImage, capturing, sendUserInput, status, videoStream]);
return {
visible,

View File

@@ -45,6 +45,14 @@ export type ChatMessage = {
sequence: number;
turnId?: string;
streaming?: boolean;
attachments?: ChatAttachment[];
};
export type ChatAttachment = {
id: string;
type: "image";
url: string;
alt: string;
};
type AppMessage = Record<string, unknown> & { type?: string };
@@ -871,6 +879,34 @@ export function useVoicePreview(
[],
);
const appendUserImage = useCallback(
(inputId: string, imageUrl: string, timestamp: string) => {
messageSeqRef.current += 1;
const sequence = messageSeqRef.current;
setMessages((previous) =>
sortMessages([
...previous,
{
id: `user-image-${inputId}`,
role: "user",
content: "",
timestamp,
sequence,
attachments: [
{
id: `image-${inputId}`,
type: "image",
url: imageUrl,
alt: "用户拍摄的照片",
},
],
},
]),
);
},
[],
);
const updateSession = useCallback(
({
dynamicVariables,
@@ -962,6 +998,7 @@ export function useVoicePreview(
supportsOutputSelection,
sendText,
sendUserInput,
appendUserImage,
updateSession,
registerClientTool,
connect,

View File

@@ -325,7 +325,24 @@ export type ConversationMessage = {
contentType: string;
content: string;
occurredAt: string;
extra: { interrupted?: boolean; turn_id?: string };
extra: {
interrupted?: boolean;
turn_id?: string;
source?: string;
node_id?: string;
input_id?: string;
};
artifacts: ConversationArtifact[];
};
export type ConversationArtifact = {
id: string;
kind: string;
contentUrl: string;
mimeType: string;
sizeBytes: number | null;
durationMs: number | null;
extra: Record<string, unknown>;
};
export type ConversationDetail = Conversation & {