feat: support uploaded images in debug voice preview

Allow paste/drag temporary image assets so vision turns work without a live camera frame. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-05 19:52:37 +08:00
parent 555c8f5fa6
commit e36ca308b8
16 changed files with 686 additions and 94 deletions
--- a/frontend/src/components/assistant-editor/debug-preview.tsx
+++ b/frontend/src/components/assistant-editor/debug-preview.tsx
@@ -7,6 +7,7 @@ import {
  Braces,
  Check,
  Copy,
+  ImageIcon,
  Loader2,
  MessageSquareText,
  Mic,
@@ -51,16 +52,46 @@ import {
  useVoicePreview,
  type ChatMessage,
  type ClientToolDefinition,
+  type UserInputPart,
  type VoicePreview,
  type VoicePreviewStatus,
 } from "@/hooks/use-voice-preview";
-import type { DynamicVariableDefinition } from "@/lib/api";
+import {
+  inputAssetsApi,
+  type DynamicVariableDefinition,
+} from "@/lib/api";

 type VizStyle = "aura" | "nebula" | "bars" | "wave";

 // 调试面板顶部主视图:聊天记录 / 视频流
 type DebugView = "chat" | "video";
 type DebugInputMode = "mic" | "text";
+type PendingDebugImage = {
+  file: File;
+  previewUrl: string;
+};
+
+const DEBUG_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
+
+function fileToDataUrl(file: File): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = () => resolve(String(reader.result));
+    reader.onerror = () => reject(new Error("无法读取图片"));
+    reader.readAsDataURL(file);
+  });
+}
+
+function hasDraggedImage(dataTransfer: DataTransfer): boolean {
+  return (
+    Array.from(dataTransfer.items).some((item) =>
+      item.type.startsWith("image/"),
+    ) ||
+    Array.from(dataTransfer.files).some((file) =>
+      file.type.startsWith("image/"),
+    )
+  );
+}

 const VIZ_OPTIONS: { style: VizStyle; label: string; icon: React.ReactNode }[] =
  [
@@ -766,6 +797,8 @@ function DebugVoicePanel({
    remoteStream,
    messages,
    sendText,
+    sendUserInput,
+    appendUserImage,
    connect,
    disconnect,
    audioRef,
@@ -773,6 +806,11 @@ function DebugVoicePanel({
  const recording = status === "connecting" || status === "connected";
  const [textDraft, setTextDraft] = useState("");
  const [inputMode, setInputMode] = useState<DebugInputMode>("mic");
+  const [pendingImage, setPendingImage] =
+    useState<PendingDebugImage | null>(null);
+  const [inputError, setInputError] = useState("");
+  const [sendingInput, setSendingInput] = useState(false);
+  const [draggingImage, setDraggingImage] = useState(false);
  const [clientDialogContainer, setClientDialogContainer] =
    useState<HTMLDivElement | null>(null);
  const [messageDialogOpen, setMessageDialogOpen] = useState(false);
@@ -790,9 +828,110 @@ function DebugVoicePanel({
      : "";
  const startDisabled = status === "connecting" || Boolean(startBlockedMessage);

-  function handleSendText() {
-    if (sendText(textDraft)) {
+  useEffect(() => {
+    return () => {
+      if (pendingImage) URL.revokeObjectURL(pendingImage.previewUrl);
+    };
+  }, [pendingImage]);
+
+  function stageImage(file: File) {
+    setInputError("");
+    if (!vision) {
+      setInputError("请先为助手开启视觉理解，再添加图片。");
+      return;
+    }
+    if (!file.type.startsWith("image/")) {
+      setInputError("只能添加图片文件。");
+      return;
+    }
+    if (file.size > DEBUG_IMAGE_MAX_BYTES) {
+      setInputError("图片不能超过 10 MB。");
+      return;
+    }
+    setPendingImage({ file, previewUrl: URL.createObjectURL(file) });
+    setInputMode("text");
+  }
+
+  async function handleSendInput() {
+    const text = textDraft.trim();
+    if (!pendingImage) {
+      if (sendText(text)) {
+        setTextDraft("");
+        setInputError("");
+      }
+      return;
+    }
+    if (status !== "connected" || sendingInput) return;
+
+    setSendingInput(true);
+    setInputError("");
+    let assetToken = "";
+    try {
+      const imageUrl = await fileToDataUrl(pendingImage.file);
+      const asset = await inputAssetsApi.uploadImage(pendingImage.file);
+      assetToken = asset.assetToken;
+      const parts: UserInputPart[] = [];
+      if (text) parts.push({ type: "input_text", text });
+      parts.push({
+        type: "input_image",
+        source: { type: "uploaded_asset", asset_token: assetToken },
+      });
+
+      const timestamp = new Date().toISOString();
+      const result = await sendUserInput(parts);
+      appendUserImage(result.inputId, imageUrl, timestamp, text);
      setTextDraft("");
+      setPendingImage(null);
+    } catch (sendError) {
+      if (assetToken) {
+        void inputAssetsApi.remove(assetToken).catch(() => {});
+      }
+      setInputError(
+        sendError instanceof Error ? sendError.message : "图片发送失败，请重试。",
+      );
+    } finally {
+      setSendingInput(false);
+    }
+  }
+
+  function handlePaste(event: React.ClipboardEvent<HTMLTextAreaElement>) {
+    const image = Array.from(event.clipboardData.items)
+      .find((item) => item.type.startsWith("image/"))
+      ?.getAsFile();
+    if (!image) return;
+    event.preventDefault();
+    stageImage(image);
+  }
+
+  function handleDragOver(event: React.DragEvent<HTMLDivElement>) {
+    if (!hasDraggedImage(event.dataTransfer)) return;
+    event.preventDefault();
+    event.dataTransfer.dropEffect = "copy";
+    setDraggingImage(true);
+  }
+
+  function handleDragLeave(event: React.DragEvent<HTMLDivElement>) {
+    const nextTarget = event.relatedTarget;
+    if (nextTarget instanceof Node && event.currentTarget.contains(nextTarget)) {
+      return;
+    }
+    setDraggingImage(false);
+  }
+
+  function handleDrop(event: React.DragEvent<HTMLDivElement>) {
+    if (!hasDraggedImage(event.dataTransfer)) return;
+    event.preventDefault();
+    setDraggingImage(false);
+    const image = Array.from(event.dataTransfer.files).find((file) =>
+      file.type.startsWith("image/"),
+    );
+    if (image) stageImage(image);
+  }
+
+  function removePendingImage() {
+    if (!sendingInput) {
+      setPendingImage(null);
+      setInputError("");
    }
  }

@@ -968,61 +1107,136 @@ function DebugVoicePanel({
      </div>

      <div className="shrink-0 border-t border-hairline bg-card p-3">
-        <div className="flex items-center gap-2">
-          <div
-            className={[
-              "flex h-10 min-w-0 items-center gap-1 rounded-[1.4rem] border border-hairline-strong bg-background px-2",
-              "flex-1",
-            ].join(" ")}
-          >
-            <div className="flex shrink-0 items-center gap-0.5 rounded-full bg-canvas-soft p-0.5">
-              <DebugInputModeButton
-                selected={inputMode === "mic"}
-                label="选择麦克风设备"
-                onClick={() => setInputMode("mic")}
-              >
-                <Mic size={15} />
-              </DebugInputModeButton>
-              <DebugInputModeButton
-                selected={inputMode === "text"}
-                label="文字输入"
-                onClick={() => setInputMode("text")}
-              >
-                <MessageSquareText size={15} />
-              </DebugInputModeButton>
+        <div className="flex items-end gap-2">
+          <div className="min-w-0 flex-1">
+            <div
+              onDragOver={handleDragOver}
+              onDragLeave={handleDragLeave}
+              onDrop={handleDrop}
+              className={[
+                "relative flex min-h-10 min-w-0 items-end gap-1 overflow-hidden rounded-[1.4rem] border bg-background px-2 transition-colors",
+                draggingImage
+                  ? "border-foreground"
+                  : "border-hairline-strong",
+              ].join(" ")}
+            >
+              <div className="mb-1 flex shrink-0 items-center gap-0.5 rounded-full bg-canvas-soft p-0.5">
+                <DebugInputModeButton
+                  selected={inputMode === "mic"}
+                  label="选择麦克风设备"
+                  onClick={() => setInputMode("mic")}
+                >
+                  <Mic size={15} />
+                </DebugInputModeButton>
+                <DebugInputModeButton
+                  selected={inputMode === "text"}
+                  label="文字输入"
+                  onClick={() => setInputMode("text")}
+                >
+                  <MessageSquareText size={15} />
+                </DebugInputModeButton>
+              </div>
+              {inputMode === "mic" ? (
+                <div className="flex min-w-0 flex-1">
+                  <MicrophoneDeviceField preview={preview} />
+                </div>
+              ) : (
+                <div className="min-w-0 flex-1 py-1">
+                  {pendingImage && (
+                    <div className="flex items-center gap-2 px-2 pb-1 pt-0.5">
+                      <div className="group relative h-14 w-14 shrink-0 overflow-hidden rounded-xl border border-hairline bg-canvas-soft">
+                        {/* eslint-disable-next-line @next/next/no-img-element */}
+                        <img
+                          src={pendingImage.previewUrl}
+                          alt="待发送图片预览"
+                          className="h-full w-full object-cover"
+                        />
+                        <button
+                          type="button"
+                          aria-label="移除待发送图片"
+                          title="移除图片"
+                          disabled={sendingInput}
+                          onClick={removePendingImage}
+                          className="absolute right-1 top-1 flex h-5 w-5 items-center justify-center rounded-full bg-background/90 text-foreground shadow-sm transition-colors hover:bg-background disabled:cursor-not-allowed"
+                        >
+                          <X size={12} />
+                        </button>
+                        {sendingInput && (
+                          <span className="absolute inset-0 flex items-center justify-center bg-background/65">
+                            <Loader2 size={18} className="animate-spin" />
+                          </span>
+                        )}
+                      </div>
+                      <div className="min-w-0">
+                        <p className="truncate text-xs font-medium text-foreground">
+                          {pendingImage.file.name || "粘贴的图片"}
+                        </p>
+                        <p className="mt-0.5 text-[11px] text-muted-soft">
+                          按 Enter 上传并发送
+                        </p>
+                      </div>
+                    </div>
+                  )}
+                  <Textarea
+                    rows={1}
+                    value={textDraft}
+                    disabled={status !== "connected" || sendingInput}
+                    onChange={(event) => {
+                      setTextDraft(event.target.value);
+                      if (inputError) setInputError("");
+                    }}
+                    onPaste={handlePaste}
+                    onKeyDown={(event) => {
+                      if (
+                        event.key === "Enter" &&
+                        !event.shiftKey &&
+                        !event.nativeEvent.isComposing
+                      ) {
+                        event.preventDefault();
+                        void handleSendInput();
+                      }
+                    }}
+                    placeholder={
+                      status === "connected"
+                        ? vision
+                          ? "输入文字，或粘贴 / 拖入图片…"
+                          : "输入文字发送给助手，将打断当前播报…"
+                        : "开始对话后可输入文字…"
+                    }
+                    className="max-h-24 min-h-8 resize-none overflow-y-auto border-transparent bg-transparent px-2 py-1 text-sm leading-6 text-foreground shadow-none outline-none placeholder:text-muted-soft focus-visible:ring-0 disabled:opacity-100"
+                  />
+                </div>
+              )}
+              {draggingImage && (
+                <div className="pointer-events-none absolute inset-0 z-20 flex items-center justify-center gap-2 rounded-[1.4rem] bg-background/95 text-xs font-medium text-foreground">
+                  <ImageIcon size={16} />
+                  松开以添加图片
+                </div>
+              )}
            </div>
-            {inputMode === "mic" ? (
-              <MicrophoneDeviceField preview={preview} />
-            ) : (
-              <Textarea
-                rows={1}
-                value={textDraft}
-                disabled={status !== "connected"}
-                onChange={(event) => setTextDraft(event.target.value)}
-                onKeyDown={(event) => {
-                  if (event.key === "Enter" && !event.shiftKey && !event.nativeEvent.isComposing) {
-                    event.preventDefault();
-                    handleSendText();
-                  }
-                }}
-                placeholder={
-                  status === "connected"
-                    ? "输入文字发送给助手，将打断当前播报…"
-                    : "开始对话后可输入文字…"
-                }
-                className="h-8 min-h-8 flex-1 resize-none overflow-hidden border-transparent bg-transparent px-2 py-1 text-sm leading-6 text-foreground shadow-none outline-none placeholder:text-muted-soft focus-visible:ring-0 disabled:opacity-100"
-              />
+            {inputError && (
+              <p className="px-2 pt-1 text-[11px] leading-4 text-destructive">
+                {inputError}
+              </p>
            )}
          </div>
          {inputMode === "text" && (
            <Button
              size="icon"
              className="h-10 w-10 shrink-0 rounded-full"
-              aria-label="发送调试消息"
-              disabled={status !== "connected" || !textDraft.trim()}
-              onClick={handleSendText}
+              aria-label={sendingInput ? "正在发送图片" : "发送调试消息"}
+              disabled={
+                status !== "connected" ||
+                sendingInput ||
+                (!textDraft.trim() && !pendingImage)
+              }
+              onClick={() => void handleSendInput()}
            >
-              <Send size={16} />
+              {sendingInput ? (
+                <Loader2 size={16} className="animate-spin" />
+              ) : (
+                <Send size={16} />
+              )}
            </Button>
          )}
          {!showIdleHub && (
--- a/frontend/src/hooks/use-voice-preview.ts
+++ b/frontend/src/hooks/use-voice-preview.ts
@@ -61,7 +61,9 @@ export type UserInputPart =
  | { type: "input_text"; text: string }
  | {
      type: "input_image";
-      source: { type: "camera_frame"; frame: "current" };
+      source:
+        | { type: "camera_frame"; frame: "current" }
+        | { type: "uploaded_asset"; asset_token: string };
    };
 export type UserInputResult = {
  inputId: string;
@@ -880,7 +882,12 @@ export function useVoicePreview(
  );

  const appendUserImage = useCallback(
-    (inputId: string, imageUrl: string, timestamp: string) => {
+    (
+      inputId: string,
+      imageUrl: string,
+      timestamp: string,
+      content = "",
+    ) => {
      messageSeqRef.current += 1;
      const sequence = messageSeqRef.current;
      setMessages((previous) =>
@@ -889,7 +896,7 @@ export function useVoicePreview(
          {
            id: `user-image-${inputId}`,
            role: "user",
-            content: "",
+            content: content.trim(),
            timestamp,
            sequence,
            attachments: [
@@ -897,7 +904,7 @@ export function useVoicePreview(
                id: `image-${inputId}`,
                type: "image",
                url: imageUrl,
-                alt: "用户拍摄的照片",
+                alt: "用户提交的图片",
              },
            ],
          },
--- a/frontend/src/lib/api.ts
+++ b/frontend/src/lib/api.ts
@@ -389,6 +389,30 @@ export const conversationsApi = {
    request<{ ok: boolean }>(`/api/conversations/${id}`, { method: "DELETE" }),
 };

+// ---------- 调试会话临时图片 ----------
+export type InputImageAsset = {
+  assetToken: string;
+  width: number;
+  height: number;
+  sizeBytes: number;
+};
+
+export const inputAssetsApi = {
+  uploadImage: (file: File) => {
+    const body = new FormData();
+    body.append("file", file);
+    return request<InputImageAsset>("/api/input-assets/image", {
+      method: "POST",
+      body,
+    });
+  },
+  remove: (assetToken: string) =>
+    request<{ ok: boolean }>(
+      `/api/input-assets/${encodeURIComponent(assetToken)}`,
+      { method: "DELETE" },
+    ),
+};
+
 // ---------- 工具 ----------
 export type ToolStatus = "active" | "archived" | "draft";
 export type ToolExecutionMode = "immediate" | "async";