From e36ca308b88811e9c42e8bbcfe6c77ec191ee567 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Wed, 5 Aug 2026 19:52:37 +0800 Subject: [PATCH] feat: support uploaded images in debug voice preview Allow paste/drag temporary image assets so vision turns work without a live camera frame. Co-authored-by: Cursor --- backend/app.py | 2 + backend/requirements.txt | 2 +- backend/routes/input_assets.py | 59 ++++ backend/services/conversation_history.py | 12 +- backend/services/input_assets.py | 132 ++++++++ backend/services/pipecat/pipeline.py | 51 ++- backend/services/pipecat/pipeline_events.py | 11 +- backend/services/pipecat/processors.py | 41 ++- backend/services/vision.py | 15 + backend/settings.py | 8 + backend/tests/test_conversation_history.py | 4 + backend/tests/test_input_assets.py | 58 ++++ backend/tests/test_user_input.py | 32 +- .../assistant-editor/debug-preview.tsx | 314 +++++++++++++++--- frontend/src/hooks/use-voice-preview.ts | 15 +- frontend/src/lib/api.ts | 24 ++ 16 files changed, 686 insertions(+), 94 deletions(-) create mode 100644 backend/routes/input_assets.py create mode 100644 backend/services/input_assets.py create mode 100644 backend/tests/test_input_assets.py diff --git a/backend/app.py b/backend/app.py index 35f3041..087ee57 100644 --- a/backend/app.py +++ b/backend/app.py @@ -26,6 +26,7 @@ from routes import ( auth, conversations, health, + input_assets, knowledge_bases, mcp_servers, model_registry, @@ -60,6 +61,7 @@ app.add_middleware( app.include_router(health.router) app.include_router(auth.router) app.include_router(conversations.router) +app.include_router(input_assets.router) app.include_router(assistants.router) app.include_router(knowledge_bases.router) app.include_router(mcp_servers.router) diff --git a/backend/requirements.txt b/backend/requirements.txt index 494ee8f..9d44e51 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -6,7 +6,7 @@ pipecat-ai[webrtc,websocket,silero,openai,mcp]==1.7.0 Pillow>=11.1.0,<13 # FastGPT 类型助手:本地 SDK(包 /api/v1/chat/completions 流式 + chatId 会话) -fastgpt-client @ file:///Users/wangx/Code/AI-VideoAssistant-Project/fastgpt-python-sdk +# fastgpt-client @ file:///Users/wangx/Code/AI-VideoAssistant-Project/fastgpt-python-sdk # Dify 类型助手:异步流式 Runtime API SDK dify-client-python==1.0.3 diff --git a/backend/routes/input_assets.py b/backend/routes/input_assets.py new file mode 100644 index 0000000..8880f27 --- /dev/null +++ b/backend/routes/input_assets.py @@ -0,0 +1,59 @@ +"""Authenticated upload endpoint for pending debug-chat image attachments.""" + +from __future__ import annotations + +import asyncio + +from fastapi import APIRouter, Depends, File, HTTPException, UploadFile +from pydantic import BaseModel + +import settings +from services.auth import require_admin +from services.input_assets import discard_input_image, store_input_image + + +router = APIRouter( + prefix="/api/input-assets", + tags=["input-assets"], + dependencies=[Depends(require_admin)], +) + + +class InputImageAssetOut(BaseModel): + assetToken: str + width: int + height: int + sizeBytes: int + + +@router.post("/image", response_model=InputImageAssetOut) +async def upload_input_image(file: UploadFile = File(...)): + data = await file.read(settings.INPUT_IMAGE_MAX_BYTES + 1) + if len(data) > settings.INPUT_IMAGE_MAX_BYTES: + limit_mb = settings.INPUT_IMAGE_MAX_BYTES // 1024 // 1024 + raise HTTPException(413, f"图片不能超过 {limit_mb} MB") + if not data: + raise HTTPException(400, "图片文件不能为空") + try: + stored = await asyncio.to_thread(store_input_image, data) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc + except Exception as exc: + raise HTTPException(503, "图片暂时无法上传，请确认对象存储可用") from exc + return InputImageAssetOut( + assetToken=stored.token, + width=stored.width, + height=stored.height, + sizeBytes=stored.size_bytes, + ) + + +@router.delete("/{asset_token}") +async def delete_input_image(asset_token: str): + try: + await asyncio.to_thread(discard_input_image, asset_token) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc + except Exception as exc: + raise HTTPException(503, "图片附件暂时无法清理") from exc + return {"ok": True} diff --git a/backend/services/conversation_history.py b/backend/services/conversation_history.py index e07b6a0..6104526 100644 --- a/backend/services/conversation_history.py +++ b/backend/services/conversation_history.py @@ -176,6 +176,8 @@ class ConversationRecorder: *, input_id: str, timestamp: object, + content: str = "", + source: str = "camera_capture", mime_type: str = "image/jpeg", ) -> None: """Persist a captured frame without delaying the realtime model turn.""" @@ -185,6 +187,8 @@ class ConversationRecorder: data, input_id=input_id, timestamp=timestamp, + content=content, + source=source, mime_type=mime_type, ), name=f"conversation-image:{self.session_id}:{input_id}", @@ -198,6 +202,8 @@ class ConversationRecorder: *, input_id: str, timestamp: object, + content: str, + source: str, mime_type: str, ) -> None: message_id = f"msg_{uuid4().hex[:20]}" @@ -221,11 +227,11 @@ class ConversationRecorder: sequence=next_sequence, role="user", content_type="image", - content="", + content=content.strip(), occurred_at=_parse_timestamp(timestamp), extra={ "input_id": input_id, - "source": "camera_capture", + "source": source, }, ) ) @@ -238,7 +244,7 @@ class ConversationRecorder: storage_uri=storage_uri(key), mime_type=mime_type, size_bytes=len(data), - extra={"input_id": input_id}, + extra={"input_id": input_id, "source": source}, ) ) conversation = await db.get(ConversationSession, self.session_id) diff --git a/backend/services/input_assets.py b/backend/services/input_assets.py new file mode 100644 index 0000000..bbba40b --- /dev/null +++ b/backend/services/input_assets.py @@ -0,0 +1,132 @@ +"""Short-lived uploaded images consumed by the realtime user-input protocol.""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import time +from dataclasses import dataclass +from io import BytesIO +from uuid import uuid4 + +from PIL import Image, ImageOps, UnidentifiedImageError + +import settings +from services.object_storage import delete_object, get_object, put_object + + +INPUT_ASSET_PREFIX = "conversation-inputs/" + + +@dataclass(frozen=True) +class StoredInputImage: + token: str + width: int + height: int + size_bytes: int + + +def _b64encode(data: bytes) -> str: + return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") + + +def _b64decode(data: str) -> bytes: + padding = "=" * (-len(data) % 4) + return base64.urlsafe_b64decode(f"{data}{padding}".encode("ascii")) + + +def _signature(payload: str) -> str: + digest = hmac.new( + settings.AUTH_SECRET_KEY.encode("utf-8"), + payload.encode("ascii"), + hashlib.sha256, + ).digest() + return _b64encode(digest) + + +def _normalize_image(data: bytes) -> tuple[bytes, int, int]: + try: + source = Image.open(BytesIO(data)) + width, height = source.size + if width <= 0 or height <= 0: + raise ValueError("图片尺寸无效") + if width * height > settings.INPUT_IMAGE_MAX_PIXELS: + raise ValueError("图片像素尺寸过大") + source.load() + except (UnidentifiedImageError, OSError) as exc: + raise ValueError("文件不是可识别的图片") from exc + + image = ImageOps.exif_transpose(source) + image.thumbnail( + (settings.INPUT_IMAGE_MAX_EDGE, settings.INPUT_IMAGE_MAX_EDGE), + Image.Resampling.LANCZOS, + ) + if image.mode in {"RGBA", "LA"} or "transparency" in image.info: + rgba = image.convert("RGBA") + background = Image.new("RGB", rgba.size, "white") + background.paste(rgba, mask=rgba.getchannel("A")) + image = background + else: + image = image.convert("RGB") + + buffer = BytesIO() + image.save(buffer, format="JPEG", quality=85, optimize=True) + return buffer.getvalue(), image.width, image.height + + +def store_input_image(data: bytes) -> StoredInputImage: + normalized, width, height = _normalize_image(data) + key = f"{INPUT_ASSET_PREFIX}{uuid4().hex}.jpg" + put_object(key, normalized, "image/jpeg") + expires_at = int(time.time()) + settings.INPUT_IMAGE_TOKEN_TTL_SECONDS + encoded_payload = _b64encode( + json.dumps( + {"key": key, "exp": expires_at}, + separators=(",", ":"), + ).encode("utf-8") + ) + token = f"{encoded_payload}.{_signature(encoded_payload)}" + return StoredInputImage( + token=token, + width=width, + height=height, + size_bytes=len(normalized), + ) + + +def _key_from_token(token: str) -> str: + if not token or "." not in token: + raise ValueError("图片附件 token 无效") + encoded_payload, signature = token.rsplit(".", 1) + if not hmac.compare_digest(_signature(encoded_payload), signature): + raise ValueError("图片附件 token 签名无效") + try: + payload = json.loads(_b64decode(encoded_payload)) + key = str(payload.get("key") or "") + expires_at = int(payload.get("exp") or 0) + except (ValueError, TypeError, json.JSONDecodeError) as exc: + raise ValueError("图片附件 token 内容无效") from exc + if expires_at < int(time.time()): + raise ValueError("图片附件已过期，请重新添加") + if not key.startswith(INPUT_ASSET_PREFIX): + raise ValueError("图片附件存储位置无效") + return key + + +def consume_input_image(token: str) -> bytes: + """Read a signed image once and remove its temporary object.""" + + key = _key_from_token(token) + data = get_object(key) + try: + delete_object(key) + except Exception: + # Consumption succeeded. A stale temporary object must not break the turn. + pass + return data + + +def discard_input_image(token: str) -> None: + delete_object(_key_from_token(token)) diff --git a/backend/services/pipecat/pipeline.py b/backend/services/pipecat/pipeline.py index a402c2a..e4e67c6 100644 --- a/backend/services/pipecat/pipeline.py +++ b/backend/services/pipecat/pipeline.py @@ -27,6 +27,7 @@ from services.pipecat.service_factory import ( ) from db.session import SessionLocal from services.knowledge import search as search_knowledge +from services.input_assets import consume_input_image from services.client_tools import ClientToolBroker from services.tool_policy import policy_for_tool from services.vision import ( @@ -36,6 +37,7 @@ from services.vision import ( config_with_main_llm_as_vision, config_with_vision_resource, image_data_uri, + image_frame_from_jpeg, image_jpeg_bytes, ) from services.workflow_engine import WorkflowEngine @@ -734,7 +736,7 @@ async def run_pipeline( greeting = await brain.greeting(cfg) async def submit_user_input(value: UserInput) -> None: - if not value.has_camera_frame: + if not value.has_image: if not value.run_immediately: brain.record_user_message(value.text) await worker.queue_frame( @@ -749,28 +751,49 @@ async def run_pipeline( raise ValueError("P0 图片输入必须立即触发回复") if not vision_enabled: raise ValueError("当前助手未启用视觉能力") - user_id = vision_state.get("client_id") - if not user_id: - raise ValueError("当前没有可用的摄像头视频流") native_vision = active_vision_uses_main_llm() analysis_cfg = None if native_vision else active_vision_config() - request = UserImageRequestFrame( - user_id=user_id, - text=value.prompt_text, - append_to_context=False, - ) - try: - image_frame = await vision_capture.request_image(llm, request) - except asyncio.TimeoutError as exc: - raise ValueError("等待摄像头视频帧超时") from exc + if value.image_asset_token: + try: + image_bytes = await asyncio.to_thread( + consume_input_image, + value.image_asset_token, + ) + image_frame = await asyncio.to_thread( + image_frame_from_jpeg, + image_bytes, + ) + except ValueError: + raise + except Exception as exc: + raise ValueError("上传图片暂时无法读取") from exc + else: + user_id = vision_state.get("client_id") + if not user_id: + raise ValueError("当前没有可用的摄像头视频流") + request = UserImageRequestFrame( + user_id=user_id, + text=value.prompt_text, + append_to_context=False, + ) + try: + image_frame = await vision_capture.request_image(llm, request) + except asyncio.TimeoutError as exc: + raise ValueError("等待摄像头视频帧超时") from exc + image_bytes = await asyncio.to_thread(image_jpeg_bytes, image_frame) if recorder: - image_bytes = await asyncio.to_thread(image_jpeg_bytes, image_frame) recorder.record_image_later( image_bytes, input_id=value.input_id, timestamp=time_now_iso8601(), + content=value.text, + source=( + "uploaded_asset" + if value.image_asset_token + else "camera_capture" + ), ) if native_vision: diff --git a/backend/services/pipecat/pipeline_events.py b/backend/services/pipecat/pipeline_events.py index c414a17..7cd527a 100644 --- a/backend/services/pipecat/pipeline_events.py +++ b/backend/services/pipecat/pipeline_events.py @@ -149,11 +149,12 @@ def bind_cascade_pipeline_events( @text_input.event_handler("on_user_input") async def on_user_input(_processor, user_input: UserInput): - await queue_transcript( - "user", - user_input.transcript_text, - time_now_iso8601(), - ) + if not user_input.has_image: + await queue_transcript( + "user", + user_input.transcript_text, + time_now_iso8601(), + ) if user_input.run_immediately and user_input.interrupt: pending_user_inputs.append(user_input) return diff --git a/backend/services/pipecat/processors.py b/backend/services/pipecat/processors.py index 88fb90e..e8628ff 100644 --- a/backend/services/pipecat/processors.py +++ b/backend/services/pipecat/processors.py @@ -50,9 +50,14 @@ class UserInput: input_id: str text: str has_camera_frame: bool + image_asset_token: str | None run_immediately: bool interrupt: bool + @property + def has_image(self) -> bool: + return self.has_camera_frame or bool(self.image_asset_token) + @property def prompt_text(self) -> str: if self.text: @@ -61,10 +66,9 @@ class UserInput: @property def transcript_text(self) -> str: - # Images are represented as structured media records. A synthetic - # sentence would be shown as chat text and persisted as if the user had - # said it, while adding no visual information for the model. - return self.text + # Image turns are persisted as one structured media message, including + # any optional text caption. Do not create a duplicate transcript row. + return "" if self.has_image else self.text class UserInputError(ValueError): @@ -155,6 +159,7 @@ def parse_user_input(message) -> UserInput | None: text = "" has_camera_frame = False + image_asset_token: str | None = None for part in parts: if not isinstance(part, dict): raise UserInputError("user-input part 格式不正确", input_id=input_id) @@ -166,23 +171,30 @@ def parse_user_input(message) -> UserInput | None: if not text: raise UserInputError("input_text 不能为空", input_id=input_id) elif part_type == "input_image": - if has_camera_frame: + if has_camera_frame or image_asset_token: raise UserInputError("P0 只支持一张图片", input_id=input_id) source = part.get("source") - if ( - not isinstance(source, dict) - or source.get("type") != "camera_frame" - or source.get("frame") != "current" - ): + if not isinstance(source, dict): + raise UserInputError("input_image source 格式不正确", input_id=input_id) + if source.get("type") == "camera_frame" and source.get("frame") == "current": + has_camera_frame = True + elif source.get("type") == "uploaded_asset": + token = str(source.get("asset_token") or "").strip() + if not token or len(token) > 4096: + raise UserInputError( + "上传图片缺少有效的 asset_token", + input_id=input_id, + ) + image_asset_token = token + else: raise UserInputError( - "P0 只支持当前摄像头画面", + "仅支持当前摄像头画面或已上传图片", input_id=input_id, ) - has_camera_frame = True else: raise UserInputError(f"不支持的输入类型: {part_type}", input_id=input_id) - if not text and not has_camera_frame: + if not text and not (has_camera_frame or image_asset_token): raise UserInputError("user-input 没有有效内容", input_id=input_id) options = message.get("options") @@ -193,6 +205,7 @@ def parse_user_input(message) -> UserInput | None: input_id=input_id, text=text, has_camera_frame=has_camera_frame, + image_asset_token=image_asset_token, run_immediately=run_immediately, interrupt=interrupt, ) @@ -565,7 +578,7 @@ class RealtimeUserInputProcessor(FrameProcessor): "当前工作流节点暂不接收用户输入", ) return - if user_input.has_camera_frame: + if user_input.has_image: await self._emit_error( user_input.input_id, "Realtime 模式暂不支持图片输入", diff --git a/backend/services/vision.py b/backend/services/vision.py index 86a166b..a819ca3 100644 --- a/backend/services/vision.py +++ b/backend/services/vision.py @@ -49,6 +49,21 @@ def image_data_uri(frame: UserImageRawFrame) -> str: return f"data:image/jpeg;base64,{encoded}" +def image_frame_from_jpeg(data: bytes) -> UserImageRawFrame: + """Decode one normalized uploaded JPEG into Pipecat's raw image frame.""" + + try: + image = Image.open(BytesIO(data)).convert("RGB") + image.load() + except (OSError, ValueError) as exc: + raise ValueError("上传图片无法解码") from exc + return UserImageRawFrame( + image=image.tobytes(), + size=image.size, + format="RGB", + ) + + async def analyze_image_with_vision_model( cfg: AssistantConfig, frame: UserImageRawFrame, diff --git a/backend/settings.py b/backend/settings.py index 7a85e5b..4208e6a 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -59,3 +59,11 @@ S3_BUCKET = os.getenv("S3_BUCKET", "ai-video") S3_REGION = os.getenv("S3_REGION", "us-east-1") KNOWLEDGE_MAX_FILE_BYTES = int(os.getenv("KNOWLEDGE_MAX_FILE_BYTES", "20971520")) KNOWLEDGE_TOP_K = int(os.getenv("KNOWLEDGE_TOP_K", "5")) + +# ---- Realtime user-input image attachments ---- +INPUT_IMAGE_MAX_BYTES = int(os.getenv("INPUT_IMAGE_MAX_BYTES", "10485760")) +INPUT_IMAGE_MAX_PIXELS = int(os.getenv("INPUT_IMAGE_MAX_PIXELS", "24000000")) +INPUT_IMAGE_MAX_EDGE = int(os.getenv("INPUT_IMAGE_MAX_EDGE", "2048")) +INPUT_IMAGE_TOKEN_TTL_SECONDS = int( + os.getenv("INPUT_IMAGE_TOKEN_TTL_SECONDS", "600") +) diff --git a/backend/tests/test_conversation_history.py b/backend/tests/test_conversation_history.py index 650f18b..885414d 100644 --- a/backend/tests/test_conversation_history.py +++ b/backend/tests/test_conversation_history.py @@ -139,6 +139,8 @@ class ConversationRecorderTest(unittest.IsolatedAsyncioTestCase): input_id="input_photo", timestamp="2026-08-05T10:00:00+08:00", mime_type="image/jpeg", + content="帮我看看", + source="uploaded_asset", ) self.assertTrue(session.committed) @@ -146,7 +148,9 @@ class ConversationRecorderTest(unittest.IsolatedAsyncioTestCase): message, artifact = session.added self.assertEqual(message.content_type, "image") self.assertEqual(message.role, "user") + self.assertEqual(message.content, "帮我看看") self.assertEqual(message.extra["input_id"], "input_photo") + self.assertEqual(message.extra["source"], "uploaded_asset") self.assertEqual(artifact.message_id, message.id) self.assertEqual(artifact.kind, "image") self.assertEqual(artifact.size_bytes, len(b"jpeg-data")) diff --git a/backend/tests/test_input_assets.py b/backend/tests/test_input_assets.py new file mode 100644 index 0000000..77ca45c --- /dev/null +++ b/backend/tests/test_input_assets.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import unittest +from io import BytesIO +from unittest.mock import patch + +from PIL import Image + +from services.input_assets import consume_input_image, store_input_image + + +def png_bytes(size: tuple[int, int] = (32, 24)) -> bytes: + output = BytesIO() + Image.new("RGBA", size, (20, 80, 160, 180)).save(output, format="PNG") + return output.getvalue() + + +class InputAssetTests(unittest.TestCase): + def test_store_normalizes_and_consume_removes_temporary_object(self): + with ( + patch("services.input_assets.put_object") as put_object, + patch("services.input_assets.get_object") as get_object, + patch("services.input_assets.delete_object") as delete_object, + ): + stored = store_input_image(png_bytes()) + + key, normalized, mime_type = put_object.call_args.args + self.assertTrue(key.startswith("conversation-inputs/")) + self.assertEqual(mime_type, "image/jpeg") + self.assertTrue(normalized.startswith(b"\xff\xd8")) + self.assertEqual((stored.width, stored.height), (32, 24)) + self.assertEqual(stored.size_bytes, len(normalized)) + + get_object.return_value = normalized + consumed = consume_input_image(stored.token) + + self.assertEqual(consumed, normalized) + get_object.assert_called_once_with(key) + delete_object.assert_called_once_with(key) + + def test_tampered_token_is_rejected_before_storage_read(self): + with patch("services.input_assets.put_object"): + stored = store_input_image(png_bytes()) + tampered = f"{stored.token}x" + + with patch("services.input_assets.get_object") as get_object: + with self.assertRaisesRegex(ValueError, "签名无效"): + consume_input_image(tampered) + + get_object.assert_not_called() + + def test_non_image_is_rejected(self): + with self.assertRaisesRegex(ValueError, "可识别的图片"): + store_input_image(b"not an image") + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/tests/test_user_input.py b/backend/tests/test_user_input.py index 681b5e3..9df12dd 100644 --- a/backend/tests/test_user_input.py +++ b/backend/tests/test_user_input.py @@ -32,7 +32,7 @@ class UserInputParserTests(unittest.TestCase): self.assertIsNotNone(value) self.assertEqual(value.text, "帮我看看") self.assertTrue(value.has_camera_frame) - self.assertEqual(value.transcript_text, "帮我看看") + self.assertEqual(value.transcript_text, "") def test_rejects_legacy_and_unsupported_image_sources(self): self.assertIsNone(parse_user_input({"type": "user-text", "text": "旧协议"})) @@ -46,14 +46,40 @@ class UserInputParserTests(unittest.TestCase): { "type": "input_image", "source": { - "type": "uploaded_asset", - "asset_id": "asset_1", + "type": "remote_url", + "url": "https://example.com/image.jpg", }, } ], } ) + def test_parses_uploaded_image_asset(self): + value = parse_user_input( + { + "type": "user-input", + "schema_version": 1, + "input_id": "input_upload", + "parts": [ + {"type": "input_text", "text": "这是什么？"}, + { + "type": "input_image", + "source": { + "type": "uploaded_asset", + "asset_token": "signed-token", + }, + }, + ], + } + ) + + self.assertIsNotNone(value) + self.assertTrue(value.has_image) + self.assertFalse(value.has_camera_frame) + self.assertEqual(value.image_asset_token, "signed-token") + self.assertEqual(value.prompt_text, "这是什么？") + self.assertEqual(value.transcript_text, "") + def test_image_only_input_has_no_synthetic_chat_text(self): value = parse_user_input( { diff --git a/frontend/src/components/assistant-editor/debug-preview.tsx b/frontend/src/components/assistant-editor/debug-preview.tsx index 395aec5..945d2c6 100644 --- a/frontend/src/components/assistant-editor/debug-preview.tsx +++ b/frontend/src/components/assistant-editor/debug-preview.tsx @@ -7,6 +7,7 @@ import { Braces, Check, Copy, + ImageIcon, Loader2, MessageSquareText, Mic, @@ -51,16 +52,46 @@ import { useVoicePreview, type ChatMessage, type ClientToolDefinition, + type UserInputPart, type VoicePreview, type VoicePreviewStatus, } from "@/hooks/use-voice-preview"; -import type { DynamicVariableDefinition } from "@/lib/api"; +import { + inputAssetsApi, + type DynamicVariableDefinition, +} from "@/lib/api"; type VizStyle = "aura" | "nebula" | "bars" | "wave"; // 调试面板顶部主视图:聊天记录 / 视频流 type DebugView = "chat" | "video"; type DebugInputMode = "mic" | "text"; +type PendingDebugImage = { + file: File; + previewUrl: string; +}; + +const DEBUG_IMAGE_MAX_BYTES = 10 * 1024 * 1024; + +function fileToDataUrl(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(String(reader.result)); + reader.onerror = () => reject(new Error("无法读取图片")); + reader.readAsDataURL(file); + }); +} + +function hasDraggedImage(dataTransfer: DataTransfer): boolean { + return ( + Array.from(dataTransfer.items).some((item) => + item.type.startsWith("image/"), + ) || + Array.from(dataTransfer.files).some((file) => + file.type.startsWith("image/"), + ) + ); +} const VIZ_OPTIONS: { style: VizStyle; label: string; icon: React.ReactNode }[] = [ @@ -766,6 +797,8 @@ function DebugVoicePanel({ remoteStream, messages, sendText, + sendUserInput, + appendUserImage, connect, disconnect, audioRef, @@ -773,6 +806,11 @@ function DebugVoicePanel({ const recording = status === "connecting" || status === "connected"; const [textDraft, setTextDraft] = useState(""); const [inputMode, setInputMode] = useState("mic"); + const [pendingImage, setPendingImage] = + useState(null); + const [inputError, setInputError] = useState(""); + const [sendingInput, setSendingInput] = useState(false); + const [draggingImage, setDraggingImage] = useState(false); const [clientDialogContainer, setClientDialogContainer] = useState(null); const [messageDialogOpen, setMessageDialogOpen] = useState(false); @@ -790,9 +828,110 @@ function DebugVoicePanel({ : ""; const startDisabled = status === "connecting" || Boolean(startBlockedMessage); - function handleSendText() { - if (sendText(textDraft)) { + useEffect(() => { + return () => { + if (pendingImage) URL.revokeObjectURL(pendingImage.previewUrl); + }; + }, [pendingImage]); + + function stageImage(file: File) { + setInputError(""); + if (!vision) { + setInputError("请先为助手开启视觉理解，再添加图片。"); + return; + } + if (!file.type.startsWith("image/")) { + setInputError("只能添加图片文件。"); + return; + } + if (file.size > DEBUG_IMAGE_MAX_BYTES) { + setInputError("图片不能超过 10 MB。"); + return; + } + setPendingImage({ file, previewUrl: URL.createObjectURL(file) }); + setInputMode("text"); + } + + async function handleSendInput() { + const text = textDraft.trim(); + if (!pendingImage) { + if (sendText(text)) { + setTextDraft(""); + setInputError(""); + } + return; + } + if (status !== "connected" || sendingInput) return; + + setSendingInput(true); + setInputError(""); + let assetToken = ""; + try { + const imageUrl = await fileToDataUrl(pendingImage.file); + const asset = await inputAssetsApi.uploadImage(pendingImage.file); + assetToken = asset.assetToken; + const parts: UserInputPart[] = []; + if (text) parts.push({ type: "input_text", text }); + parts.push({ + type: "input_image", + source: { type: "uploaded_asset", asset_token: assetToken }, + }); + + const timestamp = new Date().toISOString(); + const result = await sendUserInput(parts); + appendUserImage(result.inputId, imageUrl, timestamp, text); setTextDraft(""); + setPendingImage(null); + } catch (sendError) { + if (assetToken) { + void inputAssetsApi.remove(assetToken).catch(() => {}); + } + setInputError( + sendError instanceof Error ? sendError.message : "图片发送失败，请重试。", + ); + } finally { + setSendingInput(false); + } + } + + function handlePaste(event: React.ClipboardEvent) { + const image = Array.from(event.clipboardData.items) + .find((item) => item.type.startsWith("image/")) + ?.getAsFile(); + if (!image) return; + event.preventDefault(); + stageImage(image); + } + + function handleDragOver(event: React.DragEvent) { + if (!hasDraggedImage(event.dataTransfer)) return; + event.preventDefault(); + event.dataTransfer.dropEffect = "copy"; + setDraggingImage(true); + } + + function handleDragLeave(event: React.DragEvent) { + const nextTarget = event.relatedTarget; + if (nextTarget instanceof Node && event.currentTarget.contains(nextTarget)) { + return; + } + setDraggingImage(false); + } + + function handleDrop(event: React.DragEvent) { + if (!hasDraggedImage(event.dataTransfer)) return; + event.preventDefault(); + setDraggingImage(false); + const image = Array.from(event.dataTransfer.files).find((file) => + file.type.startsWith("image/"), + ); + if (image) stageImage(image); + } + + function removePendingImage() { + if (!sendingInput) { + setPendingImage(null); + setInputError(""); } } @@ -968,61 +1107,136 @@ function DebugVoicePanel({
-
-
-
- setInputMode("mic")} - > - - - setInputMode("text")} - > - - +
+
+
+
+ setInputMode("mic")} + > + + + setInputMode("text")} + > + + +
+ {inputMode === "mic" ? ( +
+ +
+ ) : ( +
+ {pendingImage && ( +
+
+ {/* eslint-disable-next-line @next/next/no-img-element */} + 待发送图片预览 + + {sendingInput && ( + + + + )} +
+
+

+ {pendingImage.file.name || "粘贴的图片"} +

+

+ 按 Enter 上传并发送 +

+
+
+ )} +