feat: support uploaded images in debug voice preview

Allow paste/drag temporary image assets so vision turns work without a live camera frame.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/services/pipecat/pipeline.py

@@ -27,6 +27,7 @@ from services.pipecat.service_factory import (
 )
 from db.session import SessionLocal
 from services.knowledge import search as search_knowledge
+from services.input_assets import consume_input_image
 from services.client_tools import ClientToolBroker
 from services.tool_policy import policy_for_tool
 from services.vision import (
@@ -36,6 +37,7 @@ from services.vision import (
     config_with_main_llm_as_vision,
     config_with_vision_resource,
     image_data_uri,
+    image_frame_from_jpeg,
     image_jpeg_bytes,
 )
 from services.workflow_engine import WorkflowEngine
@@ -734,7 +736,7 @@ async def run_pipeline(
     greeting = await brain.greeting(cfg)
 
     async def submit_user_input(value: UserInput) -> None:
-        if not value.has_camera_frame:
+        if not value.has_image:
             if not value.run_immediately:
                 brain.record_user_message(value.text)
             await worker.queue_frame(
@@ -749,28 +751,49 @@ async def run_pipeline(
             raise ValueError("P0 图片输入必须立即触发回复")
         if not vision_enabled:
             raise ValueError("当前助手未启用视觉能力")
-        user_id = vision_state.get("client_id")
-        if not user_id:
-            raise ValueError("当前没有可用的摄像头视频流")
         native_vision = active_vision_uses_main_llm()
         analysis_cfg = None if native_vision else active_vision_config()
 
-        request = UserImageRequestFrame(
-            user_id=user_id,
-            text=value.prompt_text,
-            append_to_context=False,
-        )
-        try:
-            image_frame = await vision_capture.request_image(llm, request)
-        except asyncio.TimeoutError as exc:
-            raise ValueError("等待摄像头视频帧超时") from exc
+        if value.image_asset_token:
+            try:
+                image_bytes = await asyncio.to_thread(
+                    consume_input_image,
+                    value.image_asset_token,
+                )
+                image_frame = await asyncio.to_thread(
+                    image_frame_from_jpeg,
+                    image_bytes,
+                )
+            except ValueError:
+                raise
+            except Exception as exc:
+                raise ValueError("上传图片暂时无法读取") from exc
+        else:
+            user_id = vision_state.get("client_id")
+            if not user_id:
+                raise ValueError("当前没有可用的摄像头视频流")
+            request = UserImageRequestFrame(
+                user_id=user_id,
+                text=value.prompt_text,
+                append_to_context=False,
+            )
+            try:
+                image_frame = await vision_capture.request_image(llm, request)
+            except asyncio.TimeoutError as exc:
+                raise ValueError("等待摄像头视频帧超时") from exc
+            image_bytes = await asyncio.to_thread(image_jpeg_bytes, image_frame)
 
         if recorder:
-            image_bytes = await asyncio.to_thread(image_jpeg_bytes, image_frame)
             recorder.record_image_later(
                 image_bytes,
                 input_id=value.input_id,
                 timestamp=time_now_iso8601(),
+                content=value.text,
+                source=(
+                    "uploaded_asset"
+                    if value.image_asset_token
+                    else "camera_capture"
+                ),
             )
 
         if native_vision:

backend/services/pipecat/pipeline_events.py

@@ -149,11 +149,12 @@ def bind_cascade_pipeline_events(
 
     @text_input.event_handler("on_user_input")
     async def on_user_input(_processor, user_input: UserInput):
-        await queue_transcript(
-            "user",
-            user_input.transcript_text,
-            time_now_iso8601(),
-        )
+        if not user_input.has_image:
+            await queue_transcript(
+                "user",
+                user_input.transcript_text,
+                time_now_iso8601(),
+            )
         if user_input.run_immediately and user_input.interrupt:
             pending_user_inputs.append(user_input)
             return

backend/services/pipecat/processors.py

@@ -50,9 +50,14 @@ class UserInput:
     input_id: str
     text: str
     has_camera_frame: bool
+    image_asset_token: str | None
     run_immediately: bool
     interrupt: bool
 
+    @property
+    def has_image(self) -> bool:
+        return self.has_camera_frame or bool(self.image_asset_token)
+
     @property
     def prompt_text(self) -> str:
         if self.text:
@@ -61,10 +66,9 @@ class UserInput:
 
     @property
     def transcript_text(self) -> str:
-        # Images are represented as structured media records. A synthetic
-        # sentence would be shown as chat text and persisted as if the user had
-        # said it, while adding no visual information for the model.
-        return self.text
+        # Image turns are persisted as one structured media message, including
+        # any optional text caption. Do not create a duplicate transcript row.
+        return "" if self.has_image else self.text
 
 
 class UserInputError(ValueError):
@@ -155,6 +159,7 @@ def parse_user_input(message) -> UserInput | None:
 
     text = ""
     has_camera_frame = False
+    image_asset_token: str | None = None
     for part in parts:
         if not isinstance(part, dict):
             raise UserInputError("user-input part 格式不正确", input_id=input_id)
@@ -166,23 +171,30 @@ def parse_user_input(message) -> UserInput | None:
             if not text:
                 raise UserInputError("input_text 不能为空", input_id=input_id)
         elif part_type == "input_image":
-            if has_camera_frame:
+            if has_camera_frame or image_asset_token:
                 raise UserInputError("P0 只支持一张图片", input_id=input_id)
             source = part.get("source")
-            if (
-                not isinstance(source, dict)
-                or source.get("type") != "camera_frame"
-                or source.get("frame") != "current"
-            ):
+            if not isinstance(source, dict):
+                raise UserInputError("input_image source 格式不正确", input_id=input_id)
+            if source.get("type") == "camera_frame" and source.get("frame") == "current":
+                has_camera_frame = True
+            elif source.get("type") == "uploaded_asset":
+                token = str(source.get("asset_token") or "").strip()
+                if not token or len(token) > 4096:
+                    raise UserInputError(
+                        "上传图片缺少有效的 asset_token",
+                        input_id=input_id,
+                    )
+                image_asset_token = token
+            else:
                 raise UserInputError(
-                    "P0 只支持当前摄像头画面",
+                    "仅支持当前摄像头画面或已上传图片",
                     input_id=input_id,
                 )
-            has_camera_frame = True
         else:
             raise UserInputError(f"不支持的输入类型: {part_type}", input_id=input_id)
 
-    if not text and not has_camera_frame:
+    if not text and not (has_camera_frame or image_asset_token):
         raise UserInputError("user-input 没有有效内容", input_id=input_id)
 
     options = message.get("options")
@@ -193,6 +205,7 @@ def parse_user_input(message) -> UserInput | None:
         input_id=input_id,
         text=text,
         has_camera_frame=has_camera_frame,
+        image_asset_token=image_asset_token,
         run_immediately=run_immediately,
         interrupt=interrupt,
     )
@@ -565,7 +578,7 @@ class RealtimeUserInputProcessor(FrameProcessor):
                 "当前工作流节点暂不接收用户输入",
             )
             return
-        if user_input.has_camera_frame:
+        if user_input.has_image:
             await self._emit_error(
                 user_input.input_id,
                 "Realtime 模式暂不支持图片输入",