Add support for image input in model resources and enhance related configurations

- Introduce a new `support_image_input` field in model resources, allowing models to indicate support for image input.
- Update the backend models, schemas, and database seed scripts to accommodate the new field.
- Enhance the AssistantConfig and related routes to handle image input capabilities, ensuring proper validation and error handling.
- Modify the frontend components to include toggles for enabling visual understanding and filtering models based on image input support.
- Implement necessary adjustments in the voice preview and pipeline to integrate video stream handling alongside audio functionalities.

.env

@@ -0,0 +1,4 @@
+# docker compose 变量(复制为项目根 .env)。公网 WebRTC 需配合 --profile remote。
+PUBLIC_IP=182.92.86.220
+TURN_SECRET=change-me-to-a-long-random-string
+TURN_URLS=turn:182.92.86.220:3478?transport=udp,turn:182.92.86.220:3478?transport=tcp

backend/db/models.py

@@ -50,6 +50,7 @@ class ModelResource(Base):
     )
     values: Mapped[dict] = mapped_column(JSONB, default=dict)
     secrets: Mapped[dict] = mapped_column(JSONB, default=dict)
+    support_image_input: Mapped[bool] = mapped_column(Boolean, default=False)
     enabled: Mapped[bool] = mapped_column(Boolean, default=True)
     is_default: Mapped[bool] = mapped_column(Boolean, default=False)
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())

backend/db/seed_model_resources.sql

@@ -1,43 +1,46 @@
 -- 模型资源种子数据。依赖应用启动时写入 interface_definitions。
 
 INSERT INTO model_resources
-  (id, name, capability, interface_type, values, secrets, enabled, is_default)
+  (id, name, capability, interface_type, values, secrets, support_image_input, enabled, is_default)
 VALUES
   ('model_001', 'DeepSeek-Chat', 'LLM', 'openai-llm',
    '{"modelId":"deepseek-chat","apiUrl":"https://api.deepseek.com/v1","temperature":0.7}',
-   '{"apiKey":"replace-me"}', TRUE, TRUE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, TRUE),
   ('model_002', 'SiliconFlow-TeleSpeechASR', 'ASR', 'openai-asr',
    '{"modelId":"TeleAI/TeleSpeechASR","apiUrl":"https://api.siliconflow.cn/v1","language":"zh"}',
-   '{"apiKey":"replace-me"}', TRUE, FALSE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE),
   ('model_003', 'SiliconFlow-Qwen3-Embedding-4B', 'Embedding', 'openai-embedding',
    '{"modelId":"Qwen/Qwen3-Embedding-4B","apiUrl":"https://api.siliconflow.cn/v1"}',
-   '{"apiKey":"replace-me"}', TRUE, TRUE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, TRUE),
   ('model_004', 'SiliconFlow-CosyVoice2-0.5B', 'TTS', 'openai-tts',
    '{"modelId":"FunAudioLLM/CosyVoice2-0.5B","apiUrl":"https://api.siliconflow.cn/v1","voice":"FunAudioLLM/CosyVoice2-0.5B:anna","speed":1.0,"sourceSampleRate":24000}',
-   '{"apiKey":"replace-me"}', TRUE, FALSE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE),
   ('model_005', '讯飞语音识别', 'ASR', 'xfyun-asr',
    '{"apiUrl":"https://iat-api.xfyun.cn/v2/iat","language":"zh_cn","domain":"iat","accent":"mandarin","dynamicCorrection":false,"frameSize":1280}',
-   '{"appId":"replace-me","apiKey":"replace-me","apiSecret":"replace-me"}', TRUE, TRUE),
+   '{"appId":"replace-me","apiKey":"replace-me","apiSecret":"replace-me"}', FALSE, TRUE, TRUE),
   ('model_006', 'Paraformer 识别', 'ASR', 'dashscope-asr',
    '{"modelId":"paraformer-realtime-v2","apiUrl":"https://dashscope.aliyuncs.com/api/v1/services/audio/asr","language":"zh"}',
-   '{"apiKey":"replace-me"}', TRUE, FALSE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE),
   ('model_007', '讯飞语音合成', 'TTS', 'xfyun-tts',
    '{"apiUrl":"https://tts-api.xfyun.cn/v2/tts","voice":"xiaoyan","speed":50,"volume":50,"pitch":50,"sourceSampleRate":16000}',
-   '{"appId":"replace-me","apiKey":"replace-me","apiSecret":"replace-me"}', TRUE, TRUE),
+   '{"appId":"replace-me","apiKey":"replace-me","apiSecret":"replace-me"}', FALSE, TRUE, TRUE),
   ('model_008', 'CosyVoice 合成', 'TTS', 'dashscope-tts',
    '{"modelId":"cosyvoice-v1","apiUrl":"https://dashscope.aliyuncs.com/api/v1/services/audio/tts","voice":"longxiaochun"}',
-   '{"apiKey":"replace-me"}', TRUE, FALSE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE),
   ('model_009', 'GPT Realtime', 'Realtime', 'openai-realtime',
    '{"modelId":"gpt-4o-realtime-preview","apiUrl":"https://api.openai.com/v1/realtime"}',
-   '{"apiKey":"replace-me"}', TRUE, TRUE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, TRUE),
   ('model_010', 'Gemini Live', 'Realtime', 'gemini-realtime',
    '{"modelId":"gemini-2.0-flash-live","apiUrl":"https://generativelanguage.googleapis.com/v1beta"}',
-   '{"apiKey":"replace-me"}', TRUE, FALSE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE),
   ('model_011', 'text-embedding-3', 'Embedding', 'openai-embedding',
    '{"modelId":"text-embedding-3-small","apiUrl":"https://api.openai.com/v1/embeddings"}',
-   '{"apiKey":"replace-me"}', TRUE, FALSE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE),
   ('model_012', 'StepAudio 2.5 Realtime', 'Realtime', 'stepfun-realtime',
    '{"modelId":"stepaudio-2.5-realtime","apiUrl":"wss://api.stepfun.com/v1/realtime","voice":"linjiajiejie","inputSampleRate":24000,"outputSampleRate":24000,"prefixPaddingMs":500,"silenceDurationMs":300,"energyAwakenessThreshold":2500}',
-   '{"apiKey":"replace-me"}', TRUE, FALSE)
+   '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE),
+  ('model_013', 'doubao-seed-2-0-mini-260428', 'LLM', 'openai-llm',
+   '{"modelId":"doubao-seed-2-0-mini-260428","apiUrl":"https://ark.cn-beijing.volces.com/api/v3","temperature":0.7,"extraBody":{"thinking":{"type":"disabled"}}}',
+   '{"apiKey":"replace-me"}', TRUE, TRUE, FALSE)
 -- Seed defaults must never overwrite resources configured through the UI.
 ON CONFLICT (id) DO NOTHING;

backend/models.py

@@ -41,6 +41,7 @@ class AssistantConfig(BaseModel):
     tts_interface_type: str = "openai-tts"
     llm_values: dict = {}
     llm_secrets: dict = {}
+    llm_support_image_input: bool = False
     stt_values: dict = {}
     stt_secrets: dict = {}
     tts_values: dict = {}
@@ -80,3 +81,4 @@ class SignalingOffer(BaseModel):
     type: str
     assistant_id: str | None = None
     inline_config: AssistantConfig | None = None
+    vision_enabled: bool = False

backend/routes/model_registry.py

@@ -53,6 +53,7 @@ def _resource_out(row: ModelResource) -> ModelResourceOut:
         interface_type=row.interface_type,
         values=row.values or {},
         secrets=mask_secrets(row.secrets or {}),
+        support_image_input=row.support_image_input,
         enabled=row.enabled,
         is_default=row.is_default,
         updated_at=row.updated_at.isoformat() if row.updated_at else None,
@@ -140,6 +141,9 @@ async def create_model_resource(
         interface_type=definition.interface_type,
         values=body.values,
         secrets=secrets,
+        support_image_input=body.support_image_input
+        if definition.capability == "LLM"
+        else False,
         enabled=body.enabled,
         is_default=body.is_default,
     )
@@ -200,6 +204,7 @@ async def duplicate_model_resource(
         interface_type=source.interface_type,
         values=dict(source.values or {}),
         secrets=dict(source.secrets or {}),
+        support_image_input=source.support_image_input,
         enabled=source.enabled,
         is_default=False,
     )
@@ -223,6 +228,9 @@ async def update_model_resource(
     row.interface_type = definition.interface_type
     row.values = body.values
     row.secrets = secrets
+    row.support_image_input = (
+        body.support_image_input if definition.capability == "LLM" else False
+    )
     row.enabled = body.enabled
     row.is_default = body.is_default
     if row.is_default:

backend/routes/voice_webrtc.py

@@ -87,6 +87,8 @@ async def _handle_offer(websocket, payload, peers):
         await pc.renegotiate(sdp=offer.sdp, type=offer.type, restart_pc=False)
     else:
         cfg = await _resolve_config(offer)  # 解析放在建连前,配置错就别建连
+        if offer.vision_enabled and not cfg.llm_support_image_input:
+            raise ValueError("当前大语言模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM")
         pc = SmallWebRTCConnection(ice_servers=aiortc_ice_servers())
         if pc_id:
             pc._pc_id = pc_id
@@ -98,8 +100,13 @@ async def _handle_offer(websocket, payload, peers):
             peers.pop(conn.pc_id, None)
 
         # 后台跑管线:WebRTC transport + 解析出的运行时配置
-        transport = build_webrtc_transport(pc)
+        transport = build_webrtc_transport(
-        asyncio.create_task(run_pipeline(transport, cfg))
+            pc,
+            video_in_enabled=offer.vision_enabled,
+        )
+        asyncio.create_task(
+            run_pipeline(transport, cfg, vision_enabled=offer.vision_enabled)
+        )
 
     answer = pc.get_answer()
     if websocket.application_state == WebSocketState.CONNECTED:

backend/schemas.py

@@ -111,6 +111,7 @@ class ModelResourceUpsert(CamelModel):
     interface_type: str
     values: dict[str, Any] = Field(default_factory=dict)
     secrets: dict[str, Any] = Field(default_factory=dict)
+    support_image_input: bool = False
     enabled: bool = True
     is_default: bool = False
 

backend/services/config_resolver.py

@@ -86,6 +86,9 @@ async def resolve_runtime_config(
         realtimeModel=str(_value(realtime_resource, "modelId", "")),
         llm_values=(llm_resource.values or {}) if llm_resource else {},
         llm_secrets=(llm_resource.secrets or {}) if llm_resource else {},
+        llm_support_image_input=(
+            bool(llm_resource.support_image_input) if llm_resource else False
+        ),
         stt_values=(stt_resource.values or {}) if stt_resource else {},
         stt_secrets=(stt_resource.secrets or {}) if stt_resource else {},
         tts_values=(tts_resource.values or {}) if tts_resource else {},

backend/services/interface_catalog.py

@@ -138,13 +138,6 @@ INTERFACE_DEFINITIONS: list[dict] = [
             field("textAggregationMode", "Text Aggregation Mode", default="token"),
         ],
     },
-    {
-        "interface_type": "dashscope-llm",
-        "name": "DashScope LLM",
-        "capability": "LLM",
-        "fields": OPENAI_COMMON
-        + [field("temperature", "Temperature", type_="number", default=0.7)],
-    },
     {
         "interface_type": "dashscope-asr",
         "name": "DashScope ASR",
@@ -165,7 +158,6 @@ INTERFACE_DEFINITIONS: list[dict] = [
     },
 ]
 
-
 def validate_fields(definition: dict, values: dict, secrets: dict) -> None:
     for item in definition["fields"]:
         source = secrets if item["group"] == "secrets" else values

backend/services/pipecat/pipeline.py

@@ -35,6 +35,7 @@ from pipecat.frames.frames import (
     OutputTransportMessageUrgentFrame,
     TextFrame,
     TTSSpeakFrame,
+    UserImageRequestFrame,
 )
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.worker import PipelineParams, PipelineWorker
@@ -45,6 +46,11 @@ from pipecat.processors.aggregators.llm_response_universal import (
     LLMUserAggregatorParams,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.runner.utils import (
+    get_transport_client_id,
+    maybe_capture_participant_camera,
+)
+from pipecat.services.llm_service import FunctionCallParams
 from pipecat.turns.user_start import (
     TranscriptionUserTurnStartStrategy,
     VADUserTurnStartStrategy,
@@ -54,6 +60,13 @@ from pipecat.utils.time import time_now_iso8601
 from pipecat.workers.runner import WorkerRunner
 
 
+VISION_TOOL_NAME = "fetch_user_image"
+VISION_SYSTEM_HINT = (
+    "当前会话打开了视觉理解。用户询问当前画面、摄像头里有什么、人物/物品/"
+    "环境状态或需要你看一眼时,调用 fetch_user_image 获取当前视频帧,再基于画面回答。"
+)
+
+
 def _text_input(message) -> tuple[str, bool] | None:
     """解析现有 user-text 与 RTVI send-text 两种前端文字消息。"""
     if not isinstance(message, dict):
@@ -204,7 +217,12 @@ class PassthroughLLMAssistantAggregator(LLMAssistantAggregator):
         self._stream_text = ""
 
 
-async def run_pipeline(transport, cfg: AssistantConfig) -> None:
+async def run_pipeline(
+    transport,
+    cfg: AssistantConfig,
+    *,
+    vision_enabled: bool = False,
+) -> None:
     """在给定 transport 上构建并运行管线,直到连接结束。
 
     Args:
@@ -212,7 +230,10 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
                    只要有 .input() / .output() / event_handler 即可。
         cfg:       助手配置(随请求内联传入)。
     """
-    logger.info(f"启动管线: assistant={cfg.name} type={cfg.type} mode={cfg.runtimeMode}")
+    logger.info(
+        f"启动管线: assistant={cfg.name} type={cfg.type} "
+        f"mode={cfg.runtimeMode} vision={vision_enabled}"
+    )
 
     # 大脑:按类型决定 LLM 槽/开场白/上下文归属。每通电话一个实例(可持会话状态)。
     brain = build_brain(cfg)
@@ -224,6 +245,8 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
         cfg.runtimeMode = "pipeline"
 
     if cfg.runtimeMode == "realtime":
+        if vision_enabled:
+            logger.warning("Realtime 模式暂未接入视频帧工具,本次仅启用语音通话")
         await run_realtime_pipeline(transport, cfg)
         return
 
@@ -263,7 +286,16 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
         greeting = await brain.greeting(cfg)
         system_content = ""
 
-    context = LLMContext(messages=[{"role": "system", "content": system_content}])
+    def with_vision_hint(text: str) -> str:
+        if not vision_enabled:
+            return text
+        if not text:
+            return VISION_SYSTEM_HINT
+        return f"{text}\n\n{VISION_SYSTEM_HINT}"
+
+    context = LLMContext(
+        messages=[{"role": "system", "content": with_vision_hint(system_content)}]
+    )
     # LLM 槽由大脑提供:内部类型=OpenAI 兼容服务;fastgpt=包 SDK 的伪 LLM。
     llm = brain.build_llm(cfg, context)
     user_aggregator = LLMUserAggregator(
@@ -282,6 +314,58 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
     )
     assistant_aggregator = PassthroughLLMAssistantAggregator(context)
     text_input = TextInputProcessor()
+    vision_state: dict[str, str | None] = {"client_id": None}
+    vision_schema = FunctionSchema(
+        name=VISION_TOOL_NAME,
+        description=(
+            "获取用户当前摄像头画面。当用户询问当前画面、看到了什么、"
+            "人/物品/环境状态或需要视觉判断时调用。"
+        ),
+        properties={
+            "question": {
+                "type": "string",
+                "description": "用户关于当前视频画面的具体问题。",
+            }
+        },
+        required=["question"],
+    )
+
+    async def fetch_user_image(params: FunctionCallParams):
+        question = str(params.arguments.get("question") or "请描述当前画面。")
+        user_id = vision_state.get("client_id")
+        if not user_id:
+            await params.result_callback(
+                {
+                    "status": "no_video_client",
+                    "message": "当前还没有可用的摄像头视频流。",
+                }
+            )
+            return
+
+        logger.debug(f"请求当前视频帧: user_id={user_id}, question={question}")
+        await params.llm.push_frame(
+            UserImageRequestFrame(
+                user_id=user_id,
+                text=question,
+                append_to_context=True,
+                function_name=params.function_name,
+                tool_call_id=params.tool_call_id,
+                result_callback=params.result_callback,
+            ),
+            FrameDirection.UPSTREAM,
+        )
+
+    if vision_enabled:
+        llm.register_function(VISION_TOOL_NAME, fetch_user_image)
+
+    def set_visible_tools(schemas: list[FunctionSchema] | None = None) -> None:
+        tools = list(schemas or [])
+        if vision_enabled:
+            tools.append(vision_schema)
+        if tools:
+            context.set_tools(ToolsSchema(standard_tools=tools))
+        else:
+            context.set_tools()
 
     # 结束节点:等结束语「说完」(BotStoppedSpeakingFrame)再挂断,确保结束语的
     # 文字(走 data channel)与音频都已下发,避免前端只听到声音、看不到文字。
@@ -366,16 +450,17 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
     def set_system_prompt(text: str) -> None:
         """替换上下文里的系统提示(节点切换时整体替换,而非追加)。"""
         messages = context.get_messages()
+        content = with_vision_hint(text)
         if messages and messages[0].get("role") == "system":
-            messages[0] = {"role": "system", "content": text}
+            messages[0] = {"role": "system", "content": content}
         else:
-            messages.insert(0, {"role": "system", "content": text})
+            messages.insert(0, {"role": "system", "content": content})
 
     def apply_node(node_id: str | None) -> None:
         """进入节点:设置系统提示 + 把出边注册为可调用的转移工具。"""
         set_system_prompt(engine.system_prompt_for(node_id))
         if engine.is_end(node_id):
-            context.set_tools()  # 终止节点无工具
+            set_visible_tools([])  # 终止节点不展示转移工具,但保留视觉工具
             return
         schemas = [
             FunctionSchema(
@@ -386,10 +471,7 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
             )
             for edge in engine.outgoing(node_id)
         ]
-        if schemas:
+        set_visible_tools(schemas)
-            context.set_tools(ToolsSchema(standard_tools=schemas))
-        else:
-            context.set_tools()  # 无出边:清空工具
 
     async def go_to_node(target: str) -> None:
         """执行转移:切当前节点、重置计数、点亮画布、设置提示/工具。
@@ -455,6 +537,8 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
                     engine.edge_fn_name(edge), make_transition_handler(edge)
                 )
         apply_node(wf_state["current"])  # 设初始节点的提示与工具
+    elif vision_enabled:
+        set_visible_tools([])
 
     async def append_user_text_to_context(text: str, *, run_llm: bool) -> None:
         await worker.queue_frame(
@@ -559,6 +643,16 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
 
     @transport.event_handler("on_client_connected")
     async def on_client_connected(_transport, _client):
+        if vision_enabled:
+            try:
+                vision_state["client_id"] = get_transport_client_id(
+                    _transport,
+                    _client,
+                )
+                await maybe_capture_participant_camera(_transport, _client)
+                logger.info(f"视觉理解已接入视频客户端: {vision_state['client_id']}")
+            except Exception as e:
+                logger.warning(f"视觉理解摄像头捕获初始化失败: {e}")
         if greeting:
             # 外部托管类型的上下文由对方服务端维护,开场白不写入本地 context
             if brain.spec.owns_context:

backend/services/pipecat/service_factory.py

@@ -72,10 +72,15 @@ def create_llm(cfg: AssistantConfig):
     """DeepSeek 等,走 OpenAI 兼容的 /v1/chat/completions。"""
     if cfg.llm_interface_type not in {"openai-llm", "dashscope-llm"}:
         raise ValueError(f"不支持的 LLM 接口类型: {cfg.llm_interface_type}")
+    extra_body = cfg.llm_values.get("extraBody")
+    extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
     return OpenAILLMService(
         api_key=cfg.llm_api_key or config.LLM_API_KEY,
         base_url=cfg.llm_base_url or config.LLM_BASE_URL,
-        settings=OpenAILLMService.Settings(model=cfg.model or config.LLM_MODEL),
+        settings=OpenAILLMService.Settings(
+            model=cfg.model or config.LLM_MODEL,
+            extra=extra,
+        ),
     )
 
 

backend/services/pipecat/transports.py

@@ -24,21 +24,26 @@ from pipecat.transports.websocket.fastapi import (
 from pipecat.serializers.protobuf import ProtobufFrameSerializer
 
 
-def _base_params() -> dict:
+def _base_params(*, video_in_enabled: bool = False) -> dict:
     """两种 transport 共享的音频参数。"""
     return dict(
         audio_in_enabled=True,
         audio_out_enabled=True,
+        video_in_enabled=video_in_enabled,
         # EndFrame 后默认补 2s 静音(防止收尾被截断)。我们的挂断已等到机器人
         # 说完才触发,这段静音纯属空等,置 0 让结束语播完立即挂断。
         audio_out_end_silence_secs=0,
     )
 
 
-def build_webrtc_transport(connection: SmallWebRTCConnection) -> SmallWebRTCTransport:
+def build_webrtc_transport(
+    connection: SmallWebRTCConnection,
+    *,
+    video_in_enabled: bool = False,
+) -> SmallWebRTCTransport:
     return SmallWebRTCTransport(
         webrtc_connection=connection,
-        params=TransportParams(**_base_params()),
+        params=TransportParams(**_base_params(video_in_enabled=video_in_enabled)),
     )
 
 

frontend/src/components/pages/AssistantPage.tsx

@@ -426,10 +426,35 @@ export function AssistantPage(props: AssistantPageProps) {
   // 按资源类型生成 {value:id, label:name} 选项
   const credOptions = (type: ModelResource["capability"]) =>
     modelResources
-      .filter((c) => c.capability === type)
+      .filter((c) => {
+        if (c.capability !== type) return false;
+        if (type === "LLM" && visionEnabled) return c.supportImageInput;
+        return true;
+      })
       .map((c) => ({ value: c.id, label: c.name }));
   const kbOptions = knowledgeBases.map((k) => ({ value: k.id, label: k.name }));
 
+  function modelSupportsImageInput(resourceId: string): boolean {
+    return Boolean(
+      modelResources.find((resource) => resource.id === resourceId)
+        ?.supportImageInput,
+    );
+  }
+
+  function handleVisionEnabledChange(enabled: boolean) {
+    setVisionEnabled(enabled);
+    if (enabled && form.model && !modelSupportsImageInput(form.model)) {
+      updateForm("model", "");
+    }
+    if (
+      enabled &&
+      openCodeForm.model &&
+      !modelSupportsImageInput(openCodeForm.model)
+    ) {
+      updateOpenCodeForm("model", "");
+    }
+  }
+
   function startCreate() {
     router.push("/assistants/new");
   }
@@ -1369,7 +1394,7 @@ export function AssistantPage(props: AssistantPageProps) {
             >
               <ToggleRow
                 title="允许用户打断"
-                description="用户说话时，助手可以停止当前播报并重新理解用户输入"
+                hint="用户说话时，助手可以停止当前播报并重新理解用户输入。"
                 checked={difyForm.enableInterrupt}
                 onChange={(checked) =>
                   updateDifyForm("enableInterrupt", checked)
@@ -1483,7 +1508,7 @@ export function AssistantPage(props: AssistantPageProps) {
             >
               <ToggleRow
                 title="允许用户打断"
-                description="用户说话时，助手可以停止当前播报并重新理解用户输入"
+                hint="用户说话时，助手可以停止当前播报并重新理解用户输入。"
                 checked={fastGptForm.enableInterrupt}
                 onChange={(checked) =>
                   updateFastGptForm("enableInterrupt", checked)
@@ -1572,6 +1597,12 @@ export function AssistantPage(props: AssistantPageProps) {
               title="模型与语音配置"
               description="配置 OpenCode 使用的大语言模型、语音识别与语音合成资源。"
             >
+              <ToggleRow
+                title="视觉理解"
+                hint="开启后，大语言模型只能选择支持图片输入的资源；开始对话时会允许助手按需理解当前视频画面。"
+                checked={visionEnabled}
+                onChange={handleVisionEnabledChange}
+              />
               <ResourceSelectField
                 label="大语言模型"
                 value={openCodeForm.model}
@@ -1602,7 +1633,7 @@ export function AssistantPage(props: AssistantPageProps) {
             >
               <ToggleRow
                 title="允许用户打断"
-                description="用户说话时，助手可以停止当前播报并重新理解用户输入"
+                hint="用户说话时，助手可以停止当前播报并重新理解用户输入。"
                 checked={openCodeForm.enableInterrupt}
                 onChange={(checked) =>
                   updateOpenCodeForm("enableInterrupt", checked)
@@ -1724,15 +1755,6 @@ export function AssistantPage(props: AssistantPageProps) {
                 </div>
               </div>
             </div>
-            <div className="mt-4 border-t border-hairline pt-4">
-              <ToggleRow
-                icon={<Eye size={16} />}
-                title="视觉理解"
-                hint="开启后，右侧调试面板可切换到视频流预览，并可同时选择麦克风与摄像头。"
-                checked={visionEnabled}
-                onChange={setVisionEnabled}
-              />
-            </div>
           </SectionCard>
 
           <SectionCard
@@ -1754,6 +1776,12 @@ export function AssistantPage(props: AssistantPageProps) {
               title="模型配置"
               description="从「模型资源」中选择大语言模型、语音识别与语音合成"
             >
+              <ToggleRow
+                title="视觉理解"
+                hint="开启后，大语言模型只能选择支持图片输入的资源；开始对话时会允许助手按需理解当前视频画面。"
+                checked={visionEnabled}
+                onChange={handleVisionEnabledChange}
+              />
               <ResourceSelectField
                 label="大语言模型"
                 value={form.model}
@@ -1824,7 +1852,7 @@ export function AssistantPage(props: AssistantPageProps) {
           >
             <ToggleRow
               title="允许用户打断"
-              description="用户说话时，助手可以停止当前播报并重新理解用户输入"
+              hint="用户说话时，助手可以停止当前播报并重新理解用户输入。"
               checked={form.enableInterrupt}
               onChange={(checked) => updateForm("enableInterrupt", checked)}
             />
@@ -1839,7 +1867,7 @@ export function AssistantPage(props: AssistantPageProps) {
 
 type VizStyle = "aura" | "nebula" | "bars" | "wave";
 
-// 调试面板顶部主视图:聊天记录 / 视频流(仅视觉理解开启时可选)
+// 调试面板顶部主视图:聊天记录 / 视频流
 type DebugView = "chat" | "video";
 type DebugInputMode = "mic" | "text";
 
@@ -1919,28 +1947,27 @@ function DebugDrawer({
   const [showTranscript, setShowTranscript] = useState(false);
   const [vizStyle, setVizStyle] = useState<VizStyle>("aura");
   const [view, setView] = useState<DebugView>("chat");
-  const effectiveView: DebugView = vision ? view : "chat";
   const stopCamera = camera.stop;
   const startCamera = camera.start;
-  const cameraShouldRun =
+  const recording =
-    vision &&
+    preview.status === "connecting" || preview.status === "connected";
-    (preview.status === "connecting" || preview.status === "connected");
+  const shouldKeepCamera = vision && recording;
 
   useEffect(() => {
-    if (!vision) {
+    if (shouldKeepCamera) {
-      // eslint-disable-next-line react-hooks/set-state-in-effect
-      setView("chat");
-      stopCamera();
-    }
-  }, [vision, stopCamera]);
-
-  useEffect(() => {
-    if (cameraShouldRun) {
       void startCamera();
     } else {
       stopCamera();
     }
-  }, [cameraShouldRun, camera.deviceId, startCamera, stopCamera]);
+  }, [shouldKeepCamera, startCamera, stopCamera]);
+
+  const selectCamera = useCallback(
+    async (deviceId: string) => {
+      const nextStream = await camera.selectCamera(deviceId);
+      await preview.replaceVideoStream(nextStream);
+    },
+    [camera, preview],
+  );
 
   const containerClass = asSheet
     ? "flex h-full min-w-0 flex-1 flex-col overflow-hidden"
@@ -1959,7 +1986,7 @@ function DebugDrawer({
           />
         </div>
         <div className="flex shrink-0 items-center gap-2">
-          {SHOW_VOICE_VIZ && effectiveView === "chat" && (
+          {SHOW_VOICE_VIZ && view === "chat" && (
             <>
               {!showTranscript && (
                 <SegmentedIconGroup label="可视化样式">
@@ -1995,33 +2022,29 @@ function DebugDrawer({
           )}
           <SegmentedIconGroup label="预览视图">
             <SegmentedIconButton
-              selected={effectiveView === "chat"}
+              selected={view === "chat"}
               label="聊天记录"
               onClick={() => setView("chat")}
             >
               <MessageSquareText size={14} />
             </SegmentedIconButton>
-            {vision && (
+            <SegmentedIconButton
-              <SegmentedIconButton
+              selected={view === "video"}
-                selected={effectiveView === "video"}
+              label="视频流"
-                label="视频流"
+              onClick={() => setView("video")}
-                onClick={() => setView("video")}
+            >
-              >
+              <Video size={14} />
-                <Video size={14} />
+            </SegmentedIconButton>
-              </SegmentedIconButton>
-            )}
           </SegmentedIconGroup>
         </div>
       </div>
-      {vision && (
+      <div className="shrink-0 border-b border-hairline px-5 py-2.5">
-        <div className="shrink-0 border-b border-hairline px-5 py-2.5">
+        <div className="flex h-10 min-w-0 items-center rounded-[1.4rem] border border-hairline-strong bg-background px-2">
-          <div className="flex h-10 min-w-0 items-center rounded-[1.4rem] border border-hairline-strong bg-background px-2">
+          <CameraDeviceField camera={camera} onSelect={selectCamera} />
-            <CameraDeviceField camera={camera} />
-          </div>
         </div>
-      )}
+      </div>
       <DebugVoicePanel
-        view={effectiveView}
+        view={view}
         showTranscript={showTranscript}
         vizStyle={vizStyle}
         assistantId={assistantId}
@@ -2100,7 +2123,13 @@ function MicrophoneDeviceField({ preview }: { preview: VoicePreview }) {
   );
 }
 
-function CameraDeviceField({ camera }: { camera: CameraPreview }) {
+function CameraDeviceField({
+  camera,
+  onSelect,
+}: {
+  camera: CameraPreview;
+  onSelect?: (deviceId: string) => void | Promise<void>;
+}) {
   return (
     <DeviceSelectField
       icon={<Video size={15} className="shrink-0 text-muted-soft" />}
@@ -2109,7 +2138,7 @@ function CameraDeviceField({ camera }: { camera: CameraPreview }) {
       fallbackLabel="摄像头"
       value={camera.deviceId}
       devices={camera.devices}
-      onSelect={(deviceId) => void camera.selectCamera(deviceId)}
+      onSelect={(deviceId) => void (onSelect ?? camera.selectCamera)(deviceId)}
     />
   );
 }
@@ -2179,9 +2208,8 @@ function DebugVoicePanel({
   const inChatView = view === "chat" && (!SHOW_VOICE_VIZ || showTranscript);
   const idleOrFailed = status === "idle" || status === "failed";
   const showIdleHub =
-    messages.length === 0 &&
     idleOrFailed &&
-    (inChatView || (vision && view === "video"));
+    (view === "video" || (messages.length === 0 && inChatView));
 
   function handleSendText() {
     if (sendText(textDraft)) {
@@ -2189,26 +2217,36 @@ function DebugVoicePanel({
     }
   }
 
+  const startConversation = useCallback(async () => {
+    const videoStream = vision ? await camera.start() : null;
+    await connect({
+      visionEnabled: vision,
+      videoStream,
+    });
+  }, [camera, connect, vision]);
+
   return (
     <div className="flex min-h-0 flex-1 flex-col">
       {/* 后端 TTS 音频经 WebRTC 媒体流过来,挂这里播放 */}
       <audio ref={audioRef} autoPlay playsInline className="hidden" />
-      {view === "video" && showIdleHub ? (
+      {view === "video" ? (
-        <DebugIdleHub
+        showIdleHub ? (
-          status={status}
+          <DebugIdleHub
-          error={error}
+            status={status}
-          assistantId={assistantId}
+            error={error}
-          connect={connect}
+            assistantId={assistantId}
-        />
+            connect={startConversation}
-      ) : view === "video" ? (
+          />
-        <DebugVideoPanel camera={camera} />
+        ) : (
+          <DebugVideoPanel camera={camera} />
+        )
       ) : !SHOW_VOICE_VIZ || showTranscript ? (
         showIdleHub ? (
           <DebugIdleHub
             status={status}
             error={error}
             assistantId={assistantId}
-            connect={connect}
+            connect={startConversation}
           />
         ) : (
           <DebugTranscriptPanel messages={messages} recording={recording} />
@@ -2283,7 +2321,7 @@ function DebugVoicePanel({
               if (recording) {
                 disconnect();
               } else {
-                void connect();
+                void startConversation();
               }
             }}
             className={[
@@ -2372,7 +2410,7 @@ function DebugVoicePanel({
                 if (recording) {
                   disconnect();
                 } else {
-                  void connect();
+                  void startConversation();
                 }
               }}
               className={[

frontend/src/components/pages/ComponentsModelsPage.tsx

@@ -5,6 +5,7 @@ import {
   ChevronDown,
   ChevronUp,
   Copy,
+  Eye,
   Loader2,
   MoreHorizontal,
   Pencil,
@@ -37,6 +38,8 @@ import { FilterPills } from "@/components/ui/filter-pills";
 import { SearchInput } from "@/components/ui/search-input";
 import { ListToolbar } from "@/components/ui/list-toolbar";
 import { Input } from "@/components/ui/input";
+import { Switch } from "@/components/ui/switch";
+import { Textarea } from "@/components/ui/textarea";
 import {
   Select,
   SelectContent,
@@ -69,6 +72,7 @@ type ResourceDraft = {
   interfaceType: string;
   values: Record<string, unknown>;
   secrets: Record<string, unknown>;
+  supportImageInput: boolean;
 };
 
 const emptyDraft: ResourceDraft = {
@@ -77,6 +81,7 @@ const emptyDraft: ResourceDraft = {
   interfaceType: "",
   values: {},
   secrets: {},
+  supportImageInput: false,
 };
 
 function defaults(definition: InterfaceDefinition): Record<string, unknown> {
@@ -115,6 +120,44 @@ function fieldValue(draft: ResourceDraft, field: InterfaceField): unknown {
     : draft.values[field.key];
 }
 
+function extraBodyText(values: Record<string, unknown>): string {
+  const value = values.extraBody;
+  if (!value || typeof value !== "object" || Array.isArray(value)) return "";
+  return JSON.stringify(value, null, 2);
+}
+
+function parseExtraBody(text: string):
+  | { ok: true; value: Record<string, unknown> | null }
+  | { ok: false; error: string } {
+  const trimmed = text.trim();
+  if (!trimmed) return { ok: true, value: null };
+  try {
+    const parsed = JSON.parse(trimmed) as unknown;
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      return { ok: false, error: "Extra Body 必须是 JSON 对象" };
+    }
+    return { ok: true, value: parsed as Record<string, unknown> };
+  } catch {
+    return { ok: false, error: "Extra Body 不是合法 JSON" };
+  }
+}
+
+function valuesWithExtraBody(
+  values: Record<string, unknown>,
+  extraBody: Record<string, unknown> | null,
+): Record<string, unknown> {
+  const next = { ...values };
+  delete next.extraBody;
+  if (extraBody) next.extraBody = extraBody;
+  return next;
+}
+
+function isSelectableDefinition(definition: InterfaceDefinition): boolean {
+  return (
+    definition.capability !== "LLM" || definition.interfaceType === "openai-llm"
+  );
+}
+
 export function ComponentsModelsPage() {
   const [resources, setResources] = useState<ModelResource[]>([]);
   const [definitions, setDefinitions] = useState<InterfaceDefinition[]>([]);
@@ -128,6 +171,7 @@ export function ComponentsModelsPage() {
   const [dialogOpen, setDialogOpen] = useState(false);
   const [editingId, setEditingId] = useState<string | null>(null);
   const [draft, setDraft] = useState<ResourceDraft>(emptyDraft);
+  const [extraBodyDraft, setExtraBodyDraft] = useState("");
   const [saving, setSaving] = useState(false);
   const [saveError, setSaveError] = useState("");
   const [testing, setTesting] = useState(false);
@@ -161,7 +205,8 @@ export function ComponentsModelsPage() {
     void load();
   }, [load]);
 
-  const availableDefinitions = definitions.filter(
+  const selectableDefinitions = definitions.filter(isSelectableDefinition);
+  const availableDefinitions = selectableDefinitions.filter(
     (definition) => definition.capability === draft.capability,
   );
   const selectedDefinition = definitions.find(
@@ -239,11 +284,12 @@ export function ComponentsModelsPage() {
       values: definition ? defaults(definition) : {},
       secrets: {},
     }));
+    setExtraBodyDraft("");
     setTestResult(null);
   }
 
   function selectCapability(capability: ModelType) {
-    const first = definitions.find(
+    const first = selectableDefinitions.find(
       (definition) => definition.capability === capability,
     );
     setDraft((previous) => ({
@@ -252,12 +298,15 @@ export function ComponentsModelsPage() {
       interfaceType: first?.interfaceType ?? "",
       values: first ? defaults(first) : {},
       secrets: {},
+      supportImageInput:
+        capability === "LLM" ? previous.supportImageInput : false,
     }));
+    setExtraBodyDraft("");
     setTestResult(null);
   }
 
   function openCreate() {
-    const first = definitions.find(
+    const first = selectableDefinitions.find(
       (definition) => definition.capability === "LLM",
     );
     setEditingId(null);
@@ -265,7 +314,9 @@ export function ComponentsModelsPage() {
       ...emptyDraft,
       interfaceType: first?.interfaceType ?? "",
       values: first ? defaults(first) : {},
+      supportImageInput: false,
     });
+    setExtraBodyDraft("");
     setSaveError("");
     setTestResult(null);
     setDialogOpen(true);
@@ -279,7 +330,9 @@ export function ComponentsModelsPage() {
       interfaceType: resource.interfaceType,
       values: resource.values,
       secrets: resource.secrets,
+      supportImageInput: resource.supportImageInput,
     });
+    setExtraBodyDraft(extraBodyText(resource.values));
     setSaveError("");
     setTestResult(null);
     setDialogOpen(true);
@@ -294,9 +347,12 @@ export function ComponentsModelsPage() {
     setTestResult(null);
   }
 
+  const parsedExtraBody = parseExtraBody(extraBodyDraft);
+  const extraBodyError = parsedExtraBody.ok ? "" : parsedExtraBody.error;
   const canSave = Boolean(
     draft.name.trim() &&
       selectedDefinition &&
+      !extraBodyError &&
       selectedDefinition.fieldSchema.fields.every((field) => {
         if (!field.required) return true;
         const value = fieldValue(draft, field);
@@ -305,11 +361,17 @@ export function ComponentsModelsPage() {
   );
 
   function payload() {
+    const extraBody = parsedExtraBody.ok ? parsedExtraBody.value : null;
     return {
       name: draft.name.trim(),
       interfaceType: draft.interfaceType,
-      values: draft.values,
+      values:
+        draft.capability === "LLM"
+          ? valuesWithExtraBody(draft.values, extraBody)
+          : draft.values,
       secrets: draft.secrets,
+      supportImageInput:
+        draft.capability === "LLM" ? draft.supportImageInput : false,
       enabled: editingId
         ? resources.find((resource) => resource.id === editingId)?.enabled ?? true
         : true,
@@ -457,8 +519,19 @@ export function ComponentsModelsPage() {
               width: "md:w-[360px]",
               cell: (resource) => (
                 <>
-                  <div className="truncate font-medium text-foreground">
+                  <div className="flex min-w-0 items-center gap-2">
-                    {resource.name}
+                    <span className="truncate font-medium text-foreground">
+                      {resource.name}
+                    </span>
+                    {resource.supportImageInput && (
+                      <span
+                        className="flex h-5 w-5 shrink-0 items-center justify-center rounded-full bg-surface-strong text-muted-foreground"
+                        title="支持图片输入"
+                        aria-label="支持图片输入"
+                      >
+                        <Eye size={12} />
+                      </span>
+                    )}
                   </div>
                   <div className="mt-1 truncate text-xs text-muted-soft">
                     {String(
@@ -652,6 +725,20 @@ export function ComponentsModelsPage() {
                     </Select>
                   </Field>
                 </div>
+                {draft.capability === "LLM" && (
+                  <div className="flex items-center justify-between rounded-xl border border-hairline p-4">
+                    <span className="text-sm font-medium">支持图片输入</span>
+                    <Switch
+                      checked={draft.supportImageInput}
+                      onCheckedChange={(checked) =>
+                        setDraft((previous) => ({
+                          ...previous,
+                          supportImageInput: checked,
+                        }))
+                      }
+                    />
+                  </div>
+                )}
               </FieldSection>
 
               <FieldSection title="连接与鉴权信息" scrollable fill>
@@ -667,6 +754,25 @@ export function ComponentsModelsPage() {
             </div>
 
             <FieldSection title="可选项" scrollable tall>
+              {draft.capability === "LLM" && (
+                <Field label="Extra Body">
+                  <Textarea
+                    rows={7}
+                    value={extraBodyDraft}
+                    onChange={(event) => {
+                      setExtraBodyDraft(event.target.value);
+                      setTestResult(null);
+                    }}
+                    placeholder={'例如：{\n  "thinking": { "type": "disabled" }\n}'}
+                    className="font-mono text-xs leading-5"
+                  />
+                  {extraBodyError && (
+                    <div className="text-xs text-destructive">
+                      {extraBodyError}
+                    </div>
+                  )}
+                </Field>
+              )}
               {optionalFields.length > 0 ? (
                 optionalFields.map((field) => (
                   <DynamicField
@@ -676,7 +782,7 @@ export function ComponentsModelsPage() {
                     onChange={(value) => updateField(field, value)}
                   />
                 ))
-              ) : (
+              ) : draft.capability === "LLM" ? null : (
                 <div className="flex h-full items-center justify-center text-sm text-muted-foreground">
                   当前接口没有可选项
                 </div>

frontend/src/hooks/use-camera-preview.ts

@@ -3,9 +3,8 @@
 /**
  * 摄像头预览:纯前端 getUserMedia 视频流,供调试抽屉的「视频流」视图使用。
  *
- * 与 use-voice-preview 分离:麦克风/语音会话走 WebRTC 到后端,摄像头目前只在
+ * 与 use-voice-preview 分离:这里负责摄像头设备枚举与本地预览;开始视觉会话时,
- * 本地预览。后续接入视觉理解管线时,把这条 video track 加进现有 PeerConnection
+ * 调用方会把这条 video track 加进语音 WebRTC PeerConnection。
- * (pc.addTrack)并在后端 transport 打开 video_in_enabled 即可。
  */
 
 import { useCallback, useEffect, useRef, useState } from "react";
@@ -50,12 +49,12 @@ export function useCameraPreview() {
     setStream(null);
   }, []);
 
-  const start = useCallback(async () => {
+  const start = useCallback(async (): Promise<MediaStream | null> => {
-    if (streamRef.current) return;
+    if (streamRef.current) return streamRef.current;
     setError(null);
     if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
       setError("当前页面无法访问摄像头，请通过 https 或 localhost 访问。");
-      return;
+      return null;
     }
     setStarting(true);
     try {
@@ -66,21 +65,24 @@ export function useCameraPreview() {
       streamRef.current = media;
       setStream(media);
       void refreshCameras();
+      return media;
     } catch (mediaError) {
       setError(cameraErrorMessage(mediaError));
+      return null;
     } finally {
       setStarting(false);
     }
   }, [refreshCameras]);
 
   const selectCamera = useCallback(
-    async (id: string) => {
+    async (id: string): Promise<MediaStream | null> => {
       deviceIdRef.current = id;
       setDeviceId(id);
       if (streamRef.current) {
         stop();
-        await start();
+        return await start();
       }
+      return null;
     },
     [start, stop],
   );

frontend/src/hooks/use-voice-preview.ts

@@ -23,6 +23,11 @@ import { API_BASE, webrtcApi } from "@/lib/api";
 
 export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed";
 
+type ConnectOptions = {
+  visionEnabled?: boolean;
+  videoStream?: MediaStream | null;
+};
+
 export type ChatMessage = {
   id: string;
   role: "user" | "assistant";
@@ -218,7 +223,7 @@ export function useVoicePreview(
     [disconnect, fail],
   );
 
-  const connect = useCallback(async () => {
+  const connect = useCallback(async (options: ConnectOptions = {}) => {
     if (startingRef.current || pcRef.current || wsRef.current) return;
     if (!assistantId) {
       setError("请先保存助手，再开始语音预览。");
@@ -456,6 +461,11 @@ export function useVoicePreview(
       } else {
         pc.addTransceiver("audio", { direction: "recvonly" });
       }
+      if (options.videoStream) {
+        options.videoStream
+          .getVideoTracks()
+          .forEach((track) => pc.addTrack(track, options.videoStream!));
+      }
 
       // 4) 生成 offer 并发给后端(assistant_id 在 payload 顶层)
       const offer = await pc.createOffer();
@@ -472,6 +482,7 @@ export function useVoicePreview(
             sdp: localDescription.sdp,
             type: localDescription.type,
             assistant_id: assistantId,
+            vision_enabled: Boolean(options.visionEnabled),
           },
         }),
       );
@@ -482,6 +493,17 @@ export function useVoicePreview(
     }
   }, [assistantId, fail, closeOnRemoteEnd, refreshDevices]);
 
+  const replaceVideoStream = useCallback(
+    async (videoStream: MediaStream | null) => {
+      const pc = pcRef.current;
+      if (!pc) return;
+      const sender = pc.getSenders().find((s) => s.track?.kind === "video");
+      if (!sender) return;
+      await sender.replaceTrack(videoStream?.getVideoTracks()[0] ?? null);
+    },
+    [],
+  );
+
   // 选择麦克风:更新选择;若会话正在发送麦克风音频,则用 WebRTC replaceTrack
   // 热切换轨道(无需重新协商),并把波形可视化重新接到新流。
   // 未连接时仅记下选择,留待下次 connect 生效。
@@ -558,6 +580,7 @@ export function useVoicePreview(
     selectDevice,
     sendText,
     connect,
+    replaceVideoStream,
     disconnect,
     audioRef,
   };

frontend/src/lib/api.ts

@@ -56,6 +56,7 @@ export type ModelResource = {
   interfaceType: string;
   values: Record<string, unknown>;
   secrets: Record<string, unknown>;
+  supportImageInput: boolean;
   enabled: boolean;
   isDefault: boolean;
   updatedAt?: string | null;