Add support for image input in model resources and enhance related configurations

- Introduce a new `support_image_input` field in model resources, allowing models to indicate support for image input. - Update the backend models, schemas, and database seed scripts to accommodate the new field. - Enhance the AssistantConfig and related routes to handle image input capabilities, ensuring proper validation and error handling. - Modify the frontend components to include toggles for enabling visual understanding and filtering models based on image input support. - Implement necessary adjustments in the voice preview and pipeline to integrate video stream handling alongside audio functionalities.
2026-07-07 19:41:45 +08:00
parent 32a52d318a
commit c5db918830
18 changed files with 411 additions and 116 deletions
--- a/.DS_Store
+++ b/.DS_Store
--- a/.env
+++ b/.env
@@ -0,0 +1,4 @@
+# docker compose 变量(复制为项目根 .env)。公网 WebRTC 需配合 --profile remote。
+PUBLIC_IP=182.92.86.220
+TURN_SECRET=change-me-to-a-long-random-string
+TURN_URLS=turn:182.92.86.220:3478?transport=udp,turn:182.92.86.220:3478?transport=tcp
--- a/backend/db/models.py
+++ b/backend/db/models.py
@@ -50,6 +50,7 @@ class ModelResource(Base):
    )
    values: Mapped[dict] = mapped_column(JSONB, default=dict)
    secrets: Mapped[dict] = mapped_column(JSONB, default=dict)
+    support_image_input: Mapped[bool] = mapped_column(Boolean, default=False)
    enabled: Mapped[bool] = mapped_column(Boolean, default=True)
    is_default: Mapped[bool] = mapped_column(Boolean, default=False)
    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
--- a/backend/db/seed_model_resources.sql
+++ b/backend/db/seed_model_resources.sql
@@ -1,43 +1,46 @@
 -- 模型资源种子数据。依赖应用启动时写入 interface_definitions。

 INSERT INTO model_resources
-  (id, name, capability, interface_type, values, secrets, enabled, is_default)
+  (id, name, capability, interface_type, values, secrets, support_image_input, enabled, is_default)
 VALUES
  ('model_001', 'DeepSeek-Chat', 'LLM', 'openai-llm',
   '{"modelId":"deepseek-chat","apiUrl":"https://api.deepseek.com/v1","temperature":0.7}',
-   '{"apiKey":"replace-me"}', TRUE, TRUE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, TRUE),
  ('model_002', 'SiliconFlow-TeleSpeechASR', 'ASR', 'openai-asr',
   '{"modelId":"TeleAI/TeleSpeechASR","apiUrl":"https://api.siliconflow.cn/v1","language":"zh"}',
-   '{"apiKey":"replace-me"}', TRUE, FALSE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE),
  ('model_003', 'SiliconFlow-Qwen3-Embedding-4B', 'Embedding', 'openai-embedding',
   '{"modelId":"Qwen/Qwen3-Embedding-4B","apiUrl":"https://api.siliconflow.cn/v1"}',
-   '{"apiKey":"replace-me"}', TRUE, TRUE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, TRUE),
  ('model_004', 'SiliconFlow-CosyVoice2-0.5B', 'TTS', 'openai-tts',
   '{"modelId":"FunAudioLLM/CosyVoice2-0.5B","apiUrl":"https://api.siliconflow.cn/v1","voice":"FunAudioLLM/CosyVoice2-0.5B:anna","speed":1.0,"sourceSampleRate":24000}',
-   '{"apiKey":"replace-me"}', TRUE, FALSE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE),
  ('model_005', '讯飞语音识别', 'ASR', 'xfyun-asr',
   '{"apiUrl":"https://iat-api.xfyun.cn/v2/iat","language":"zh_cn","domain":"iat","accent":"mandarin","dynamicCorrection":false,"frameSize":1280}',
-   '{"appId":"replace-me","apiKey":"replace-me","apiSecret":"replace-me"}', TRUE, TRUE),
+   '{"appId":"replace-me","apiKey":"replace-me","apiSecret":"replace-me"}', FALSE, TRUE, TRUE),
  ('model_006', 'Paraformer 识别', 'ASR', 'dashscope-asr',
   '{"modelId":"paraformer-realtime-v2","apiUrl":"https://dashscope.aliyuncs.com/api/v1/services/audio/asr","language":"zh"}',
-   '{"apiKey":"replace-me"}', TRUE, FALSE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE),
  ('model_007', '讯飞语音合成', 'TTS', 'xfyun-tts',
   '{"apiUrl":"https://tts-api.xfyun.cn/v2/tts","voice":"xiaoyan","speed":50,"volume":50,"pitch":50,"sourceSampleRate":16000}',
-   '{"appId":"replace-me","apiKey":"replace-me","apiSecret":"replace-me"}', TRUE, TRUE),
+   '{"appId":"replace-me","apiKey":"replace-me","apiSecret":"replace-me"}', FALSE, TRUE, TRUE),
  ('model_008', 'CosyVoice 合成', 'TTS', 'dashscope-tts',
   '{"modelId":"cosyvoice-v1","apiUrl":"https://dashscope.aliyuncs.com/api/v1/services/audio/tts","voice":"longxiaochun"}',
-   '{"apiKey":"replace-me"}', TRUE, FALSE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE),
  ('model_009', 'GPT Realtime', 'Realtime', 'openai-realtime',
   '{"modelId":"gpt-4o-realtime-preview","apiUrl":"https://api.openai.com/v1/realtime"}',
-   '{"apiKey":"replace-me"}', TRUE, TRUE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, TRUE),
  ('model_010', 'Gemini Live', 'Realtime', 'gemini-realtime',
   '{"modelId":"gemini-2.0-flash-live","apiUrl":"https://generativelanguage.googleapis.com/v1beta"}',
-   '{"apiKey":"replace-me"}', TRUE, FALSE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE),
  ('model_011', 'text-embedding-3', 'Embedding', 'openai-embedding',
   '{"modelId":"text-embedding-3-small","apiUrl":"https://api.openai.com/v1/embeddings"}',
-   '{"apiKey":"replace-me"}', TRUE, FALSE),
+   '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE),
  ('model_012', 'StepAudio 2.5 Realtime', 'Realtime', 'stepfun-realtime',
   '{"modelId":"stepaudio-2.5-realtime","apiUrl":"wss://api.stepfun.com/v1/realtime","voice":"linjiajiejie","inputSampleRate":24000,"outputSampleRate":24000,"prefixPaddingMs":500,"silenceDurationMs":300,"energyAwakenessThreshold":2500}',
-   '{"apiKey":"replace-me"}', TRUE, FALSE)
+   '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE),
+  ('model_013', 'doubao-seed-2-0-mini-260428', 'LLM', 'openai-llm',
+   '{"modelId":"doubao-seed-2-0-mini-260428","apiUrl":"https://ark.cn-beijing.volces.com/api/v3","temperature":0.7,"extraBody":{"thinking":{"type":"disabled"}}}',
+   '{"apiKey":"replace-me"}', TRUE, TRUE, FALSE)
 -- Seed defaults must never overwrite resources configured through the UI.
 ON CONFLICT (id) DO NOTHING;
--- a/backend/models.py
+++ b/backend/models.py
@@ -41,6 +41,7 @@ class AssistantConfig(BaseModel):
    tts_interface_type: str = "openai-tts"
    llm_values: dict = {}
    llm_secrets: dict = {}
+    llm_support_image_input: bool = False
    stt_values: dict = {}
    stt_secrets: dict = {}
    tts_values: dict = {}
@@ -80,3 +81,4 @@ class SignalingOffer(BaseModel):
    type: str
    assistant_id: str | None = None
    inline_config: AssistantConfig | None = None
+    vision_enabled: bool = False
--- a/backend/routes/model_registry.py
+++ b/backend/routes/model_registry.py
@@ -53,6 +53,7 @@ def _resource_out(row: ModelResource) -> ModelResourceOut:
        interface_type=row.interface_type,
        values=row.values or {},
        secrets=mask_secrets(row.secrets or {}),
+        support_image_input=row.support_image_input,
        enabled=row.enabled,
        is_default=row.is_default,
        updated_at=row.updated_at.isoformat() if row.updated_at else None,
@@ -140,6 +141,9 @@ async def create_model_resource(
        interface_type=definition.interface_type,
        values=body.values,
        secrets=secrets,
+        support_image_input=body.support_image_input
+        if definition.capability == "LLM"
+        else False,
        enabled=body.enabled,
        is_default=body.is_default,
    )
@@ -200,6 +204,7 @@ async def duplicate_model_resource(
        interface_type=source.interface_type,
        values=dict(source.values or {}),
        secrets=dict(source.secrets or {}),
+        support_image_input=source.support_image_input,
        enabled=source.enabled,
        is_default=False,
    )
@@ -223,6 +228,9 @@ async def update_model_resource(
    row.interface_type = definition.interface_type
    row.values = body.values
    row.secrets = secrets
+    row.support_image_input = (
+        body.support_image_input if definition.capability == "LLM" else False
+    )
    row.enabled = body.enabled
    row.is_default = body.is_default
    if row.is_default:
--- a/backend/routes/voice_webrtc.py
+++ b/backend/routes/voice_webrtc.py
@@ -87,6 +87,8 @@ async def _handle_offer(websocket, payload, peers):
        await pc.renegotiate(sdp=offer.sdp, type=offer.type, restart_pc=False)
    else:
        cfg = await _resolve_config(offer)  # 解析放在建连前,配置错就别建连
+        if offer.vision_enabled and not cfg.llm_support_image_input:
+            raise ValueError("当前大语言模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM")
        pc = SmallWebRTCConnection(ice_servers=aiortc_ice_servers())
        if pc_id:
            pc._pc_id = pc_id
@@ -98,8 +100,13 @@ async def _handle_offer(websocket, payload, peers):
            peers.pop(conn.pc_id, None)

        # 后台跑管线:WebRTC transport + 解析出的运行时配置
-        transport = build_webrtc_transport(pc)
-        asyncio.create_task(run_pipeline(transport, cfg))
+        transport = build_webrtc_transport(
+            pc,
+            video_in_enabled=offer.vision_enabled,
+        )
+        asyncio.create_task(
+            run_pipeline(transport, cfg, vision_enabled=offer.vision_enabled)
+        )

    answer = pc.get_answer()
    if websocket.application_state == WebSocketState.CONNECTED:
--- a/backend/schemas.py
+++ b/backend/schemas.py
@@ -111,6 +111,7 @@ class ModelResourceUpsert(CamelModel):
    interface_type: str
    values: dict[str, Any] = Field(default_factory=dict)
    secrets: dict[str, Any] = Field(default_factory=dict)
+    support_image_input: bool = False
    enabled: bool = True
    is_default: bool = False

--- a/backend/services/config_resolver.py
+++ b/backend/services/config_resolver.py
@@ -86,6 +86,9 @@ async def resolve_runtime_config(
        realtimeModel=str(_value(realtime_resource, "modelId", "")),
        llm_values=(llm_resource.values or {}) if llm_resource else {},
        llm_secrets=(llm_resource.secrets or {}) if llm_resource else {},
+        llm_support_image_input=(
+            bool(llm_resource.support_image_input) if llm_resource else False
+        ),
        stt_values=(stt_resource.values or {}) if stt_resource else {},
        stt_secrets=(stt_resource.secrets or {}) if stt_resource else {},
        tts_values=(tts_resource.values or {}) if tts_resource else {},
--- a/backend/services/interface_catalog.py
+++ b/backend/services/interface_catalog.py
@@ -138,13 +138,6 @@ INTERFACE_DEFINITIONS: list[dict] = [
            field("textAggregationMode", "Text Aggregation Mode", default="token"),
        ],
    },
-    {
-        "interface_type": "dashscope-llm",
-        "name": "DashScope LLM",
-        "capability": "LLM",
-        "fields": OPENAI_COMMON
-        + [field("temperature", "Temperature", type_="number", default=0.7)],
-    },
    {
        "interface_type": "dashscope-asr",
        "name": "DashScope ASR",
@@ -165,7 +158,6 @@ INTERFACE_DEFINITIONS: list[dict] = [
    },
 ]

-
 def validate_fields(definition: dict, values: dict, secrets: dict) -> None:
    for item in definition["fields"]:
        source = secrets if item["group"] == "secrets" else values
--- a/backend/services/pipecat/pipeline.py
+++ b/backend/services/pipecat/pipeline.py
@@ -35,6 +35,7 @@ from pipecat.frames.frames import (
    OutputTransportMessageUrgentFrame,
    TextFrame,
    TTSSpeakFrame,
+    UserImageRequestFrame,
 )
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.worker import PipelineParams, PipelineWorker
@@ -45,6 +46,11 @@ from pipecat.processors.aggregators.llm_response_universal import (
    LLMUserAggregatorParams,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.runner.utils import (
+    get_transport_client_id,
+    maybe_capture_participant_camera,
+)
+from pipecat.services.llm_service import FunctionCallParams
 from pipecat.turns.user_start import (
    TranscriptionUserTurnStartStrategy,
    VADUserTurnStartStrategy,
@@ -54,6 +60,13 @@ from pipecat.utils.time import time_now_iso8601
 from pipecat.workers.runner import WorkerRunner


+VISION_TOOL_NAME = "fetch_user_image"
+VISION_SYSTEM_HINT = (
+    "当前会话打开了视觉理解。用户询问当前画面、摄像头里有什么、人物/物品/"
+    "环境状态或需要你看一眼时,调用 fetch_user_image 获取当前视频帧,再基于画面回答。"
+)
+
+
 def _text_input(message) -> tuple[str, bool] | None:
    """解析现有 user-text 与 RTVI send-text 两种前端文字消息。"""
    if not isinstance(message, dict):
@@ -204,7 +217,12 @@ class PassthroughLLMAssistantAggregator(LLMAssistantAggregator):
        self._stream_text = ""


-async def run_pipeline(transport, cfg: AssistantConfig) -> None:
+async def run_pipeline(
+    transport,
+    cfg: AssistantConfig,
+    *,
+    vision_enabled: bool = False,
+) -> None:
    """在给定 transport 上构建并运行管线,直到连接结束。

    Args:
@@ -212,7 +230,10 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
                   只要有 .input() / .output() / event_handler 即可。
        cfg:       助手配置(随请求内联传入)。
    """
-    logger.info(f"启动管线: assistant={cfg.name} type={cfg.type} mode={cfg.runtimeMode}")
+    logger.info(
+        f"启动管线: assistant={cfg.name} type={cfg.type} "
+        f"mode={cfg.runtimeMode} vision={vision_enabled}"
+    )

    # 大脑:按类型决定 LLM 槽/开场白/上下文归属。每通电话一个实例(可持会话状态)。
    brain = build_brain(cfg)
@@ -224,6 +245,8 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
        cfg.runtimeMode = "pipeline"

    if cfg.runtimeMode == "realtime":
+        if vision_enabled:
+            logger.warning("Realtime 模式暂未接入视频帧工具,本次仅启用语音通话")
        await run_realtime_pipeline(transport, cfg)
        return

@@ -263,7 +286,16 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
        greeting = await brain.greeting(cfg)
        system_content = ""

-    context = LLMContext(messages=[{"role": "system", "content": system_content}])
+    def with_vision_hint(text: str) -> str:
+        if not vision_enabled:
+            return text
+        if not text:
+            return VISION_SYSTEM_HINT
+        return f"{text}\n\n{VISION_SYSTEM_HINT}"
+
+    context = LLMContext(
+        messages=[{"role": "system", "content": with_vision_hint(system_content)}]
+    )
    # LLM 槽由大脑提供:内部类型=OpenAI 兼容服务;fastgpt=包 SDK 的伪 LLM。
    llm = brain.build_llm(cfg, context)
    user_aggregator = LLMUserAggregator(
@@ -282,6 +314,58 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
    )
    assistant_aggregator = PassthroughLLMAssistantAggregator(context)
    text_input = TextInputProcessor()
+    vision_state: dict[str, str | None] = {"client_id": None}
+    vision_schema = FunctionSchema(
+        name=VISION_TOOL_NAME,
+        description=(
+            "获取用户当前摄像头画面。当用户询问当前画面、看到了什么、"
+            "人/物品/环境状态或需要视觉判断时调用。"
+        ),
+        properties={
+            "question": {
+                "type": "string",
+                "description": "用户关于当前视频画面的具体问题。",
+            }
+        },
+        required=["question"],
+    )
+
+    async def fetch_user_image(params: FunctionCallParams):
+        question = str(params.arguments.get("question") or "请描述当前画面。")
+        user_id = vision_state.get("client_id")
+        if not user_id:
+            await params.result_callback(
+                {
+                    "status": "no_video_client",
+                    "message": "当前还没有可用的摄像头视频流。",
+                }
+            )
+            return
+
+        logger.debug(f"请求当前视频帧: user_id={user_id}, question={question}")
+        await params.llm.push_frame(
+            UserImageRequestFrame(
+                user_id=user_id,
+                text=question,
+                append_to_context=True,
+                function_name=params.function_name,
+                tool_call_id=params.tool_call_id,
+                result_callback=params.result_callback,
+            ),
+            FrameDirection.UPSTREAM,
+        )
+
+    if vision_enabled:
+        llm.register_function(VISION_TOOL_NAME, fetch_user_image)
+
+    def set_visible_tools(schemas: list[FunctionSchema] | None = None) -> None:
+        tools = list(schemas or [])
+        if vision_enabled:
+            tools.append(vision_schema)
+        if tools:
+            context.set_tools(ToolsSchema(standard_tools=tools))
+        else:
+            context.set_tools()

    # 结束节点:等结束语「说完」(BotStoppedSpeakingFrame)再挂断,确保结束语的
    # 文字(走 data channel)与音频都已下发,避免前端只听到声音、看不到文字。
@@ -366,16 +450,17 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
    def set_system_prompt(text: str) -> None:
        """替换上下文里的系统提示(节点切换时整体替换,而非追加)。"""
        messages = context.get_messages()
+        content = with_vision_hint(text)
        if messages and messages[0].get("role") == "system":
-            messages[0] = {"role": "system", "content": text}
+            messages[0] = {"role": "system", "content": content}
        else:
-            messages.insert(0, {"role": "system", "content": text})
+            messages.insert(0, {"role": "system", "content": content})

    def apply_node(node_id: str | None) -> None:
        """进入节点:设置系统提示 + 把出边注册为可调用的转移工具。"""
        set_system_prompt(engine.system_prompt_for(node_id))
        if engine.is_end(node_id):
-            context.set_tools()  # 终止节点无工具
+            set_visible_tools([])  # 终止节点不展示转移工具,但保留视觉工具
            return
        schemas = [
            FunctionSchema(
@@ -386,10 +471,7 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
            )
            for edge in engine.outgoing(node_id)
        ]
-        if schemas:
-            context.set_tools(ToolsSchema(standard_tools=schemas))
-        else:
-            context.set_tools()  # 无出边:清空工具
+        set_visible_tools(schemas)

    async def go_to_node(target: str) -> None:
        """执行转移:切当前节点、重置计数、点亮画布、设置提示/工具。
@@ -455,6 +537,8 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
                    engine.edge_fn_name(edge), make_transition_handler(edge)
                )
        apply_node(wf_state["current"])  # 设初始节点的提示与工具
+    elif vision_enabled:
+        set_visible_tools([])

    async def append_user_text_to_context(text: str, *, run_llm: bool) -> None:
        await worker.queue_frame(
@@ -559,6 +643,16 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:

    @transport.event_handler("on_client_connected")
    async def on_client_connected(_transport, _client):
+        if vision_enabled:
+            try:
+                vision_state["client_id"] = get_transport_client_id(
+                    _transport,
+                    _client,
+                )
+                await maybe_capture_participant_camera(_transport, _client)
+                logger.info(f"视觉理解已接入视频客户端: {vision_state['client_id']}")
+            except Exception as e:
+                logger.warning(f"视觉理解摄像头捕获初始化失败: {e}")
        if greeting:
            # 外部托管类型的上下文由对方服务端维护,开场白不写入本地 context
            if brain.spec.owns_context:
--- a/backend/services/pipecat/service_factory.py
+++ b/backend/services/pipecat/service_factory.py
@@ -72,10 +72,15 @@ def create_llm(cfg: AssistantConfig):
    """DeepSeek 等,走 OpenAI 兼容的 /v1/chat/completions。"""
    if cfg.llm_interface_type not in {"openai-llm", "dashscope-llm"}:
        raise ValueError(f"不支持的 LLM 接口类型: {cfg.llm_interface_type}")
+    extra_body = cfg.llm_values.get("extraBody")
+    extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
    return OpenAILLMService(
        api_key=cfg.llm_api_key or config.LLM_API_KEY,
        base_url=cfg.llm_base_url or config.LLM_BASE_URL,
-        settings=OpenAILLMService.Settings(model=cfg.model or config.LLM_MODEL),
+        settings=OpenAILLMService.Settings(
+            model=cfg.model or config.LLM_MODEL,
+            extra=extra,
+        ),
    )


--- a/backend/services/pipecat/transports.py
+++ b/backend/services/pipecat/transports.py
@@ -24,21 +24,26 @@ from pipecat.transports.websocket.fastapi import (
 from pipecat.serializers.protobuf import ProtobufFrameSerializer


-def _base_params() -> dict:
+def _base_params(*, video_in_enabled: bool = False) -> dict:
    """两种 transport 共享的音频参数。"""
    return dict(
        audio_in_enabled=True,
        audio_out_enabled=True,
+        video_in_enabled=video_in_enabled,
        # EndFrame 后默认补 2s 静音(防止收尾被截断)。我们的挂断已等到机器人
        # 说完才触发,这段静音纯属空等,置 0 让结束语播完立即挂断。
        audio_out_end_silence_secs=0,
    )


-def build_webrtc_transport(connection: SmallWebRTCConnection) -> SmallWebRTCTransport:
+def build_webrtc_transport(
+    connection: SmallWebRTCConnection,
+    *,
+    video_in_enabled: bool = False,
+) -> SmallWebRTCTransport:
    return SmallWebRTCTransport(
        webrtc_connection=connection,
-        params=TransportParams(**_base_params()),
+        params=TransportParams(**_base_params(video_in_enabled=video_in_enabled)),
    )


--- a/frontend/src/components/pages/AssistantPage.tsx
+++ b/frontend/src/components/pages/AssistantPage.tsx
@@ -426,10 +426,35 @@ export function AssistantPage(props: AssistantPageProps) {
  // 按资源类型生成 {value:id, label:name} 选项
  const credOptions = (type: ModelResource["capability"]) =>
    modelResources
-      .filter((c) => c.capability === type)
+      .filter((c) => {
+        if (c.capability !== type) return false;
+        if (type === "LLM" && visionEnabled) return c.supportImageInput;
+        return true;
+      })
      .map((c) => ({ value: c.id, label: c.name }));
  const kbOptions = knowledgeBases.map((k) => ({ value: k.id, label: k.name }));

+  function modelSupportsImageInput(resourceId: string): boolean {
+    return Boolean(
+      modelResources.find((resource) => resource.id === resourceId)
+        ?.supportImageInput,
+    );
+  }
+
+  function handleVisionEnabledChange(enabled: boolean) {
+    setVisionEnabled(enabled);
+    if (enabled && form.model && !modelSupportsImageInput(form.model)) {
+      updateForm("model", "");
+    }
+    if (
+      enabled &&
+      openCodeForm.model &&
+      !modelSupportsImageInput(openCodeForm.model)
+    ) {
+      updateOpenCodeForm("model", "");
+    }
+  }
+
  function startCreate() {
    router.push("/assistants/new");
  }
@@ -1369,7 +1394,7 @@ export function AssistantPage(props: AssistantPageProps) {
            >
              <ToggleRow
                title="允许用户打断"
-                description="用户说话时，助手可以停止当前播报并重新理解用户输入"
+                hint="用户说话时，助手可以停止当前播报并重新理解用户输入。"
                checked={difyForm.enableInterrupt}
                onChange={(checked) =>
                  updateDifyForm("enableInterrupt", checked)
@@ -1483,7 +1508,7 @@ export function AssistantPage(props: AssistantPageProps) {
            >
              <ToggleRow
                title="允许用户打断"
-                description="用户说话时，助手可以停止当前播报并重新理解用户输入"
+                hint="用户说话时，助手可以停止当前播报并重新理解用户输入。"
                checked={fastGptForm.enableInterrupt}
                onChange={(checked) =>
                  updateFastGptForm("enableInterrupt", checked)
@@ -1572,6 +1597,12 @@ export function AssistantPage(props: AssistantPageProps) {
              title="模型与语音配置"
              description="配置 OpenCode 使用的大语言模型、语音识别与语音合成资源。"
            >
+              <ToggleRow
+                title="视觉理解"
+                hint="开启后，大语言模型只能选择支持图片输入的资源；开始对话时会允许助手按需理解当前视频画面。"
+                checked={visionEnabled}
+                onChange={handleVisionEnabledChange}
+              />
              <ResourceSelectField
                label="大语言模型"
                value={openCodeForm.model}
@@ -1602,7 +1633,7 @@ export function AssistantPage(props: AssistantPageProps) {
            >
              <ToggleRow
                title="允许用户打断"
-                description="用户说话时，助手可以停止当前播报并重新理解用户输入"
+                hint="用户说话时，助手可以停止当前播报并重新理解用户输入。"
                checked={openCodeForm.enableInterrupt}
                onChange={(checked) =>
                  updateOpenCodeForm("enableInterrupt", checked)
@@ -1724,15 +1755,6 @@ export function AssistantPage(props: AssistantPageProps) {
                </div>
              </div>
            </div>
-            <div className="mt-4 border-t border-hairline pt-4">
-              <ToggleRow
-                icon={<Eye size={16} />}
-                title="视觉理解"
-                hint="开启后，右侧调试面板可切换到视频流预览，并可同时选择麦克风与摄像头。"
-                checked={visionEnabled}
-                onChange={setVisionEnabled}
-              />
-            </div>
          </SectionCard>

          <SectionCard
@@ -1754,6 +1776,12 @@ export function AssistantPage(props: AssistantPageProps) {
              title="模型配置"
              description="从「模型资源」中选择大语言模型、语音识别与语音合成"
            >
+              <ToggleRow
+                title="视觉理解"
+                hint="开启后，大语言模型只能选择支持图片输入的资源；开始对话时会允许助手按需理解当前视频画面。"
+                checked={visionEnabled}
+                onChange={handleVisionEnabledChange}
+              />
              <ResourceSelectField
                label="大语言模型"
                value={form.model}
@@ -1824,7 +1852,7 @@ export function AssistantPage(props: AssistantPageProps) {
          >
            <ToggleRow
              title="允许用户打断"
-              description="用户说话时，助手可以停止当前播报并重新理解用户输入"
+              hint="用户说话时，助手可以停止当前播报并重新理解用户输入。"
              checked={form.enableInterrupt}
              onChange={(checked) => updateForm("enableInterrupt", checked)}
            />
@@ -1839,7 +1867,7 @@ export function AssistantPage(props: AssistantPageProps) {

 type VizStyle = "aura" | "nebula" | "bars" | "wave";

-// 调试面板顶部主视图:聊天记录 / 视频流(仅视觉理解开启时可选)
+// 调试面板顶部主视图:聊天记录 / 视频流
 type DebugView = "chat" | "video";
 type DebugInputMode = "mic" | "text";

@@ -1919,28 +1947,27 @@ function DebugDrawer({
  const [showTranscript, setShowTranscript] = useState(false);
  const [vizStyle, setVizStyle] = useState<VizStyle>("aura");
  const [view, setView] = useState<DebugView>("chat");
-  const effectiveView: DebugView = vision ? view : "chat";
  const stopCamera = camera.stop;
  const startCamera = camera.start;
-  const cameraShouldRun =
-    vision &&
-    (preview.status === "connecting" || preview.status === "connected");
+  const recording =
+    preview.status === "connecting" || preview.status === "connected";
+  const shouldKeepCamera = vision && recording;

  useEffect(() => {
-    if (!vision) {
-      // eslint-disable-next-line react-hooks/set-state-in-effect
-      setView("chat");
-      stopCamera();
-    }
-  }, [vision, stopCamera]);
-
-  useEffect(() => {
-    if (cameraShouldRun) {
+    if (shouldKeepCamera) {
      void startCamera();
    } else {
      stopCamera();
    }
-  }, [cameraShouldRun, camera.deviceId, startCamera, stopCamera]);
+  }, [shouldKeepCamera, startCamera, stopCamera]);
+
+  const selectCamera = useCallback(
+    async (deviceId: string) => {
+      const nextStream = await camera.selectCamera(deviceId);
+      await preview.replaceVideoStream(nextStream);
+    },
+    [camera, preview],
+  );

  const containerClass = asSheet
    ? "flex h-full min-w-0 flex-1 flex-col overflow-hidden"
@@ -1959,7 +1986,7 @@ function DebugDrawer({
          />
        </div>
        <div className="flex shrink-0 items-center gap-2">
-          {SHOW_VOICE_VIZ && effectiveView === "chat" && (
+          {SHOW_VOICE_VIZ && view === "chat" && (
            <>
              {!showTranscript && (
                <SegmentedIconGroup label="可视化样式">
@@ -1995,33 +2022,29 @@ function DebugDrawer({
          )}
          <SegmentedIconGroup label="预览视图">
            <SegmentedIconButton
-              selected={effectiveView === "chat"}
+              selected={view === "chat"}
              label="聊天记录"
              onClick={() => setView("chat")}
            >
              <MessageSquareText size={14} />
            </SegmentedIconButton>
-            {vision && (
-              <SegmentedIconButton
-                selected={effectiveView === "video"}
-                label="视频流"
-                onClick={() => setView("video")}
-              >
-                <Video size={14} />
-              </SegmentedIconButton>
-            )}
+            <SegmentedIconButton
+              selected={view === "video"}
+              label="视频流"
+              onClick={() => setView("video")}
+            >
+              <Video size={14} />
+            </SegmentedIconButton>
          </SegmentedIconGroup>
        </div>
      </div>
-      {vision && (
-        <div className="shrink-0 border-b border-hairline px-5 py-2.5">
-          <div className="flex h-10 min-w-0 items-center rounded-[1.4rem] border border-hairline-strong bg-background px-2">
-            <CameraDeviceField camera={camera} />
-          </div>
+      <div className="shrink-0 border-b border-hairline px-5 py-2.5">
+        <div className="flex h-10 min-w-0 items-center rounded-[1.4rem] border border-hairline-strong bg-background px-2">
+          <CameraDeviceField camera={camera} onSelect={selectCamera} />
        </div>
-      )}
+      </div>
      <DebugVoicePanel
-        view={effectiveView}
+        view={view}
        showTranscript={showTranscript}
        vizStyle={vizStyle}
        assistantId={assistantId}
@@ -2100,7 +2123,13 @@ function MicrophoneDeviceField({ preview }: { preview: VoicePreview }) {
  );
 }

-function CameraDeviceField({ camera }: { camera: CameraPreview }) {
+function CameraDeviceField({
+  camera,
+  onSelect,
+}: {
+  camera: CameraPreview;
+  onSelect?: (deviceId: string) => void | Promise<void>;
+}) {
  return (
    <DeviceSelectField
      icon={<Video size={15} className="shrink-0 text-muted-soft" />}
@@ -2109,7 +2138,7 @@ function CameraDeviceField({ camera }: { camera: CameraPreview }) {
      fallbackLabel="摄像头"
      value={camera.deviceId}
      devices={camera.devices}
-      onSelect={(deviceId) => void camera.selectCamera(deviceId)}
+      onSelect={(deviceId) => void (onSelect ?? camera.selectCamera)(deviceId)}
    />
  );
 }
@@ -2179,9 +2208,8 @@ function DebugVoicePanel({
  const inChatView = view === "chat" && (!SHOW_VOICE_VIZ || showTranscript);
  const idleOrFailed = status === "idle" || status === "failed";
  const showIdleHub =
-    messages.length === 0 &&
    idleOrFailed &&
-    (inChatView || (vision && view === "video"));
+    (view === "video" || (messages.length === 0 && inChatView));

  function handleSendText() {
    if (sendText(textDraft)) {
@@ -2189,26 +2217,36 @@ function DebugVoicePanel({
    }
  }

+  const startConversation = useCallback(async () => {
+    const videoStream = vision ? await camera.start() : null;
+    await connect({
+      visionEnabled: vision,
+      videoStream,
+    });
+  }, [camera, connect, vision]);
+
  return (
    <div className="flex min-h-0 flex-1 flex-col">
      {/* 后端 TTS 音频经 WebRTC 媒体流过来,挂这里播放 */}
      <audio ref={audioRef} autoPlay playsInline className="hidden" />
-      {view === "video" && showIdleHub ? (
-        <DebugIdleHub
-          status={status}
-          error={error}
-          assistantId={assistantId}
-          connect={connect}
-        />
-      ) : view === "video" ? (
-        <DebugVideoPanel camera={camera} />
+      {view === "video" ? (
+        showIdleHub ? (
+          <DebugIdleHub
+            status={status}
+            error={error}
+            assistantId={assistantId}
+            connect={startConversation}
+          />
+        ) : (
+          <DebugVideoPanel camera={camera} />
+        )
      ) : !SHOW_VOICE_VIZ || showTranscript ? (
        showIdleHub ? (
          <DebugIdleHub
            status={status}
            error={error}
            assistantId={assistantId}
-            connect={connect}
+            connect={startConversation}
          />
        ) : (
          <DebugTranscriptPanel messages={messages} recording={recording} />
@@ -2283,7 +2321,7 @@ function DebugVoicePanel({
              if (recording) {
                disconnect();
              } else {
-                void connect();
+                void startConversation();
              }
            }}
            className={[
@@ -2372,7 +2410,7 @@ function DebugVoicePanel({
                if (recording) {
                  disconnect();
                } else {
-                  void connect();
+                  void startConversation();
                }
              }}
              className={[
--- a/frontend/src/components/pages/ComponentsModelsPage.tsx
+++ b/frontend/src/components/pages/ComponentsModelsPage.tsx
@@ -5,6 +5,7 @@ import {
  ChevronDown,
  ChevronUp,
  Copy,
+  Eye,
  Loader2,
  MoreHorizontal,
  Pencil,
@@ -37,6 +38,8 @@ import { FilterPills } from "@/components/ui/filter-pills";
 import { SearchInput } from "@/components/ui/search-input";
 import { ListToolbar } from "@/components/ui/list-toolbar";
 import { Input } from "@/components/ui/input";
+import { Switch } from "@/components/ui/switch";
+import { Textarea } from "@/components/ui/textarea";
 import {
  Select,
  SelectContent,
@@ -69,6 +72,7 @@ type ResourceDraft = {
  interfaceType: string;
  values: Record<string, unknown>;
  secrets: Record<string, unknown>;
+  supportImageInput: boolean;
 };

 const emptyDraft: ResourceDraft = {
@@ -77,6 +81,7 @@ const emptyDraft: ResourceDraft = {
  interfaceType: "",
  values: {},
  secrets: {},
+  supportImageInput: false,
 };

 function defaults(definition: InterfaceDefinition): Record<string, unknown> {
@@ -115,6 +120,44 @@ function fieldValue(draft: ResourceDraft, field: InterfaceField): unknown {
    : draft.values[field.key];
 }

+function extraBodyText(values: Record<string, unknown>): string {
+  const value = values.extraBody;
+  if (!value || typeof value !== "object" || Array.isArray(value)) return "";
+  return JSON.stringify(value, null, 2);
+}
+
+function parseExtraBody(text: string):
+  | { ok: true; value: Record<string, unknown> | null }
+  | { ok: false; error: string } {
+  const trimmed = text.trim();
+  if (!trimmed) return { ok: true, value: null };
+  try {
+    const parsed = JSON.parse(trimmed) as unknown;
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      return { ok: false, error: "Extra Body 必须是 JSON 对象" };
+    }
+    return { ok: true, value: parsed as Record<string, unknown> };
+  } catch {
+    return { ok: false, error: "Extra Body 不是合法 JSON" };
+  }
+}
+
+function valuesWithExtraBody(
+  values: Record<string, unknown>,
+  extraBody: Record<string, unknown> | null,
+): Record<string, unknown> {
+  const next = { ...values };
+  delete next.extraBody;
+  if (extraBody) next.extraBody = extraBody;
+  return next;
+}
+
+function isSelectableDefinition(definition: InterfaceDefinition): boolean {
+  return (
+    definition.capability !== "LLM" || definition.interfaceType === "openai-llm"
+  );
+}
+
 export function ComponentsModelsPage() {
  const [resources, setResources] = useState<ModelResource[]>([]);
  const [definitions, setDefinitions] = useState<InterfaceDefinition[]>([]);
@@ -128,6 +171,7 @@ export function ComponentsModelsPage() {
  const [dialogOpen, setDialogOpen] = useState(false);
  const [editingId, setEditingId] = useState<string | null>(null);
  const [draft, setDraft] = useState<ResourceDraft>(emptyDraft);
+  const [extraBodyDraft, setExtraBodyDraft] = useState("");
  const [saving, setSaving] = useState(false);
  const [saveError, setSaveError] = useState("");
  const [testing, setTesting] = useState(false);
@@ -161,7 +205,8 @@ export function ComponentsModelsPage() {
    void load();
  }, [load]);

-  const availableDefinitions = definitions.filter(
+  const selectableDefinitions = definitions.filter(isSelectableDefinition);
+  const availableDefinitions = selectableDefinitions.filter(
    (definition) => definition.capability === draft.capability,
  );
  const selectedDefinition = definitions.find(
@@ -239,11 +284,12 @@ export function ComponentsModelsPage() {
      values: definition ? defaults(definition) : {},
      secrets: {},
    }));
+    setExtraBodyDraft("");
    setTestResult(null);
  }

  function selectCapability(capability: ModelType) {
-    const first = definitions.find(
+    const first = selectableDefinitions.find(
      (definition) => definition.capability === capability,
    );
    setDraft((previous) => ({
@@ -252,12 +298,15 @@ export function ComponentsModelsPage() {
      interfaceType: first?.interfaceType ?? "",
      values: first ? defaults(first) : {},
      secrets: {},
+      supportImageInput:
+        capability === "LLM" ? previous.supportImageInput : false,
    }));
+    setExtraBodyDraft("");
    setTestResult(null);
  }

  function openCreate() {
-    const first = definitions.find(
+    const first = selectableDefinitions.find(
      (definition) => definition.capability === "LLM",
    );
    setEditingId(null);
@@ -265,7 +314,9 @@ export function ComponentsModelsPage() {
      ...emptyDraft,
      interfaceType: first?.interfaceType ?? "",
      values: first ? defaults(first) : {},
+      supportImageInput: false,
    });
+    setExtraBodyDraft("");
    setSaveError("");
    setTestResult(null);
    setDialogOpen(true);
@@ -279,7 +330,9 @@ export function ComponentsModelsPage() {
      interfaceType: resource.interfaceType,
      values: resource.values,
      secrets: resource.secrets,
+      supportImageInput: resource.supportImageInput,
    });
+    setExtraBodyDraft(extraBodyText(resource.values));
    setSaveError("");
    setTestResult(null);
    setDialogOpen(true);
@@ -294,9 +347,12 @@ export function ComponentsModelsPage() {
    setTestResult(null);
  }

+  const parsedExtraBody = parseExtraBody(extraBodyDraft);
+  const extraBodyError = parsedExtraBody.ok ? "" : parsedExtraBody.error;
  const canSave = Boolean(
    draft.name.trim() &&
      selectedDefinition &&
+      !extraBodyError &&
      selectedDefinition.fieldSchema.fields.every((field) => {
        if (!field.required) return true;
        const value = fieldValue(draft, field);
@@ -305,11 +361,17 @@ export function ComponentsModelsPage() {
  );

  function payload() {
+    const extraBody = parsedExtraBody.ok ? parsedExtraBody.value : null;
    return {
      name: draft.name.trim(),
      interfaceType: draft.interfaceType,
-      values: draft.values,
+      values:
+        draft.capability === "LLM"
+          ? valuesWithExtraBody(draft.values, extraBody)
+          : draft.values,
      secrets: draft.secrets,
+      supportImageInput:
+        draft.capability === "LLM" ? draft.supportImageInput : false,
      enabled: editingId
        ? resources.find((resource) => resource.id === editingId)?.enabled ?? true
        : true,
@@ -457,8 +519,19 @@ export function ComponentsModelsPage() {
              width: "md:w-[360px]",
              cell: (resource) => (
                <>
-                  <div className="truncate font-medium text-foreground">
-                    {resource.name}
+                  <div className="flex min-w-0 items-center gap-2">
+                    <span className="truncate font-medium text-foreground">
+                      {resource.name}
+                    </span>
+                    {resource.supportImageInput && (
+                      <span
+                        className="flex h-5 w-5 shrink-0 items-center justify-center rounded-full bg-surface-strong text-muted-foreground"
+                        title="支持图片输入"
+                        aria-label="支持图片输入"
+                      >
+                        <Eye size={12} />
+                      </span>
+                    )}
                  </div>
                  <div className="mt-1 truncate text-xs text-muted-soft">
                    {String(
@@ -652,6 +725,20 @@ export function ComponentsModelsPage() {
                    </Select>
                  </Field>
                </div>
+                {draft.capability === "LLM" && (
+                  <div className="flex items-center justify-between rounded-xl border border-hairline p-4">
+                    <span className="text-sm font-medium">支持图片输入</span>
+                    <Switch
+                      checked={draft.supportImageInput}
+                      onCheckedChange={(checked) =>
+                        setDraft((previous) => ({
+                          ...previous,
+                          supportImageInput: checked,
+                        }))
+                      }
+                    />
+                  </div>
+                )}
              </FieldSection>

              <FieldSection title="连接与鉴权信息" scrollable fill>
@@ -667,6 +754,25 @@ export function ComponentsModelsPage() {
            </div>

            <FieldSection title="可选项" scrollable tall>
+              {draft.capability === "LLM" && (
+                <Field label="Extra Body">
+                  <Textarea
+                    rows={7}
+                    value={extraBodyDraft}
+                    onChange={(event) => {
+                      setExtraBodyDraft(event.target.value);
+                      setTestResult(null);
+                    }}
+                    placeholder={'例如：{\n  "thinking": { "type": "disabled" }\n}'}
+                    className="font-mono text-xs leading-5"
+                  />
+                  {extraBodyError && (
+                    <div className="text-xs text-destructive">
+                      {extraBodyError}
+                    </div>
+                  )}
+                </Field>
+              )}
              {optionalFields.length > 0 ? (
                optionalFields.map((field) => (
                  <DynamicField
@@ -676,7 +782,7 @@ export function ComponentsModelsPage() {
                    onChange={(value) => updateField(field, value)}
                  />
                ))
-              ) : (
+              ) : draft.capability === "LLM" ? null : (
                <div className="flex h-full items-center justify-center text-sm text-muted-foreground">
                  当前接口没有可选项
                </div>
--- a/frontend/src/hooks/use-camera-preview.ts
+++ b/frontend/src/hooks/use-camera-preview.ts
@@ -3,9 +3,8 @@
 /**
 * 摄像头预览:纯前端 getUserMedia 视频流,供调试抽屉的「视频流」视图使用。
 *
- * 与 use-voice-preview 分离:麦克风/语音会话走 WebRTC 到后端,摄像头目前只在
- * 本地预览。后续接入视觉理解管线时,把这条 video track 加进现有 PeerConnection
- * (pc.addTrack)并在后端 transport 打开 video_in_enabled 即可。
+ * 与 use-voice-preview 分离:这里负责摄像头设备枚举与本地预览;开始视觉会话时,
+ * 调用方会把这条 video track 加进语音 WebRTC PeerConnection。
 */

 import { useCallback, useEffect, useRef, useState } from "react";
@@ -50,12 +49,12 @@ export function useCameraPreview() {
    setStream(null);
  }, []);

-  const start = useCallback(async () => {
-    if (streamRef.current) return;
+  const start = useCallback(async (): Promise<MediaStream | null> => {
+    if (streamRef.current) return streamRef.current;
    setError(null);
    if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
      setError("当前页面无法访问摄像头，请通过 https 或 localhost 访问。");
-      return;
+      return null;
    }
    setStarting(true);
    try {
@@ -66,21 +65,24 @@ export function useCameraPreview() {
      streamRef.current = media;
      setStream(media);
      void refreshCameras();
+      return media;
    } catch (mediaError) {
      setError(cameraErrorMessage(mediaError));
+      return null;
    } finally {
      setStarting(false);
    }
  }, [refreshCameras]);

  const selectCamera = useCallback(
-    async (id: string) => {
+    async (id: string): Promise<MediaStream | null> => {
      deviceIdRef.current = id;
      setDeviceId(id);
      if (streamRef.current) {
        stop();
-        await start();
+        return await start();
      }
+      return null;
    },
    [start, stop],
  );
--- a/frontend/src/hooks/use-voice-preview.ts
+++ b/frontend/src/hooks/use-voice-preview.ts
@@ -23,6 +23,11 @@ import { API_BASE, webrtcApi } from "@/lib/api";

 export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed";

+type ConnectOptions = {
+  visionEnabled?: boolean;
+  videoStream?: MediaStream | null;
+};
+
 export type ChatMessage = {
  id: string;
  role: "user" | "assistant";
@@ -218,7 +223,7 @@ export function useVoicePreview(
    [disconnect, fail],
  );

-  const connect = useCallback(async () => {
+  const connect = useCallback(async (options: ConnectOptions = {}) => {
    if (startingRef.current || pcRef.current || wsRef.current) return;
    if (!assistantId) {
      setError("请先保存助手，再开始语音预览。");
@@ -456,6 +461,11 @@ export function useVoicePreview(
      } else {
        pc.addTransceiver("audio", { direction: "recvonly" });
      }
+      if (options.videoStream) {
+        options.videoStream
+          .getVideoTracks()
+          .forEach((track) => pc.addTrack(track, options.videoStream!));
+      }

      // 4) 生成 offer 并发给后端(assistant_id 在 payload 顶层)
      const offer = await pc.createOffer();
@@ -472,6 +482,7 @@ export function useVoicePreview(
            sdp: localDescription.sdp,
            type: localDescription.type,
            assistant_id: assistantId,
+            vision_enabled: Boolean(options.visionEnabled),
          },
        }),
      );
@@ -482,6 +493,17 @@ export function useVoicePreview(
    }
  }, [assistantId, fail, closeOnRemoteEnd, refreshDevices]);

+  const replaceVideoStream = useCallback(
+    async (videoStream: MediaStream | null) => {
+      const pc = pcRef.current;
+      if (!pc) return;
+      const sender = pc.getSenders().find((s) => s.track?.kind === "video");
+      if (!sender) return;
+      await sender.replaceTrack(videoStream?.getVideoTracks()[0] ?? null);
+    },
+    [],
+  );
+
  // 选择麦克风:更新选择;若会话正在发送麦克风音频,则用 WebRTC replaceTrack
  // 热切换轨道(无需重新协商),并把波形可视化重新接到新流。
  // 未连接时仅记下选择,留待下次 connect 生效。
@@ -558,6 +580,7 @@ export function useVoicePreview(
    selectDevice,
    sendText,
    connect,
+    replaceVideoStream,
    disconnect,
    audioRef,
  };
--- a/frontend/src/lib/api.ts
+++ b/frontend/src/lib/api.ts
@@ -56,6 +56,7 @@ export type ModelResource = {
  interfaceType: string;
  values: Record<string, unknown>;
  secrets: Record<string, unknown>;
+  supportImageInput: boolean;
  enabled: boolean;
  isDefault: boolean;
  updatedAt?: string | null;