Add vision model support and related configurations in Assistant

- Introduce new fields in AssistantConfig, schemas, and database models to support vision capabilities, including `vision_enabled` and `vision_model_resource_id`.
- Enhance validation logic in routes to ensure proper handling of vision models and their requirements.
- Update the AssistantPage and related frontend components to include options for enabling vision understanding and selecting appropriate vision models.
- Modify database seed scripts to include vision-related data for assistants, ensuring consistent setup.
- Refactor related functions to integrate vision model handling in the audio-visual processing pipeline.

backend/routes/assistants.py

@@ -24,6 +24,23 @@ def _validate_workflow(body: AssistantUpsert) -> None:
         raise HTTPException(400, "工作流校验失败:" + ";".join(errors))
 
 
+async def _validate_vision_model(
+    session: AsyncSession, body: AssistantUpsert
+) -> None:
+    if body.vision_enabled:
+        if body.vision_model_resource_id:
+            resource = await session.get(ModelResource, body.vision_model_resource_id)
+        else:
+            resource_id = body.model_resource_ids.get("LLM")
+            resource = (
+                await session.get(ModelResource, resource_id) if resource_id else None
+            )
+        if not resource or resource.capability != "LLM":
+            raise HTTPException(400, "视觉模型必须引用 LLM 模型资源")
+        if not resource.support_image_input:
+            raise HTTPException(400, "视觉模型必须支持图片输入")
+
+
 async def _sync_bindings(
     session: AsyncSession, assistant_id: str, resource_ids: dict[str, str]
 ) -> None:
@@ -69,6 +86,8 @@ async def _to_out(session: AsyncSession, assistant: Assistant) -> AssistantOut:
         runtime_mode=assistant.runtime_mode,  # type: ignore[arg-type]
         greeting=assistant.greeting,
         enable_interrupt=assistant.enable_interrupt,
+        vision_enabled=assistant.vision_enabled,
+        vision_model_resource_id=assistant.vision_model_resource_id,
         model_resource_ids=await _resource_ids(session, assistant.id),
         knowledge_base_id=assistant.knowledge_base_id,
         prompt=assistant.prompt,
@@ -93,6 +112,7 @@ async def create_assistant(
     body: AssistantUpsert, session: AsyncSession = Depends(get_session)
 ):
     _validate_workflow(body)
+    await _validate_vision_model(session, body)
     data = body.model_dump()
     resource_ids = data.pop("model_resource_ids")
     assistant = Assistant(id=f"asst_{uuid.uuid4().hex[:12]}", **data)
@@ -128,6 +148,8 @@ async def duplicate_assistant(
         runtime_mode=source.runtime_mode,
         greeting=source.greeting,
         enable_interrupt=source.enable_interrupt,
+        vision_enabled=source.vision_enabled,
+        vision_model_resource_id=source.vision_model_resource_id,
         knowledge_base_id=source.knowledge_base_id,
         prompt=source.prompt,
         api_url=source.api_url,
@@ -153,6 +175,7 @@ async def update_assistant(
     if not assistant:
         raise HTTPException(404, "助手不存在")
     _validate_workflow(body)
+    await _validate_vision_model(session, body)
     data = body.model_dump()
     resource_ids = data.pop("model_resource_ids")
     data["api_key"] = resolve_incoming_key(data["api_key"], assistant.api_key)

backend/routes/voice_webrtc.py

@@ -74,6 +74,16 @@ async def _resolve_config(offer: SignalingOffer) -> AssistantConfig:
     raise ValueError("offer 缺少 assistant_id 或 inline_config")
 
 
+def _apply_vision_model(cfg: AssistantConfig) -> None:
+    cfg.model = cfg.vision_model
+    cfg.llm_interface_type = cfg.vision_llm_interface_type
+    cfg.llm_values = cfg.vision_llm_values
+    cfg.llm_secrets = cfg.vision_llm_secrets
+    cfg.llm_support_image_input = cfg.vision_llm_support_image_input
+    cfg.llm_api_key = cfg.vision_llm_api_key
+    cfg.llm_base_url = cfg.vision_llm_base_url
+
+
 async def _handle_offer(websocket, payload, peers):
     from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
     from services.pipecat.pipeline import run_pipeline
@@ -87,8 +97,13 @@ async def _handle_offer(websocket, payload, peers):
         await pc.renegotiate(sdp=offer.sdp, type=offer.type, restart_pc=False)
     else:
         cfg = await _resolve_config(offer)  # 解析放在建连前,配置错就别建连
-        if offer.vision_enabled and not cfg.llm_support_image_input:
-            raise ValueError("当前大语言模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM")
+        vision_enabled = offer.vision_enabled or cfg.vision_enabled
+        if vision_enabled:
+            if not cfg.vision_llm_support_image_input:
+                raise ValueError(
+                    "当前视觉模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM"
+                )
+            _apply_vision_model(cfg)
         pc = SmallWebRTCConnection(ice_servers=aiortc_ice_servers())
         if pc_id:
             pc._pc_id = pc_id
@@ -102,10 +117,10 @@ async def _handle_offer(websocket, payload, peers):
         # 后台跑管线:WebRTC transport + 解析出的运行时配置
         transport = build_webrtc_transport(
             pc,
-            video_in_enabled=offer.vision_enabled,
+            video_in_enabled=vision_enabled,
         )
         asyncio.create_task(
-            run_pipeline(transport, cfg, vision_enabled=offer.vision_enabled)
+            run_pipeline(transport, cfg, vision_enabled=vision_enabled)
         )
 
     answer = pc.get_answer()