Add support for image input in model resources and enhance related configurations

- Introduce a new `support_image_input` field in model resources, allowing models to indicate support for image input.
- Update the backend models, schemas, and database seed scripts to accommodate the new field.
- Enhance the AssistantConfig and related routes to handle image input capabilities, ensuring proper validation and error handling.
- Modify the frontend components to include toggles for enabling visual understanding and filtering models based on image input support.
- Implement necessary adjustments in the voice preview and pipeline to integrate video stream handling alongside audio functionalities.

backend/routes/model_registry.py

@@ -53,6 +53,7 @@ def _resource_out(row: ModelResource) -> ModelResourceOut:
         interface_type=row.interface_type,
         values=row.values or {},
         secrets=mask_secrets(row.secrets or {}),
+        support_image_input=row.support_image_input,
         enabled=row.enabled,
         is_default=row.is_default,
         updated_at=row.updated_at.isoformat() if row.updated_at else None,
@@ -140,6 +141,9 @@ async def create_model_resource(
         interface_type=definition.interface_type,
         values=body.values,
         secrets=secrets,
+        support_image_input=body.support_image_input
+        if definition.capability == "LLM"
+        else False,
         enabled=body.enabled,
         is_default=body.is_default,
     )
@@ -200,6 +204,7 @@ async def duplicate_model_resource(
         interface_type=source.interface_type,
         values=dict(source.values or {}),
         secrets=dict(source.secrets or {}),
+        support_image_input=source.support_image_input,
         enabled=source.enabled,
         is_default=False,
     )
@@ -223,6 +228,9 @@ async def update_model_resource(
     row.interface_type = definition.interface_type
     row.values = body.values
     row.secrets = secrets
+    row.support_image_input = (
+        body.support_image_input if definition.capability == "LLM" else False
+    )
     row.enabled = body.enabled
     row.is_default = body.is_default
     if row.is_default:

backend/routes/voice_webrtc.py

@@ -87,6 +87,8 @@ async def _handle_offer(websocket, payload, peers):
         await pc.renegotiate(sdp=offer.sdp, type=offer.type, restart_pc=False)
     else:
         cfg = await _resolve_config(offer)  # 解析放在建连前,配置错就别建连
+        if offer.vision_enabled and not cfg.llm_support_image_input:
+            raise ValueError("当前大语言模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM")
         pc = SmallWebRTCConnection(ice_servers=aiortc_ice_servers())
         if pc_id:
             pc._pc_id = pc_id
@@ -98,8 +100,13 @@ async def _handle_offer(websocket, payload, peers):
             peers.pop(conn.pc_id, None)
 
         # 后台跑管线:WebRTC transport + 解析出的运行时配置
-        transport = build_webrtc_transport(pc)
-        asyncio.create_task(run_pipeline(transport, cfg))
+        transport = build_webrtc_transport(
+            pc,
+            video_in_enabled=offer.vision_enabled,
+        )
+        asyncio.create_task(
+            run_pipeline(transport, cfg, vision_enabled=offer.vision_enabled)
+        )
 
     answer = pc.get_answer()
     if websocket.application_state == WebSocketState.CONNECTED: