Add vision model support and related configurations in Assistant

- Introduce new fields in AssistantConfig, schemas, and database models to support vision capabilities, including `vision_enabled` and `vision_model_resource_id`. - Enhance validation logic in routes to ensure proper handling of vision models and their requirements. - Update the AssistantPage and related frontend components to include options for enabling vision understanding and selecting appropriate vision models. - Modify database seed scripts to include vision-related data for assistants, ensuring consistent setup. - Refactor related functions to integrate vision model handling in the audio-visual processing pipeline.
2026-07-07 21:50:15 +08:00
parent c51a70e134
commit 5f71bf1681
10 changed files with 180 additions and 41 deletions
--- a/backend/db/models.py
+++ b/backend/db/models.py
@@ -97,6 +97,12 @@ class Assistant(Base):
    runtime_mode: Mapped[str] = mapped_column(String(16), default="pipeline")
    greeting: Mapped[str] = mapped_column(String(2048), default="")
    enable_interrupt: Mapped[bool] = mapped_column(Boolean, default=True)
+    vision_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
+    vision_model_resource_id: Mapped[str | None] = mapped_column(
+        String(40),
+        ForeignKey("model_resources.id", ondelete="SET NULL"),
+        nullable=True,
+    )

    # KB 引用:被引用时禁止删 KB(RESTRICT),无默认兜底
    knowledge_base_id: Mapped[str | None] = mapped_column(
--- a/backend/db/seed_assistants.sql
+++ b/backend/db/seed_assistants.sql
@@ -8,18 +8,24 @@ ON CONFLICT (id) DO NOTHING;

 INSERT INTO assistants (
  id, name, type, runtime_mode, greeting, enable_interrupt,
+  vision_enabled, vision_model_resource_id,
  knowledge_base_id, prompt, api_url, api_key, app_id, graph
 ) VALUES
  ('asst_001', '政务咨询助手', 'prompt', 'pipeline', '您好，我是政务助手，请问有什么可以帮您？', TRUE,
+   FALSE, NULL,
   'kb_001', '你是一名专业的政务咨询助手，回答准确、简洁，不编造政策内容。', '', '', '', '{}'),
  ('asst_002', '热线工单助手', 'workflow', 'pipeline', '', TRUE,
+   FALSE, NULL,
   NULL, '', '', '', '',
   '{"nodes":[{"id":"1","type":"startCall","position":{"x":0,"y":0},"data":{"name":"开场","prompt":"你好，请问需要办理什么业务？"}}],"edges":[]}'),
  ('asst_003', 'Dify 客服助手', 'dify', 'pipeline', '', TRUE,
+   FALSE, NULL,
   NULL, '', 'https://api.dify.ai/v1', 'app-dify-demo-key', '', '{}'),
  ('asst_004', 'FastGPT 售后助手', 'fastgpt', 'pipeline', '', TRUE,
+   FALSE, NULL,
   NULL, '', 'https://api.fastgpt.in/api/v1/chat/completions', 'fastgpt-demo-key', 'app-fastgpt-001', '{}'),
  ('asst_005', 'OpenCode 代码助手', 'opencode', 'pipeline', '', TRUE,
+   FALSE, NULL,
   NULL, '你是一个代码助手的语音界面，用简洁口语回答工程问题。', 'http://localhost:4096', 'opencode-demo-key', '', '{}')
 ON CONFLICT (id) DO NOTHING;

--- a/backend/models.py
+++ b/backend/models.py
@@ -42,6 +42,15 @@ class AssistantConfig(BaseModel):
    llm_values: dict = {}
    llm_secrets: dict = {}
    llm_support_image_input: bool = False
+    vision_enabled: bool = False
+    vision_model_resource_id: str | None = None
+    vision_model: str = ""
+    vision_llm_interface_type: str = "openai-llm"
+    vision_llm_values: dict = {}
+    vision_llm_secrets: dict = {}
+    vision_llm_support_image_input: bool = False
+    vision_llm_api_key: str = ""
+    vision_llm_base_url: str = ""
    stt_values: dict = {}
    stt_secrets: dict = {}
    tts_values: dict = {}
--- a/backend/routes/assistants.py
+++ b/backend/routes/assistants.py
@@ -24,6 +24,23 @@ def _validate_workflow(body: AssistantUpsert) -> None:
        raise HTTPException(400, "工作流校验失败:" + ";".join(errors))


+async def _validate_vision_model(
+    session: AsyncSession, body: AssistantUpsert
+) -> None:
+    if body.vision_enabled:
+        if body.vision_model_resource_id:
+            resource = await session.get(ModelResource, body.vision_model_resource_id)
+        else:
+            resource_id = body.model_resource_ids.get("LLM")
+            resource = (
+                await session.get(ModelResource, resource_id) if resource_id else None
+            )
+        if not resource or resource.capability != "LLM":
+            raise HTTPException(400, "视觉模型必须引用 LLM 模型资源")
+        if not resource.support_image_input:
+            raise HTTPException(400, "视觉模型必须支持图片输入")
+
+
 async def _sync_bindings(
    session: AsyncSession, assistant_id: str, resource_ids: dict[str, str]
 ) -> None:
@@ -69,6 +86,8 @@ async def _to_out(session: AsyncSession, assistant: Assistant) -> AssistantOut:
        runtime_mode=assistant.runtime_mode,  # type: ignore[arg-type]
        greeting=assistant.greeting,
        enable_interrupt=assistant.enable_interrupt,
+        vision_enabled=assistant.vision_enabled,
+        vision_model_resource_id=assistant.vision_model_resource_id,
        model_resource_ids=await _resource_ids(session, assistant.id),
        knowledge_base_id=assistant.knowledge_base_id,
        prompt=assistant.prompt,
@@ -93,6 +112,7 @@ async def create_assistant(
    body: AssistantUpsert, session: AsyncSession = Depends(get_session)
 ):
    _validate_workflow(body)
+    await _validate_vision_model(session, body)
    data = body.model_dump()
    resource_ids = data.pop("model_resource_ids")
    assistant = Assistant(id=f"asst_{uuid.uuid4().hex[:12]}", **data)
@@ -128,6 +148,8 @@ async def duplicate_assistant(
        runtime_mode=source.runtime_mode,
        greeting=source.greeting,
        enable_interrupt=source.enable_interrupt,
+        vision_enabled=source.vision_enabled,
+        vision_model_resource_id=source.vision_model_resource_id,
        knowledge_base_id=source.knowledge_base_id,
        prompt=source.prompt,
        api_url=source.api_url,
@@ -153,6 +175,7 @@ async def update_assistant(
    if not assistant:
        raise HTTPException(404, "助手不存在")
    _validate_workflow(body)
+    await _validate_vision_model(session, body)
    data = body.model_dump()
    resource_ids = data.pop("model_resource_ids")
    data["api_key"] = resolve_incoming_key(data["api_key"], assistant.api_key)
--- a/backend/routes/voice_webrtc.py
+++ b/backend/routes/voice_webrtc.py
@@ -74,6 +74,16 @@ async def _resolve_config(offer: SignalingOffer) -> AssistantConfig:
    raise ValueError("offer 缺少 assistant_id 或 inline_config")


+def _apply_vision_model(cfg: AssistantConfig) -> None:
+    cfg.model = cfg.vision_model
+    cfg.llm_interface_type = cfg.vision_llm_interface_type
+    cfg.llm_values = cfg.vision_llm_values
+    cfg.llm_secrets = cfg.vision_llm_secrets
+    cfg.llm_support_image_input = cfg.vision_llm_support_image_input
+    cfg.llm_api_key = cfg.vision_llm_api_key
+    cfg.llm_base_url = cfg.vision_llm_base_url
+
+
 async def _handle_offer(websocket, payload, peers):
    from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
    from services.pipecat.pipeline import run_pipeline
@@ -87,8 +97,13 @@ async def _handle_offer(websocket, payload, peers):
        await pc.renegotiate(sdp=offer.sdp, type=offer.type, restart_pc=False)
    else:
        cfg = await _resolve_config(offer)  # 解析放在建连前,配置错就别建连
-        if offer.vision_enabled and not cfg.llm_support_image_input:
-            raise ValueError("当前大语言模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM")
+        vision_enabled = offer.vision_enabled or cfg.vision_enabled
+        if vision_enabled:
+            if not cfg.vision_llm_support_image_input:
+                raise ValueError(
+                    "当前视觉模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM"
+                )
+            _apply_vision_model(cfg)
        pc = SmallWebRTCConnection(ice_servers=aiortc_ice_servers())
        if pc_id:
            pc._pc_id = pc_id
@@ -102,10 +117,10 @@ async def _handle_offer(websocket, payload, peers):
        # 后台跑管线:WebRTC transport + 解析出的运行时配置
        transport = build_webrtc_transport(
            pc,
-            video_in_enabled=offer.vision_enabled,
+            video_in_enabled=vision_enabled,
        )
        asyncio.create_task(
-            run_pipeline(transport, cfg, vision_enabled=offer.vision_enabled)
+            run_pipeline(transport, cfg, vision_enabled=vision_enabled)
        )

    answer = pc.get_answer()
--- a/backend/schemas.py
+++ b/backend/schemas.py
@@ -52,6 +52,8 @@ class AssistantUpsert(CamelModel):
    runtime_mode: RuntimeMode = "pipeline"
    greeting: str = ""
    enable_interrupt: bool = True
+    vision_enabled: bool = False
+    vision_model_resource_id: str | None = None

    model_resource_ids: dict[ModelType, str] = Field(default_factory=dict)
    knowledge_base_id: str | None = None
--- a/backend/services/config_resolver.py
+++ b/backend/services/config_resolver.py
@@ -58,6 +58,11 @@ async def resolve_runtime_config(
    stt_resource = await _resource_for(session, assistant.id, "ASR")
    tts_resource = await _resource_for(session, assistant.id, "TTS")
    realtime_resource = await _resource_for(session, assistant.id, "Realtime")
+    vision_resource = (
+        await session.get(ModelResource, assistant.vision_model_resource_id)
+        if assistant.vision_model_resource_id
+        else llm_resource
+    )

    return AssistantConfig(
        name=assistant.name,
@@ -89,6 +94,17 @@ async def resolve_runtime_config(
        llm_support_image_input=(
            bool(llm_resource.support_image_input) if llm_resource else False
        ),
+        vision_enabled=assistant.vision_enabled,
+        vision_model_resource_id=assistant.vision_model_resource_id,
+        vision_model=str(_value(vision_resource, "modelId", "")),
+        vision_llm_interface_type=(
+            vision_resource.interface_type if vision_resource else "openai-llm"
+        ),
+        vision_llm_values=(vision_resource.values or {}) if vision_resource else {},
+        vision_llm_secrets=(vision_resource.secrets or {}) if vision_resource else {},
+        vision_llm_support_image_input=(
+            bool(vision_resource.support_image_input) if vision_resource else False
+        ),
        stt_values=(stt_resource.values or {}) if stt_resource else {},
        stt_secrets=(stt_resource.secrets or {}) if stt_resource else {},
        tts_values=(tts_resource.values or {}) if tts_resource else {},
@@ -101,6 +117,12 @@ async def resolve_runtime_config(
        # 运行时连接信息(真 key + url):模型资源优先,否则 .env 兜底
        llm_api_key=_secret(llm_resource, "apiKey", config.LLM_API_KEY),
        llm_base_url=str(_value(llm_resource, "apiUrl", config.LLM_BASE_URL)),
+        vision_llm_api_key=_secret(
+            vision_resource, "apiKey", config.LLM_API_KEY
+        ),
+        vision_llm_base_url=str(
+            _value(vision_resource, "apiUrl", config.LLM_BASE_URL)
+        ),
        stt_api_key=_secret(stt_resource, "apiKey", config.STT_API_KEY),
        stt_base_url=str(_value(stt_resource, "apiUrl", config.STT_BASE_URL)),
        tts_api_key=_secret(tts_resource, "apiKey", config.TTS_API_KEY),