Add vision model support and related configurations in Assistant

- Introduce new fields in AssistantConfig, schemas, and database models to support vision capabilities, including `vision_enabled` and `vision_model_resource_id`. - Enhance validation logic in routes to ensure proper handling of vision models and their requirements. - Update the AssistantPage and related frontend components to include options for enabling vision understanding and selecting appropriate vision models. - Modify database seed scripts to include vision-related data for assistants, ensuring consistent setup. - Refactor related functions to integrate vision model handling in the audio-visual processing pipeline.
2026-07-07 21:50:15 +08:00
parent c51a70e134
commit 5f71bf1681
10 changed files with 180 additions and 41 deletions
--- a/15
+++ b/15
@@ -7,7 +7,7 @@
 PSQL = docker compose exec -T postgres psql -U postgres -d postgres

 .DEFAULT_GOAL := help
-.PHONY: help up down restart logs api-logs db db-list db-seed db-seed-model-resources db-seed-assistants db-clear db-reset
+.PHONY: help up down restart logs api-logs db db-list db-init db-seed db-seed-model-resources db-seed-assistants db-clear db-reset

 help: ## 列出所有可用目标
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \
@@ -33,6 +33,11 @@ api-logs: ## 只看后端日志
 db: ## 进入交互式 psql
 	docker compose exec postgres psql -U postgres -d postgres

+db-init: ## 启动 API 并等待建表完成(表由 init_db/create_all 在 API 启动时创建)
+	docker compose up -d postgres api
+	@echo "Waiting for API health (schema init)..."
+	@until docker compose exec -T api python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')" 2>/dev/null; do sleep 1; done
+
 db-list: ## 列出模型资源与助手
 	@$(PSQL) -c "SELECT id, name, capability, interface_type, is_default FROM model_resources ORDER BY id;"
 	@$(PSQL) -c "SELECT id, name, type FROM assistants ORDER BY id;"
@@ -43,9 +48,9 @@ db-seed-model-resources: ## 灌入 12 条模型资源种子(幂等)
 db-seed-assistants: ## 灌入 知识库 + 助手 种子(幂等;依赖模型资源已就绪)
 	$(PSQL) < backend/db/seed_assistants.sql

-db-seed: db-seed-model-resources db-seed-assistants ## 全量灌种子(模型资源→知识库→助手,幂等,可重复执行)
+db-seed: db-init db-seed-model-resources db-seed-assistants ## 全量灌种子(模型资源→知识库→助手,幂等,可重复执行)

-db-clear: ## 清空 助手/知识库/模型资源(按依赖顺序)
-	$(PSQL) -c "TRUNCATE assistant_model_bindings, assistants, knowledge_bases, model_resources CASCADE;"
+db-clear: ## 清空 助手/知识库/模型资源(按依赖顺序;表不存在时跳过)
+	@$(PSQL) -c "DO \$$\$$ BEGIN IF EXISTS (SELECT FROM pg_tables WHERE schemaname = 'public' AND tablename = 'model_resources') THEN TRUNCATE assistant_model_bindings, assistants, knowledge_bases, model_resources CASCADE; END IF; END \$$\$$;"

-db-reset: db-clear db-seed ## 清空后重新灌全部种子
+db-reset: db-init db-clear db-seed ## 清空后重新灌全部种子(空库时仅建表+灌种)
--- a/backend/db/models.py
+++ b/backend/db/models.py
@@ -97,6 +97,12 @@ class Assistant(Base):
    runtime_mode: Mapped[str] = mapped_column(String(16), default="pipeline")
    greeting: Mapped[str] = mapped_column(String(2048), default="")
    enable_interrupt: Mapped[bool] = mapped_column(Boolean, default=True)
+    vision_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
+    vision_model_resource_id: Mapped[str | None] = mapped_column(
+        String(40),
+        ForeignKey("model_resources.id", ondelete="SET NULL"),
+        nullable=True,
+    )

    # KB 引用:被引用时禁止删 KB(RESTRICT),无默认兜底
    knowledge_base_id: Mapped[str | None] = mapped_column(
--- a/backend/db/seed_assistants.sql
+++ b/backend/db/seed_assistants.sql
@@ -8,18 +8,24 @@ ON CONFLICT (id) DO NOTHING;

 INSERT INTO assistants (
  id, name, type, runtime_mode, greeting, enable_interrupt,
+  vision_enabled, vision_model_resource_id,
  knowledge_base_id, prompt, api_url, api_key, app_id, graph
 ) VALUES
  ('asst_001', '政务咨询助手', 'prompt', 'pipeline', '您好，我是政务助手，请问有什么可以帮您？', TRUE,
+   FALSE, NULL,
   'kb_001', '你是一名专业的政务咨询助手，回答准确、简洁，不编造政策内容。', '', '', '', '{}'),
  ('asst_002', '热线工单助手', 'workflow', 'pipeline', '', TRUE,
+   FALSE, NULL,
   NULL, '', '', '', '',
   '{"nodes":[{"id":"1","type":"startCall","position":{"x":0,"y":0},"data":{"name":"开场","prompt":"你好，请问需要办理什么业务？"}}],"edges":[]}'),
  ('asst_003', 'Dify 客服助手', 'dify', 'pipeline', '', TRUE,
+   FALSE, NULL,
   NULL, '', 'https://api.dify.ai/v1', 'app-dify-demo-key', '', '{}'),
  ('asst_004', 'FastGPT 售后助手', 'fastgpt', 'pipeline', '', TRUE,
+   FALSE, NULL,
   NULL, '', 'https://api.fastgpt.in/api/v1/chat/completions', 'fastgpt-demo-key', 'app-fastgpt-001', '{}'),
  ('asst_005', 'OpenCode 代码助手', 'opencode', 'pipeline', '', TRUE,
+   FALSE, NULL,
   NULL, '你是一个代码助手的语音界面，用简洁口语回答工程问题。', 'http://localhost:4096', 'opencode-demo-key', '', '{}')
 ON CONFLICT (id) DO NOTHING;

--- a/backend/models.py
+++ b/backend/models.py
@@ -42,6 +42,15 @@ class AssistantConfig(BaseModel):
    llm_values: dict = {}
    llm_secrets: dict = {}
    llm_support_image_input: bool = False
+    vision_enabled: bool = False
+    vision_model_resource_id: str | None = None
+    vision_model: str = ""
+    vision_llm_interface_type: str = "openai-llm"
+    vision_llm_values: dict = {}
+    vision_llm_secrets: dict = {}
+    vision_llm_support_image_input: bool = False
+    vision_llm_api_key: str = ""
+    vision_llm_base_url: str = ""
    stt_values: dict = {}
    stt_secrets: dict = {}
    tts_values: dict = {}
--- a/backend/routes/assistants.py
+++ b/backend/routes/assistants.py
@@ -24,6 +24,23 @@ def _validate_workflow(body: AssistantUpsert) -> None:
        raise HTTPException(400, "工作流校验失败:" + ";".join(errors))


+async def _validate_vision_model(
+    session: AsyncSession, body: AssistantUpsert
+) -> None:
+    if body.vision_enabled:
+        if body.vision_model_resource_id:
+            resource = await session.get(ModelResource, body.vision_model_resource_id)
+        else:
+            resource_id = body.model_resource_ids.get("LLM")
+            resource = (
+                await session.get(ModelResource, resource_id) if resource_id else None
+            )
+        if not resource or resource.capability != "LLM":
+            raise HTTPException(400, "视觉模型必须引用 LLM 模型资源")
+        if not resource.support_image_input:
+            raise HTTPException(400, "视觉模型必须支持图片输入")
+
+
 async def _sync_bindings(
    session: AsyncSession, assistant_id: str, resource_ids: dict[str, str]
 ) -> None:
@@ -69,6 +86,8 @@ async def _to_out(session: AsyncSession, assistant: Assistant) -> AssistantOut:
        runtime_mode=assistant.runtime_mode,  # type: ignore[arg-type]
        greeting=assistant.greeting,
        enable_interrupt=assistant.enable_interrupt,
+        vision_enabled=assistant.vision_enabled,
+        vision_model_resource_id=assistant.vision_model_resource_id,
        model_resource_ids=await _resource_ids(session, assistant.id),
        knowledge_base_id=assistant.knowledge_base_id,
        prompt=assistant.prompt,
@@ -93,6 +112,7 @@ async def create_assistant(
    body: AssistantUpsert, session: AsyncSession = Depends(get_session)
 ):
    _validate_workflow(body)
+    await _validate_vision_model(session, body)
    data = body.model_dump()
    resource_ids = data.pop("model_resource_ids")
    assistant = Assistant(id=f"asst_{uuid.uuid4().hex[:12]}", **data)
@@ -128,6 +148,8 @@ async def duplicate_assistant(
        runtime_mode=source.runtime_mode,
        greeting=source.greeting,
        enable_interrupt=source.enable_interrupt,
+        vision_enabled=source.vision_enabled,
+        vision_model_resource_id=source.vision_model_resource_id,
        knowledge_base_id=source.knowledge_base_id,
        prompt=source.prompt,
        api_url=source.api_url,
@@ -153,6 +175,7 @@ async def update_assistant(
    if not assistant:
        raise HTTPException(404, "助手不存在")
    _validate_workflow(body)
+    await _validate_vision_model(session, body)
    data = body.model_dump()
    resource_ids = data.pop("model_resource_ids")
    data["api_key"] = resolve_incoming_key(data["api_key"], assistant.api_key)
--- a/backend/routes/voice_webrtc.py
+++ b/backend/routes/voice_webrtc.py
@@ -74,6 +74,16 @@ async def _resolve_config(offer: SignalingOffer) -> AssistantConfig:
    raise ValueError("offer 缺少 assistant_id 或 inline_config")


+def _apply_vision_model(cfg: AssistantConfig) -> None:
+    cfg.model = cfg.vision_model
+    cfg.llm_interface_type = cfg.vision_llm_interface_type
+    cfg.llm_values = cfg.vision_llm_values
+    cfg.llm_secrets = cfg.vision_llm_secrets
+    cfg.llm_support_image_input = cfg.vision_llm_support_image_input
+    cfg.llm_api_key = cfg.vision_llm_api_key
+    cfg.llm_base_url = cfg.vision_llm_base_url
+
+
 async def _handle_offer(websocket, payload, peers):
    from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
    from services.pipecat.pipeline import run_pipeline
@@ -87,8 +97,13 @@ async def _handle_offer(websocket, payload, peers):
        await pc.renegotiate(sdp=offer.sdp, type=offer.type, restart_pc=False)
    else:
        cfg = await _resolve_config(offer)  # 解析放在建连前,配置错就别建连
-        if offer.vision_enabled and not cfg.llm_support_image_input:
-            raise ValueError("当前大语言模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM")
+        vision_enabled = offer.vision_enabled or cfg.vision_enabled
+        if vision_enabled:
+            if not cfg.vision_llm_support_image_input:
+                raise ValueError(
+                    "当前视觉模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM"
+                )
+            _apply_vision_model(cfg)
        pc = SmallWebRTCConnection(ice_servers=aiortc_ice_servers())
        if pc_id:
            pc._pc_id = pc_id
@@ -102,10 +117,10 @@ async def _handle_offer(websocket, payload, peers):
        # 后台跑管线:WebRTC transport + 解析出的运行时配置
        transport = build_webrtc_transport(
            pc,
-            video_in_enabled=offer.vision_enabled,
+            video_in_enabled=vision_enabled,
        )
        asyncio.create_task(
-            run_pipeline(transport, cfg, vision_enabled=offer.vision_enabled)
+            run_pipeline(transport, cfg, vision_enabled=vision_enabled)
        )

    answer = pc.get_answer()
--- a/backend/schemas.py
+++ b/backend/schemas.py
@@ -52,6 +52,8 @@ class AssistantUpsert(CamelModel):
    runtime_mode: RuntimeMode = "pipeline"
    greeting: str = ""
    enable_interrupt: bool = True
+    vision_enabled: bool = False
+    vision_model_resource_id: str | None = None

    model_resource_ids: dict[ModelType, str] = Field(default_factory=dict)
    knowledge_base_id: str | None = None
--- a/backend/services/config_resolver.py
+++ b/backend/services/config_resolver.py
@@ -58,6 +58,11 @@ async def resolve_runtime_config(
    stt_resource = await _resource_for(session, assistant.id, "ASR")
    tts_resource = await _resource_for(session, assistant.id, "TTS")
    realtime_resource = await _resource_for(session, assistant.id, "Realtime")
+    vision_resource = (
+        await session.get(ModelResource, assistant.vision_model_resource_id)
+        if assistant.vision_model_resource_id
+        else llm_resource
+    )

    return AssistantConfig(
        name=assistant.name,
@@ -89,6 +94,17 @@ async def resolve_runtime_config(
        llm_support_image_input=(
            bool(llm_resource.support_image_input) if llm_resource else False
        ),
+        vision_enabled=assistant.vision_enabled,
+        vision_model_resource_id=assistant.vision_model_resource_id,
+        vision_model=str(_value(vision_resource, "modelId", "")),
+        vision_llm_interface_type=(
+            vision_resource.interface_type if vision_resource else "openai-llm"
+        ),
+        vision_llm_values=(vision_resource.values or {}) if vision_resource else {},
+        vision_llm_secrets=(vision_resource.secrets or {}) if vision_resource else {},
+        vision_llm_support_image_input=(
+            bool(vision_resource.support_image_input) if vision_resource else False
+        ),
        stt_values=(stt_resource.values or {}) if stt_resource else {},
        stt_secrets=(stt_resource.secrets or {}) if stt_resource else {},
        tts_values=(tts_resource.values or {}) if tts_resource else {},
@@ -101,6 +117,12 @@ async def resolve_runtime_config(
        # 运行时连接信息(真 key + url):模型资源优先,否则 .env 兜底
        llm_api_key=_secret(llm_resource, "apiKey", config.LLM_API_KEY),
        llm_base_url=str(_value(llm_resource, "apiUrl", config.LLM_BASE_URL)),
+        vision_llm_api_key=_secret(
+            vision_resource, "apiKey", config.LLM_API_KEY
+        ),
+        vision_llm_base_url=str(
+            _value(vision_resource, "apiUrl", config.LLM_BASE_URL)
+        ),
        stt_api_key=_secret(stt_resource, "apiKey", config.STT_API_KEY),
        stt_base_url=str(_value(stt_resource, "apiUrl", config.STT_BASE_URL)),
        tts_api_key=_secret(tts_resource, "apiKey", config.TTS_API_KEY),
--- a/frontend/src/components/pages/AssistantPage.tsx
+++ b/frontend/src/components/pages/AssistantPage.tsx
@@ -125,6 +125,8 @@ type AssistantForm = {
  voice: string;
  knowledgeBase: string;
  enableInterrupt: boolean;
+  visionEnabled: boolean;
+  visionModelResourceId: string;
 };

 type FastGptForm = {
@@ -155,6 +157,8 @@ type OpenCodeForm = {
  asr: string;
  voice: string;
  enableInterrupt: boolean;
+  visionEnabled: boolean;
+  visionModelResourceId: string;
 };

 type AssistantType = "提示词" | "工作流" | "Dify" | "FastGPT" | "OpenCode";
@@ -214,6 +218,8 @@ function blankPromptForm(name: string): AssistantForm {
    voice: "",
    knowledgeBase: "",
    enableInterrupt: true,
+    visionEnabled: false,
+    visionModelResourceId: "",
  };
 }

@@ -250,6 +256,8 @@ function blankOpenCodeForm(name: string): OpenCodeForm {
    asr: "",
    voice: "",
    enableInterrupt: true,
+    visionEnabled: false,
+    visionModelResourceId: "",
  };
 }
 function formatTimestamp(iso?: string | null): string {
@@ -331,8 +339,6 @@ export function AssistantPage(props: AssistantPageProps) {
  const editingId = props.mode === "edit" ? props.assistantId : null;

  const [form, setForm] = useState<AssistantForm>(() => blankPromptForm(""));
-  // 视觉理解(暂不持久化,不进 savedSnapshot,避免影响「未保存改动」判定)
-  const [visionEnabled, setVisionEnabled] = useState(false);
  const [fastGptForm, setFastGptForm] = useState<FastGptForm>(() =>
    blankFastGptForm(""),
  );
@@ -426,32 +432,39 @@ export function AssistantPage(props: AssistantPageProps) {
  // 按资源类型生成 {value:id, label:name} 选项
  const credOptions = (type: ModelResource["capability"]) =>
    modelResources
-      .filter((c) => {
-        if (c.capability !== type) return false;
-        if (type === "LLM" && visionEnabled) return c.supportImageInput;
-        return true;
-      })
+      .filter((c) => c.capability === type)
      .map((c) => ({ value: c.id, label: c.name }));
+  const visionModelOptionsFor = (currentModelId: string) => modelResources
+    .filter(
+      (c) =>
+        c.capability === "LLM" &&
+        c.supportImageInput &&
+        c.id !== currentModelId,
+    )
+    .map((c) => ({ value: c.id, label: c.name }));
  const kbOptions = knowledgeBases.map((k) => ({ value: k.id, label: k.name }));

-  function modelSupportsImageInput(resourceId: string): boolean {
-    return Boolean(
-      modelResources.find((resource) => resource.id === resourceId)
-        ?.supportImageInput,
-    );
+  function handlePromptVisionEnabledChange(enabled: boolean) {
+    updateForm("visionEnabled", enabled);
+    if (!enabled) updateForm("visionModelResourceId", "");
  }

-  function handleVisionEnabledChange(enabled: boolean) {
-    setVisionEnabled(enabled);
-    if (enabled && form.model && !modelSupportsImageInput(form.model)) {
-      updateForm("model", "");
+  function handlePromptModelChange(value: string) {
+    updateForm("model", value);
+    if (form.visionModelResourceId === value) {
+      updateForm("visionModelResourceId", "");
    }
-    if (
-      enabled &&
-      openCodeForm.model &&
-      !modelSupportsImageInput(openCodeForm.model)
-    ) {
-      updateOpenCodeForm("model", "");
+  }
+
+  function handleOpenCodeVisionEnabledChange(enabled: boolean) {
+    updateOpenCodeForm("visionEnabled", enabled);
+    if (!enabled) updateOpenCodeForm("visionModelResourceId", "");
+  }
+
+  function handleOpenCodeModelChange(value: string) {
+    updateOpenCodeForm("model", value);
+    if (openCodeForm.visionModelResourceId === value) {
+      updateOpenCodeForm("visionModelResourceId", "");
    }
  }

@@ -473,6 +486,8 @@ export function AssistantPage(props: AssistantPageProps) {
      voice: a.modelResourceIds.TTS ?? "",
      knowledgeBase: a.knowledgeBaseId ?? "",
      enableInterrupt: a.enableInterrupt,
+      visionEnabled: a.visionEnabled,
+      visionModelResourceId: a.visionModelResourceId ?? "",
    };
    setForm(next);
    return next;
@@ -533,6 +548,8 @@ export function AssistantPage(props: AssistantPageProps) {
      runtimeMode: "pipeline",
      greeting: "",
      enableInterrupt: true,
+      visionEnabled: false,
+      visionModelResourceId: null,
      modelResourceIds: {},
      knowledgeBaseId: null,
      prompt: "",
@@ -591,6 +608,8 @@ export function AssistantPage(props: AssistantPageProps) {
        runtimeMode: form.runtimeMode,
        greeting: form.greeting,
        enableInterrupt: form.enableInterrupt,
+        visionEnabled: form.visionEnabled,
+        visionModelResourceId: form.visionModelResourceId || null,
        modelResourceIds: {
          ...(form.model ? { LLM: form.model } : {}),
          ...(form.asr ? { ASR: form.asr } : {}),
@@ -679,6 +698,8 @@ export function AssistantPage(props: AssistantPageProps) {
      asr: a.modelResourceIds.ASR ?? "",
      voice: a.modelResourceIds.TTS ?? "",
      enableInterrupt: a.enableInterrupt,
+      visionEnabled: a.visionEnabled,
+      visionModelResourceId: a.visionModelResourceId ?? "",
    };
    setOpenCodeForm(next);
    return next;
@@ -748,6 +769,8 @@ export function AssistantPage(props: AssistantPageProps) {
        name: openCodeForm.name.trim(),
        type: "opencode",
        enableInterrupt: openCodeForm.enableInterrupt,
+        visionEnabled: openCodeForm.visionEnabled,
+        visionModelResourceId: openCodeForm.visionModelResourceId || null,
        modelResourceIds: {
          ...(openCodeForm.model ? { LLM: openCodeForm.model } : {}),
          ...(openCodeForm.asr ? { ASR: openCodeForm.asr } : {}),
@@ -1518,7 +1541,11 @@ export function AssistantPage(props: AssistantPageProps) {
            </SectionCard>
          </div>

-          <DebugDrawer assistantId={editingId} hasUnsavedChanges={dirty} />
+          <DebugDrawer
+            assistantId={editingId}
+            hasUnsavedChanges={dirty}
+            vision={openCodeForm.visionEnabled}
+          />
        </div>
      </div>
    );
@@ -1600,14 +1627,25 @@ export function AssistantPage(props: AssistantPageProps) {
            >
              <ToggleRow
                title="视觉理解"
-                hint="开启后，大语言模型只能选择支持图片输入的资源；开始对话时会允许助手按需理解当前视频画面。"
-                checked={visionEnabled}
-                onChange={handleVisionEnabledChange}
+                hint="开启后，开始对话时会允许助手按需理解当前视频画面。视觉模型选「模型自己」时，大语言模型本身必须支持图片输入。"
+                checked={openCodeForm.visionEnabled}
+                onChange={handleOpenCodeVisionEnabledChange}
              />
+              {openCodeForm.visionEnabled && (
+                <ResourceSelectField
+                  label="视觉模型"
+                  value={openCodeForm.visionModelResourceId}
+                  onChange={(value) =>
+                    updateOpenCodeForm("visionModelResourceId", value)
+                  }
+                  options={visionModelOptionsFor(openCodeForm.model)}
+                  noneLabel="模型自己"
+                />
+              )}
              <ResourceSelectField
                label="大语言模型"
                value={openCodeForm.model}
-                onChange={(value) => updateOpenCodeForm("model", value)}
+                onChange={handleOpenCodeModelChange}
                options={credOptions("LLM")}
                noneLabel="无"
              />
@@ -1779,14 +1817,25 @@ export function AssistantPage(props: AssistantPageProps) {
            >
              <ToggleRow
                title="视觉理解"
-                hint="开启后，大语言模型只能选择支持图片输入的资源；开始对话时会允许助手按需理解当前视频画面。"
-                checked={visionEnabled}
-                onChange={handleVisionEnabledChange}
+                hint="开启后，开始对话时会允许助手按需理解当前视频画面。视觉模型选「模型自己」时，大语言模型本身必须支持图片输入。"
+                checked={form.visionEnabled}
+                onChange={handlePromptVisionEnabledChange}
              />
+              {form.visionEnabled && (
+                <ResourceSelectField
+                  label="视觉模型"
+                  value={form.visionModelResourceId}
+                  onChange={(value) =>
+                    updateForm("visionModelResourceId", value)
+                  }
+                  options={visionModelOptionsFor(form.model)}
+                  noneLabel="模型自己"
+                />
+              )}
              <ResourceSelectField
                label="大语言模型"
                value={form.model}
-                onChange={(value) => updateForm("model", value)}
+                onChange={handlePromptModelChange}
                options={credOptions("LLM")}
                noneLabel="无"
              />
@@ -1863,7 +1912,7 @@ export function AssistantPage(props: AssistantPageProps) {
        <DebugDrawer
          assistantId={editingId}
          hasUnsavedChanges={dirty}
-          vision={visionEnabled}
+          vision={form.visionEnabled}
        />
      </div>
    </div>
--- a/frontend/src/lib/api.ts
+++ b/frontend/src/lib/api.ts
@@ -128,6 +128,8 @@ export type Assistant = {
  runtimeMode: RuntimeMode;
  greeting: string;
  enableInterrupt: boolean;
+  visionEnabled: boolean;
+  visionModelResourceId: string | null;
  modelResourceIds: Partial<Record<ModelType, string>>;
  knowledgeBaseId: string | null;
  prompt: string;