Add vision model support and related configurations in Assistant

- Introduce new fields in AssistantConfig, schemas, and database models to support vision capabilities, including `vision_enabled` and `vision_model_resource_id`.
- Enhance validation logic in routes to ensure proper handling of vision models and their requirements.
- Update the AssistantPage and related frontend components to include options for enabling vision understanding and selecting appropriate vision models.
- Modify database seed scripts to include vision-related data for assistants, ensuring consistent setup.
- Refactor related functions to integrate vision model handling in the audio-visual processing pipeline.

Makefile

@@ -7,7 +7,7 @@
 PSQL = docker compose exec -T postgres psql -U postgres -d postgres
 
 .DEFAULT_GOAL := help
-.PHONY: help up down restart logs api-logs db db-list db-seed db-seed-model-resources db-seed-assistants db-clear db-reset
+.PHONY: help up down restart logs api-logs db db-list db-init db-seed db-seed-model-resources db-seed-assistants db-clear db-reset
 
 help: ## 列出所有可用目标
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \
@@ -33,6 +33,11 @@ api-logs: ## 只看后端日志
 db: ## 进入交互式 psql
 	docker compose exec postgres psql -U postgres -d postgres
 
+db-init: ## 启动 API 并等待建表完成(表由 init_db/create_all 在 API 启动时创建)
+	docker compose up -d postgres api
+	@echo "Waiting for API health (schema init)..."
+	@until docker compose exec -T api python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')" 2>/dev/null; do sleep 1; done
+
 db-list: ## 列出模型资源与助手
 	@$(PSQL) -c "SELECT id, name, capability, interface_type, is_default FROM model_resources ORDER BY id;"
 	@$(PSQL) -c "SELECT id, name, type FROM assistants ORDER BY id;"
@@ -43,9 +48,9 @@ db-seed-model-resources: ## 灌入 12 条模型资源种子(幂等)
 db-seed-assistants: ## 灌入 知识库 + 助手 种子(幂等;依赖模型资源已就绪)
 	$(PSQL) < backend/db/seed_assistants.sql
 
-db-seed: db-seed-model-resources db-seed-assistants ## 全量灌种子(模型资源→知识库→助手,幂等,可重复执行)
+db-seed: db-init db-seed-model-resources db-seed-assistants ## 全量灌种子(模型资源→知识库→助手,幂等,可重复执行)
 
-db-clear: ## 清空 助手/知识库/模型资源(按依赖顺序)
+db-clear: ## 清空 助手/知识库/模型资源(按依赖顺序;表不存在时跳过)
-	$(PSQL) -c "TRUNCATE assistant_model_bindings, assistants, knowledge_bases, model_resources CASCADE;"
+	@$(PSQL) -c "DO \$$\$$ BEGIN IF EXISTS (SELECT FROM pg_tables WHERE schemaname = 'public' AND tablename = 'model_resources') THEN TRUNCATE assistant_model_bindings, assistants, knowledge_bases, model_resources CASCADE; END IF; END \$$\$$;"
 
-db-reset: db-clear db-seed ## 清空后重新灌全部种子
+db-reset: db-init db-clear db-seed ## 清空后重新灌全部种子(空库时仅建表+灌种)

backend/db/models.py

@@ -97,6 +97,12 @@ class Assistant(Base):
     runtime_mode: Mapped[str] = mapped_column(String(16), default="pipeline")
     greeting: Mapped[str] = mapped_column(String(2048), default="")
     enable_interrupt: Mapped[bool] = mapped_column(Boolean, default=True)
+    vision_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
+    vision_model_resource_id: Mapped[str | None] = mapped_column(
+        String(40),
+        ForeignKey("model_resources.id", ondelete="SET NULL"),
+        nullable=True,
+    )
 
     # KB 引用:被引用时禁止删 KB(RESTRICT),无默认兜底
     knowledge_base_id: Mapped[str | None] = mapped_column(

backend/db/seed_assistants.sql

@@ -8,18 +8,24 @@ ON CONFLICT (id) DO NOTHING;
 
 INSERT INTO assistants (
   id, name, type, runtime_mode, greeting, enable_interrupt,
+  vision_enabled, vision_model_resource_id,
   knowledge_base_id, prompt, api_url, api_key, app_id, graph
 ) VALUES
   ('asst_001', '政务咨询助手', 'prompt', 'pipeline', '您好，我是政务助手，请问有什么可以帮您？', TRUE,
+   FALSE, NULL,
    'kb_001', '你是一名专业的政务咨询助手，回答准确、简洁，不编造政策内容。', '', '', '', '{}'),
   ('asst_002', '热线工单助手', 'workflow', 'pipeline', '', TRUE,
+   FALSE, NULL,
    NULL, '', '', '', '',
    '{"nodes":[{"id":"1","type":"startCall","position":{"x":0,"y":0},"data":{"name":"开场","prompt":"你好，请问需要办理什么业务？"}}],"edges":[]}'),
   ('asst_003', 'Dify 客服助手', 'dify', 'pipeline', '', TRUE,
+   FALSE, NULL,
    NULL, '', 'https://api.dify.ai/v1', 'app-dify-demo-key', '', '{}'),
   ('asst_004', 'FastGPT 售后助手', 'fastgpt', 'pipeline', '', TRUE,
+   FALSE, NULL,
    NULL, '', 'https://api.fastgpt.in/api/v1/chat/completions', 'fastgpt-demo-key', 'app-fastgpt-001', '{}'),
   ('asst_005', 'OpenCode 代码助手', 'opencode', 'pipeline', '', TRUE,
+   FALSE, NULL,
    NULL, '你是一个代码助手的语音界面，用简洁口语回答工程问题。', 'http://localhost:4096', 'opencode-demo-key', '', '{}')
 ON CONFLICT (id) DO NOTHING;
 

backend/models.py

@@ -42,6 +42,15 @@ class AssistantConfig(BaseModel):
     llm_values: dict = {}
     llm_secrets: dict = {}
     llm_support_image_input: bool = False
+    vision_enabled: bool = False
+    vision_model_resource_id: str | None = None
+    vision_model: str = ""
+    vision_llm_interface_type: str = "openai-llm"
+    vision_llm_values: dict = {}
+    vision_llm_secrets: dict = {}
+    vision_llm_support_image_input: bool = False
+    vision_llm_api_key: str = ""
+    vision_llm_base_url: str = ""
     stt_values: dict = {}
     stt_secrets: dict = {}
     tts_values: dict = {}

backend/routes/assistants.py

@@ -24,6 +24,23 @@ def _validate_workflow(body: AssistantUpsert) -> None:
         raise HTTPException(400, "工作流校验失败:" + ";".join(errors))
 
 
+async def _validate_vision_model(
+    session: AsyncSession, body: AssistantUpsert
+) -> None:
+    if body.vision_enabled:
+        if body.vision_model_resource_id:
+            resource = await session.get(ModelResource, body.vision_model_resource_id)
+        else:
+            resource_id = body.model_resource_ids.get("LLM")
+            resource = (
+                await session.get(ModelResource, resource_id) if resource_id else None
+            )
+        if not resource or resource.capability != "LLM":
+            raise HTTPException(400, "视觉模型必须引用 LLM 模型资源")
+        if not resource.support_image_input:
+            raise HTTPException(400, "视觉模型必须支持图片输入")
+
+
 async def _sync_bindings(
     session: AsyncSession, assistant_id: str, resource_ids: dict[str, str]
 ) -> None:
@@ -69,6 +86,8 @@ async def _to_out(session: AsyncSession, assistant: Assistant) -> AssistantOut:
         runtime_mode=assistant.runtime_mode,  # type: ignore[arg-type]
         greeting=assistant.greeting,
         enable_interrupt=assistant.enable_interrupt,
+        vision_enabled=assistant.vision_enabled,
+        vision_model_resource_id=assistant.vision_model_resource_id,
         model_resource_ids=await _resource_ids(session, assistant.id),
         knowledge_base_id=assistant.knowledge_base_id,
         prompt=assistant.prompt,
@@ -93,6 +112,7 @@ async def create_assistant(
     body: AssistantUpsert, session: AsyncSession = Depends(get_session)
 ):
     _validate_workflow(body)
+    await _validate_vision_model(session, body)
     data = body.model_dump()
     resource_ids = data.pop("model_resource_ids")
     assistant = Assistant(id=f"asst_{uuid.uuid4().hex[:12]}", **data)
@@ -128,6 +148,8 @@ async def duplicate_assistant(
         runtime_mode=source.runtime_mode,
         greeting=source.greeting,
         enable_interrupt=source.enable_interrupt,
+        vision_enabled=source.vision_enabled,
+        vision_model_resource_id=source.vision_model_resource_id,
         knowledge_base_id=source.knowledge_base_id,
         prompt=source.prompt,
         api_url=source.api_url,
@@ -153,6 +175,7 @@ async def update_assistant(
     if not assistant:
         raise HTTPException(404, "助手不存在")
     _validate_workflow(body)
+    await _validate_vision_model(session, body)
     data = body.model_dump()
     resource_ids = data.pop("model_resource_ids")
     data["api_key"] = resolve_incoming_key(data["api_key"], assistant.api_key)

backend/routes/voice_webrtc.py

@@ -74,6 +74,16 @@ async def _resolve_config(offer: SignalingOffer) -> AssistantConfig:
     raise ValueError("offer 缺少 assistant_id 或 inline_config")
 
 
+def _apply_vision_model(cfg: AssistantConfig) -> None:
+    cfg.model = cfg.vision_model
+    cfg.llm_interface_type = cfg.vision_llm_interface_type
+    cfg.llm_values = cfg.vision_llm_values
+    cfg.llm_secrets = cfg.vision_llm_secrets
+    cfg.llm_support_image_input = cfg.vision_llm_support_image_input
+    cfg.llm_api_key = cfg.vision_llm_api_key
+    cfg.llm_base_url = cfg.vision_llm_base_url
+
+
 async def _handle_offer(websocket, payload, peers):
     from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
     from services.pipecat.pipeline import run_pipeline
@@ -87,8 +97,13 @@ async def _handle_offer(websocket, payload, peers):
         await pc.renegotiate(sdp=offer.sdp, type=offer.type, restart_pc=False)
     else:
         cfg = await _resolve_config(offer)  # 解析放在建连前,配置错就别建连
-        if offer.vision_enabled and not cfg.llm_support_image_input:
+        vision_enabled = offer.vision_enabled or cfg.vision_enabled
-            raise ValueError("当前大语言模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM")
+        if vision_enabled:
+            if not cfg.vision_llm_support_image_input:
+                raise ValueError(
+                    "当前视觉模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM"
+                )
+            _apply_vision_model(cfg)
         pc = SmallWebRTCConnection(ice_servers=aiortc_ice_servers())
         if pc_id:
             pc._pc_id = pc_id
@@ -102,10 +117,10 @@ async def _handle_offer(websocket, payload, peers):
         # 后台跑管线:WebRTC transport + 解析出的运行时配置
         transport = build_webrtc_transport(
             pc,
-            video_in_enabled=offer.vision_enabled,
+            video_in_enabled=vision_enabled,
         )
         asyncio.create_task(
-            run_pipeline(transport, cfg, vision_enabled=offer.vision_enabled)
+            run_pipeline(transport, cfg, vision_enabled=vision_enabled)
         )
 
     answer = pc.get_answer()

backend/schemas.py

@@ -52,6 +52,8 @@ class AssistantUpsert(CamelModel):
     runtime_mode: RuntimeMode = "pipeline"
     greeting: str = ""
     enable_interrupt: bool = True
+    vision_enabled: bool = False
+    vision_model_resource_id: str | None = None
 
     model_resource_ids: dict[ModelType, str] = Field(default_factory=dict)
     knowledge_base_id: str | None = None

backend/services/config_resolver.py

@@ -58,6 +58,11 @@ async def resolve_runtime_config(
     stt_resource = await _resource_for(session, assistant.id, "ASR")
     tts_resource = await _resource_for(session, assistant.id, "TTS")
     realtime_resource = await _resource_for(session, assistant.id, "Realtime")
+    vision_resource = (
+        await session.get(ModelResource, assistant.vision_model_resource_id)
+        if assistant.vision_model_resource_id
+        else llm_resource
+    )
 
     return AssistantConfig(
         name=assistant.name,
@@ -89,6 +94,17 @@ async def resolve_runtime_config(
         llm_support_image_input=(
             bool(llm_resource.support_image_input) if llm_resource else False
         ),
+        vision_enabled=assistant.vision_enabled,
+        vision_model_resource_id=assistant.vision_model_resource_id,
+        vision_model=str(_value(vision_resource, "modelId", "")),
+        vision_llm_interface_type=(
+            vision_resource.interface_type if vision_resource else "openai-llm"
+        ),
+        vision_llm_values=(vision_resource.values or {}) if vision_resource else {},
+        vision_llm_secrets=(vision_resource.secrets or {}) if vision_resource else {},
+        vision_llm_support_image_input=(
+            bool(vision_resource.support_image_input) if vision_resource else False
+        ),
         stt_values=(stt_resource.values or {}) if stt_resource else {},
         stt_secrets=(stt_resource.secrets or {}) if stt_resource else {},
         tts_values=(tts_resource.values or {}) if tts_resource else {},
@@ -101,6 +117,12 @@ async def resolve_runtime_config(
         # 运行时连接信息(真 key + url):模型资源优先,否则 .env 兜底
         llm_api_key=_secret(llm_resource, "apiKey", config.LLM_API_KEY),
         llm_base_url=str(_value(llm_resource, "apiUrl", config.LLM_BASE_URL)),
+        vision_llm_api_key=_secret(
+            vision_resource, "apiKey", config.LLM_API_KEY
+        ),
+        vision_llm_base_url=str(
+            _value(vision_resource, "apiUrl", config.LLM_BASE_URL)
+        ),
         stt_api_key=_secret(stt_resource, "apiKey", config.STT_API_KEY),
         stt_base_url=str(_value(stt_resource, "apiUrl", config.STT_BASE_URL)),
         tts_api_key=_secret(tts_resource, "apiKey", config.TTS_API_KEY),

frontend/src/components/pages/AssistantPage.tsx

@@ -125,6 +125,8 @@ type AssistantForm = {
   voice: string;
   knowledgeBase: string;
   enableInterrupt: boolean;
+  visionEnabled: boolean;
+  visionModelResourceId: string;
 };
 
 type FastGptForm = {
@@ -155,6 +157,8 @@ type OpenCodeForm = {
   asr: string;
   voice: string;
   enableInterrupt: boolean;
+  visionEnabled: boolean;
+  visionModelResourceId: string;
 };
 
 type AssistantType = "提示词" | "工作流" | "Dify" | "FastGPT" | "OpenCode";
@@ -214,6 +218,8 @@ function blankPromptForm(name: string): AssistantForm {
     voice: "",
     knowledgeBase: "",
     enableInterrupt: true,
+    visionEnabled: false,
+    visionModelResourceId: "",
   };
 }
 
@@ -250,6 +256,8 @@ function blankOpenCodeForm(name: string): OpenCodeForm {
     asr: "",
     voice: "",
     enableInterrupt: true,
+    visionEnabled: false,
+    visionModelResourceId: "",
   };
 }
 function formatTimestamp(iso?: string | null): string {
@@ -331,8 +339,6 @@ export function AssistantPage(props: AssistantPageProps) {
   const editingId = props.mode === "edit" ? props.assistantId : null;
 
   const [form, setForm] = useState<AssistantForm>(() => blankPromptForm(""));
-  // 视觉理解(暂不持久化,不进 savedSnapshot,避免影响「未保存改动」判定)
-  const [visionEnabled, setVisionEnabled] = useState(false);
   const [fastGptForm, setFastGptForm] = useState<FastGptForm>(() =>
     blankFastGptForm(""),
   );
@@ -426,32 +432,39 @@ export function AssistantPage(props: AssistantPageProps) {
   // 按资源类型生成 {value:id, label:name} 选项
   const credOptions = (type: ModelResource["capability"]) =>
     modelResources
-      .filter((c) => {
+      .filter((c) => c.capability === type)
-        if (c.capability !== type) return false;
-        if (type === "LLM" && visionEnabled) return c.supportImageInput;
-        return true;
-      })
       .map((c) => ({ value: c.id, label: c.name }));
+  const visionModelOptionsFor = (currentModelId: string) => modelResources
+    .filter(
+      (c) =>
+        c.capability === "LLM" &&
+        c.supportImageInput &&
+        c.id !== currentModelId,
+    )
+    .map((c) => ({ value: c.id, label: c.name }));
   const kbOptions = knowledgeBases.map((k) => ({ value: k.id, label: k.name }));
 
-  function modelSupportsImageInput(resourceId: string): boolean {
+  function handlePromptVisionEnabledChange(enabled: boolean) {
-    return Boolean(
+    updateForm("visionEnabled", enabled);
-      modelResources.find((resource) => resource.id === resourceId)
+    if (!enabled) updateForm("visionModelResourceId", "");
-        ?.supportImageInput,
-    );
   }
 
-  function handleVisionEnabledChange(enabled: boolean) {
+  function handlePromptModelChange(value: string) {
-    setVisionEnabled(enabled);
+    updateForm("model", value);
-    if (enabled && form.model && !modelSupportsImageInput(form.model)) {
+    if (form.visionModelResourceId === value) {
-      updateForm("model", "");
+      updateForm("visionModelResourceId", "");
     }
-    if (
+  }
-      enabled &&
+
-      openCodeForm.model &&
+  function handleOpenCodeVisionEnabledChange(enabled: boolean) {
-      !modelSupportsImageInput(openCodeForm.model)
+    updateOpenCodeForm("visionEnabled", enabled);
-    ) {
+    if (!enabled) updateOpenCodeForm("visionModelResourceId", "");
-      updateOpenCodeForm("model", "");
+  }
+
+  function handleOpenCodeModelChange(value: string) {
+    updateOpenCodeForm("model", value);
+    if (openCodeForm.visionModelResourceId === value) {
+      updateOpenCodeForm("visionModelResourceId", "");
     }
   }
 
@@ -473,6 +486,8 @@ export function AssistantPage(props: AssistantPageProps) {
       voice: a.modelResourceIds.TTS ?? "",
       knowledgeBase: a.knowledgeBaseId ?? "",
       enableInterrupt: a.enableInterrupt,
+      visionEnabled: a.visionEnabled,
+      visionModelResourceId: a.visionModelResourceId ?? "",
     };
     setForm(next);
     return next;
@@ -533,6 +548,8 @@ export function AssistantPage(props: AssistantPageProps) {
       runtimeMode: "pipeline",
       greeting: "",
       enableInterrupt: true,
+      visionEnabled: false,
+      visionModelResourceId: null,
       modelResourceIds: {},
       knowledgeBaseId: null,
       prompt: "",
@@ -591,6 +608,8 @@ export function AssistantPage(props: AssistantPageProps) {
         runtimeMode: form.runtimeMode,
         greeting: form.greeting,
         enableInterrupt: form.enableInterrupt,
+        visionEnabled: form.visionEnabled,
+        visionModelResourceId: form.visionModelResourceId || null,
         modelResourceIds: {
           ...(form.model ? { LLM: form.model } : {}),
           ...(form.asr ? { ASR: form.asr } : {}),
@@ -679,6 +698,8 @@ export function AssistantPage(props: AssistantPageProps) {
       asr: a.modelResourceIds.ASR ?? "",
       voice: a.modelResourceIds.TTS ?? "",
       enableInterrupt: a.enableInterrupt,
+      visionEnabled: a.visionEnabled,
+      visionModelResourceId: a.visionModelResourceId ?? "",
     };
     setOpenCodeForm(next);
     return next;
@@ -748,6 +769,8 @@ export function AssistantPage(props: AssistantPageProps) {
         name: openCodeForm.name.trim(),
         type: "opencode",
         enableInterrupt: openCodeForm.enableInterrupt,
+        visionEnabled: openCodeForm.visionEnabled,
+        visionModelResourceId: openCodeForm.visionModelResourceId || null,
         modelResourceIds: {
           ...(openCodeForm.model ? { LLM: openCodeForm.model } : {}),
           ...(openCodeForm.asr ? { ASR: openCodeForm.asr } : {}),
@@ -1518,7 +1541,11 @@ export function AssistantPage(props: AssistantPageProps) {
             </SectionCard>
           </div>
 
-          <DebugDrawer assistantId={editingId} hasUnsavedChanges={dirty} />
+          <DebugDrawer
+            assistantId={editingId}
+            hasUnsavedChanges={dirty}
+            vision={openCodeForm.visionEnabled}
+          />
         </div>
       </div>
     );
@@ -1600,14 +1627,25 @@ export function AssistantPage(props: AssistantPageProps) {
             >
               <ToggleRow
                 title="视觉理解"
-                hint="开启后，大语言模型只能选择支持图片输入的资源；开始对话时会允许助手按需理解当前视频画面。"
+                hint="开启后，开始对话时会允许助手按需理解当前视频画面。视觉模型选「模型自己」时，大语言模型本身必须支持图片输入。"
-                checked={visionEnabled}
+                checked={openCodeForm.visionEnabled}
-                onChange={handleVisionEnabledChange}
+                onChange={handleOpenCodeVisionEnabledChange}
               />
+              {openCodeForm.visionEnabled && (
+                <ResourceSelectField
+                  label="视觉模型"
+                  value={openCodeForm.visionModelResourceId}
+                  onChange={(value) =>
+                    updateOpenCodeForm("visionModelResourceId", value)
+                  }
+                  options={visionModelOptionsFor(openCodeForm.model)}
+                  noneLabel="模型自己"
+                />
+              )}
               <ResourceSelectField
                 label="大语言模型"
                 value={openCodeForm.model}
-                onChange={(value) => updateOpenCodeForm("model", value)}
+                onChange={handleOpenCodeModelChange}
                 options={credOptions("LLM")}
                 noneLabel="无"
               />
@@ -1779,14 +1817,25 @@ export function AssistantPage(props: AssistantPageProps) {
             >
               <ToggleRow
                 title="视觉理解"
-                hint="开启后，大语言模型只能选择支持图片输入的资源；开始对话时会允许助手按需理解当前视频画面。"
+                hint="开启后，开始对话时会允许助手按需理解当前视频画面。视觉模型选「模型自己」时，大语言模型本身必须支持图片输入。"
-                checked={visionEnabled}
+                checked={form.visionEnabled}
-                onChange={handleVisionEnabledChange}
+                onChange={handlePromptVisionEnabledChange}
               />
+              {form.visionEnabled && (
+                <ResourceSelectField
+                  label="视觉模型"
+                  value={form.visionModelResourceId}
+                  onChange={(value) =>
+                    updateForm("visionModelResourceId", value)
+                  }
+                  options={visionModelOptionsFor(form.model)}
+                  noneLabel="模型自己"
+                />
+              )}
               <ResourceSelectField
                 label="大语言模型"
                 value={form.model}
-                onChange={(value) => updateForm("model", value)}
+                onChange={handlePromptModelChange}
                 options={credOptions("LLM")}
                 noneLabel="无"
               />
@@ -1863,7 +1912,7 @@ export function AssistantPage(props: AssistantPageProps) {
         <DebugDrawer
           assistantId={editingId}
           hasUnsavedChanges={dirty}
-          vision={visionEnabled}
+          vision={form.visionEnabled}
         />
       </div>
     </div>

frontend/src/lib/api.ts

@@ -128,6 +128,8 @@ export type Assistant = {
   runtimeMode: RuntimeMode;
   greeting: string;
   enableInterrupt: boolean;
+  visionEnabled: boolean;
+  visionModelResourceId: string | null;
   modelResourceIds: Partial<Record<ModelType, string>>;
   knowledgeBaseId: string | null;
   prompt: string;