diff --git a/Makefile b/Makefile index 8438d55..dcbcbe1 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ PSQL = docker compose exec -T postgres psql -U postgres -d postgres .DEFAULT_GOAL := help -.PHONY: help up down restart logs api-logs db db-list db-seed db-seed-model-resources db-seed-assistants db-clear db-reset +.PHONY: help up down restart logs api-logs db db-list db-init db-seed db-seed-model-resources db-seed-assistants db-clear db-reset help: ## 列出所有可用目标 @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ @@ -33,6 +33,11 @@ api-logs: ## 只看后端日志 db: ## 进入交互式 psql docker compose exec postgres psql -U postgres -d postgres +db-init: ## 启动 API 并等待建表完成(表由 init_db/create_all 在 API 启动时创建) + docker compose up -d postgres api + @echo "Waiting for API health (schema init)..." + @until docker compose exec -T api python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')" 2>/dev/null; do sleep 1; done + db-list: ## 列出模型资源与助手 @$(PSQL) -c "SELECT id, name, capability, interface_type, is_default FROM model_resources ORDER BY id;" @$(PSQL) -c "SELECT id, name, type FROM assistants ORDER BY id;" @@ -43,9 +48,9 @@ db-seed-model-resources: ## 灌入 12 条模型资源种子(幂等) db-seed-assistants: ## 灌入 知识库 + 助手 种子(幂等;依赖模型资源已就绪) $(PSQL) < backend/db/seed_assistants.sql -db-seed: db-seed-model-resources db-seed-assistants ## 全量灌种子(模型资源→知识库→助手,幂等,可重复执行) +db-seed: db-init db-seed-model-resources db-seed-assistants ## 全量灌种子(模型资源→知识库→助手,幂等,可重复执行) -db-clear: ## 清空 助手/知识库/模型资源(按依赖顺序) - $(PSQL) -c "TRUNCATE assistant_model_bindings, assistants, knowledge_bases, model_resources CASCADE;" +db-clear: ## 清空 助手/知识库/模型资源(按依赖顺序;表不存在时跳过) + @$(PSQL) -c "DO \$$\$$ BEGIN IF EXISTS (SELECT FROM pg_tables WHERE schemaname = 'public' AND tablename = 'model_resources') THEN TRUNCATE assistant_model_bindings, assistants, knowledge_bases, model_resources CASCADE; END IF; END \$$\$$;" -db-reset: db-clear db-seed ## 清空后重新灌全部种子 +db-reset: db-init db-clear db-seed ## 清空后重新灌全部种子(空库时仅建表+灌种) diff --git a/backend/db/models.py b/backend/db/models.py index cea76a2..ff5e2ba 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -97,6 +97,12 @@ class Assistant(Base): runtime_mode: Mapped[str] = mapped_column(String(16), default="pipeline") greeting: Mapped[str] = mapped_column(String(2048), default="") enable_interrupt: Mapped[bool] = mapped_column(Boolean, default=True) + vision_enabled: Mapped[bool] = mapped_column(Boolean, default=False) + vision_model_resource_id: Mapped[str | None] = mapped_column( + String(40), + ForeignKey("model_resources.id", ondelete="SET NULL"), + nullable=True, + ) # KB 引用:被引用时禁止删 KB(RESTRICT),无默认兜底 knowledge_base_id: Mapped[str | None] = mapped_column( diff --git a/backend/db/seed_assistants.sql b/backend/db/seed_assistants.sql index b619dcc..14bac38 100644 --- a/backend/db/seed_assistants.sql +++ b/backend/db/seed_assistants.sql @@ -8,18 +8,24 @@ ON CONFLICT (id) DO NOTHING; INSERT INTO assistants ( id, name, type, runtime_mode, greeting, enable_interrupt, + vision_enabled, vision_model_resource_id, knowledge_base_id, prompt, api_url, api_key, app_id, graph ) VALUES ('asst_001', '政务咨询助手', 'prompt', 'pipeline', '您好,我是政务助手,请问有什么可以帮您?', TRUE, + FALSE, NULL, 'kb_001', '你是一名专业的政务咨询助手,回答准确、简洁,不编造政策内容。', '', '', '', '{}'), ('asst_002', '热线工单助手', 'workflow', 'pipeline', '', TRUE, + FALSE, NULL, NULL, '', '', '', '', '{"nodes":[{"id":"1","type":"startCall","position":{"x":0,"y":0},"data":{"name":"开场","prompt":"你好,请问需要办理什么业务?"}}],"edges":[]}'), ('asst_003', 'Dify 客服助手', 'dify', 'pipeline', '', TRUE, + FALSE, NULL, NULL, '', 'https://api.dify.ai/v1', 'app-dify-demo-key', '', '{}'), ('asst_004', 'FastGPT 售后助手', 'fastgpt', 'pipeline', '', TRUE, + FALSE, NULL, NULL, '', 'https://api.fastgpt.in/api/v1/chat/completions', 'fastgpt-demo-key', 'app-fastgpt-001', '{}'), ('asst_005', 'OpenCode 代码助手', 'opencode', 'pipeline', '', TRUE, + FALSE, NULL, NULL, '你是一个代码助手的语音界面,用简洁口语回答工程问题。', 'http://localhost:4096', 'opencode-demo-key', '', '{}') ON CONFLICT (id) DO NOTHING; diff --git a/backend/models.py b/backend/models.py index f648e8f..af80213 100644 --- a/backend/models.py +++ b/backend/models.py @@ -42,6 +42,15 @@ class AssistantConfig(BaseModel): llm_values: dict = {} llm_secrets: dict = {} llm_support_image_input: bool = False + vision_enabled: bool = False + vision_model_resource_id: str | None = None + vision_model: str = "" + vision_llm_interface_type: str = "openai-llm" + vision_llm_values: dict = {} + vision_llm_secrets: dict = {} + vision_llm_support_image_input: bool = False + vision_llm_api_key: str = "" + vision_llm_base_url: str = "" stt_values: dict = {} stt_secrets: dict = {} tts_values: dict = {} diff --git a/backend/routes/assistants.py b/backend/routes/assistants.py index ee82502..0ac23d8 100644 --- a/backend/routes/assistants.py +++ b/backend/routes/assistants.py @@ -24,6 +24,23 @@ def _validate_workflow(body: AssistantUpsert) -> None: raise HTTPException(400, "工作流校验失败:" + ";".join(errors)) +async def _validate_vision_model( + session: AsyncSession, body: AssistantUpsert +) -> None: + if body.vision_enabled: + if body.vision_model_resource_id: + resource = await session.get(ModelResource, body.vision_model_resource_id) + else: + resource_id = body.model_resource_ids.get("LLM") + resource = ( + await session.get(ModelResource, resource_id) if resource_id else None + ) + if not resource or resource.capability != "LLM": + raise HTTPException(400, "视觉模型必须引用 LLM 模型资源") + if not resource.support_image_input: + raise HTTPException(400, "视觉模型必须支持图片输入") + + async def _sync_bindings( session: AsyncSession, assistant_id: str, resource_ids: dict[str, str] ) -> None: @@ -69,6 +86,8 @@ async def _to_out(session: AsyncSession, assistant: Assistant) -> AssistantOut: runtime_mode=assistant.runtime_mode, # type: ignore[arg-type] greeting=assistant.greeting, enable_interrupt=assistant.enable_interrupt, + vision_enabled=assistant.vision_enabled, + vision_model_resource_id=assistant.vision_model_resource_id, model_resource_ids=await _resource_ids(session, assistant.id), knowledge_base_id=assistant.knowledge_base_id, prompt=assistant.prompt, @@ -93,6 +112,7 @@ async def create_assistant( body: AssistantUpsert, session: AsyncSession = Depends(get_session) ): _validate_workflow(body) + await _validate_vision_model(session, body) data = body.model_dump() resource_ids = data.pop("model_resource_ids") assistant = Assistant(id=f"asst_{uuid.uuid4().hex[:12]}", **data) @@ -128,6 +148,8 @@ async def duplicate_assistant( runtime_mode=source.runtime_mode, greeting=source.greeting, enable_interrupt=source.enable_interrupt, + vision_enabled=source.vision_enabled, + vision_model_resource_id=source.vision_model_resource_id, knowledge_base_id=source.knowledge_base_id, prompt=source.prompt, api_url=source.api_url, @@ -153,6 +175,7 @@ async def update_assistant( if not assistant: raise HTTPException(404, "助手不存在") _validate_workflow(body) + await _validate_vision_model(session, body) data = body.model_dump() resource_ids = data.pop("model_resource_ids") data["api_key"] = resolve_incoming_key(data["api_key"], assistant.api_key) diff --git a/backend/routes/voice_webrtc.py b/backend/routes/voice_webrtc.py index f947f75..b2b03e3 100644 --- a/backend/routes/voice_webrtc.py +++ b/backend/routes/voice_webrtc.py @@ -74,6 +74,16 @@ async def _resolve_config(offer: SignalingOffer) -> AssistantConfig: raise ValueError("offer 缺少 assistant_id 或 inline_config") +def _apply_vision_model(cfg: AssistantConfig) -> None: + cfg.model = cfg.vision_model + cfg.llm_interface_type = cfg.vision_llm_interface_type + cfg.llm_values = cfg.vision_llm_values + cfg.llm_secrets = cfg.vision_llm_secrets + cfg.llm_support_image_input = cfg.vision_llm_support_image_input + cfg.llm_api_key = cfg.vision_llm_api_key + cfg.llm_base_url = cfg.vision_llm_base_url + + async def _handle_offer(websocket, payload, peers): from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection from services.pipecat.pipeline import run_pipeline @@ -87,8 +97,13 @@ async def _handle_offer(websocket, payload, peers): await pc.renegotiate(sdp=offer.sdp, type=offer.type, restart_pc=False) else: cfg = await _resolve_config(offer) # 解析放在建连前,配置错就别建连 - if offer.vision_enabled and not cfg.llm_support_image_input: - raise ValueError("当前大语言模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM") + vision_enabled = offer.vision_enabled or cfg.vision_enabled + if vision_enabled: + if not cfg.vision_llm_support_image_input: + raise ValueError( + "当前视觉模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM" + ) + _apply_vision_model(cfg) pc = SmallWebRTCConnection(ice_servers=aiortc_ice_servers()) if pc_id: pc._pc_id = pc_id @@ -102,10 +117,10 @@ async def _handle_offer(websocket, payload, peers): # 后台跑管线:WebRTC transport + 解析出的运行时配置 transport = build_webrtc_transport( pc, - video_in_enabled=offer.vision_enabled, + video_in_enabled=vision_enabled, ) asyncio.create_task( - run_pipeline(transport, cfg, vision_enabled=offer.vision_enabled) + run_pipeline(transport, cfg, vision_enabled=vision_enabled) ) answer = pc.get_answer() diff --git a/backend/schemas.py b/backend/schemas.py index ff29562..2a01634 100644 --- a/backend/schemas.py +++ b/backend/schemas.py @@ -52,6 +52,8 @@ class AssistantUpsert(CamelModel): runtime_mode: RuntimeMode = "pipeline" greeting: str = "" enable_interrupt: bool = True + vision_enabled: bool = False + vision_model_resource_id: str | None = None model_resource_ids: dict[ModelType, str] = Field(default_factory=dict) knowledge_base_id: str | None = None diff --git a/backend/services/config_resolver.py b/backend/services/config_resolver.py index 1ab155a..f115e00 100644 --- a/backend/services/config_resolver.py +++ b/backend/services/config_resolver.py @@ -58,6 +58,11 @@ async def resolve_runtime_config( stt_resource = await _resource_for(session, assistant.id, "ASR") tts_resource = await _resource_for(session, assistant.id, "TTS") realtime_resource = await _resource_for(session, assistant.id, "Realtime") + vision_resource = ( + await session.get(ModelResource, assistant.vision_model_resource_id) + if assistant.vision_model_resource_id + else llm_resource + ) return AssistantConfig( name=assistant.name, @@ -89,6 +94,17 @@ async def resolve_runtime_config( llm_support_image_input=( bool(llm_resource.support_image_input) if llm_resource else False ), + vision_enabled=assistant.vision_enabled, + vision_model_resource_id=assistant.vision_model_resource_id, + vision_model=str(_value(vision_resource, "modelId", "")), + vision_llm_interface_type=( + vision_resource.interface_type if vision_resource else "openai-llm" + ), + vision_llm_values=(vision_resource.values or {}) if vision_resource else {}, + vision_llm_secrets=(vision_resource.secrets or {}) if vision_resource else {}, + vision_llm_support_image_input=( + bool(vision_resource.support_image_input) if vision_resource else False + ), stt_values=(stt_resource.values or {}) if stt_resource else {}, stt_secrets=(stt_resource.secrets or {}) if stt_resource else {}, tts_values=(tts_resource.values or {}) if tts_resource else {}, @@ -101,6 +117,12 @@ async def resolve_runtime_config( # 运行时连接信息(真 key + url):模型资源优先,否则 .env 兜底 llm_api_key=_secret(llm_resource, "apiKey", config.LLM_API_KEY), llm_base_url=str(_value(llm_resource, "apiUrl", config.LLM_BASE_URL)), + vision_llm_api_key=_secret( + vision_resource, "apiKey", config.LLM_API_KEY + ), + vision_llm_base_url=str( + _value(vision_resource, "apiUrl", config.LLM_BASE_URL) + ), stt_api_key=_secret(stt_resource, "apiKey", config.STT_API_KEY), stt_base_url=str(_value(stt_resource, "apiUrl", config.STT_BASE_URL)), tts_api_key=_secret(tts_resource, "apiKey", config.TTS_API_KEY), diff --git a/frontend/src/components/pages/AssistantPage.tsx b/frontend/src/components/pages/AssistantPage.tsx index 8f6f4ac..decc93d 100644 --- a/frontend/src/components/pages/AssistantPage.tsx +++ b/frontend/src/components/pages/AssistantPage.tsx @@ -125,6 +125,8 @@ type AssistantForm = { voice: string; knowledgeBase: string; enableInterrupt: boolean; + visionEnabled: boolean; + visionModelResourceId: string; }; type FastGptForm = { @@ -155,6 +157,8 @@ type OpenCodeForm = { asr: string; voice: string; enableInterrupt: boolean; + visionEnabled: boolean; + visionModelResourceId: string; }; type AssistantType = "提示词" | "工作流" | "Dify" | "FastGPT" | "OpenCode"; @@ -214,6 +218,8 @@ function blankPromptForm(name: string): AssistantForm { voice: "", knowledgeBase: "", enableInterrupt: true, + visionEnabled: false, + visionModelResourceId: "", }; } @@ -250,6 +256,8 @@ function blankOpenCodeForm(name: string): OpenCodeForm { asr: "", voice: "", enableInterrupt: true, + visionEnabled: false, + visionModelResourceId: "", }; } function formatTimestamp(iso?: string | null): string { @@ -331,8 +339,6 @@ export function AssistantPage(props: AssistantPageProps) { const editingId = props.mode === "edit" ? props.assistantId : null; const [form, setForm] = useState(() => blankPromptForm("")); - // 视觉理解(暂不持久化,不进 savedSnapshot,避免影响「未保存改动」判定) - const [visionEnabled, setVisionEnabled] = useState(false); const [fastGptForm, setFastGptForm] = useState(() => blankFastGptForm(""), ); @@ -426,32 +432,39 @@ export function AssistantPage(props: AssistantPageProps) { // 按资源类型生成 {value:id, label:name} 选项 const credOptions = (type: ModelResource["capability"]) => modelResources - .filter((c) => { - if (c.capability !== type) return false; - if (type === "LLM" && visionEnabled) return c.supportImageInput; - return true; - }) + .filter((c) => c.capability === type) .map((c) => ({ value: c.id, label: c.name })); + const visionModelOptionsFor = (currentModelId: string) => modelResources + .filter( + (c) => + c.capability === "LLM" && + c.supportImageInput && + c.id !== currentModelId, + ) + .map((c) => ({ value: c.id, label: c.name })); const kbOptions = knowledgeBases.map((k) => ({ value: k.id, label: k.name })); - function modelSupportsImageInput(resourceId: string): boolean { - return Boolean( - modelResources.find((resource) => resource.id === resourceId) - ?.supportImageInput, - ); + function handlePromptVisionEnabledChange(enabled: boolean) { + updateForm("visionEnabled", enabled); + if (!enabled) updateForm("visionModelResourceId", ""); } - function handleVisionEnabledChange(enabled: boolean) { - setVisionEnabled(enabled); - if (enabled && form.model && !modelSupportsImageInput(form.model)) { - updateForm("model", ""); + function handlePromptModelChange(value: string) { + updateForm("model", value); + if (form.visionModelResourceId === value) { + updateForm("visionModelResourceId", ""); } - if ( - enabled && - openCodeForm.model && - !modelSupportsImageInput(openCodeForm.model) - ) { - updateOpenCodeForm("model", ""); + } + + function handleOpenCodeVisionEnabledChange(enabled: boolean) { + updateOpenCodeForm("visionEnabled", enabled); + if (!enabled) updateOpenCodeForm("visionModelResourceId", ""); + } + + function handleOpenCodeModelChange(value: string) { + updateOpenCodeForm("model", value); + if (openCodeForm.visionModelResourceId === value) { + updateOpenCodeForm("visionModelResourceId", ""); } } @@ -473,6 +486,8 @@ export function AssistantPage(props: AssistantPageProps) { voice: a.modelResourceIds.TTS ?? "", knowledgeBase: a.knowledgeBaseId ?? "", enableInterrupt: a.enableInterrupt, + visionEnabled: a.visionEnabled, + visionModelResourceId: a.visionModelResourceId ?? "", }; setForm(next); return next; @@ -533,6 +548,8 @@ export function AssistantPage(props: AssistantPageProps) { runtimeMode: "pipeline", greeting: "", enableInterrupt: true, + visionEnabled: false, + visionModelResourceId: null, modelResourceIds: {}, knowledgeBaseId: null, prompt: "", @@ -591,6 +608,8 @@ export function AssistantPage(props: AssistantPageProps) { runtimeMode: form.runtimeMode, greeting: form.greeting, enableInterrupt: form.enableInterrupt, + visionEnabled: form.visionEnabled, + visionModelResourceId: form.visionModelResourceId || null, modelResourceIds: { ...(form.model ? { LLM: form.model } : {}), ...(form.asr ? { ASR: form.asr } : {}), @@ -679,6 +698,8 @@ export function AssistantPage(props: AssistantPageProps) { asr: a.modelResourceIds.ASR ?? "", voice: a.modelResourceIds.TTS ?? "", enableInterrupt: a.enableInterrupt, + visionEnabled: a.visionEnabled, + visionModelResourceId: a.visionModelResourceId ?? "", }; setOpenCodeForm(next); return next; @@ -748,6 +769,8 @@ export function AssistantPage(props: AssistantPageProps) { name: openCodeForm.name.trim(), type: "opencode", enableInterrupt: openCodeForm.enableInterrupt, + visionEnabled: openCodeForm.visionEnabled, + visionModelResourceId: openCodeForm.visionModelResourceId || null, modelResourceIds: { ...(openCodeForm.model ? { LLM: openCodeForm.model } : {}), ...(openCodeForm.asr ? { ASR: openCodeForm.asr } : {}), @@ -1518,7 +1541,11 @@ export function AssistantPage(props: AssistantPageProps) { - + ); @@ -1600,14 +1627,25 @@ export function AssistantPage(props: AssistantPageProps) { > + {openCodeForm.visionEnabled && ( + + updateOpenCodeForm("visionModelResourceId", value) + } + options={visionModelOptionsFor(openCodeForm.model)} + noneLabel="模型自己" + /> + )} updateOpenCodeForm("model", value)} + onChange={handleOpenCodeModelChange} options={credOptions("LLM")} noneLabel="无" /> @@ -1779,14 +1817,25 @@ export function AssistantPage(props: AssistantPageProps) { > + {form.visionEnabled && ( + + updateForm("visionModelResourceId", value) + } + options={visionModelOptionsFor(form.model)} + noneLabel="模型自己" + /> + )} updateForm("model", value)} + onChange={handlePromptModelChange} options={credOptions("LLM")} noneLabel="无" /> @@ -1863,7 +1912,7 @@ export function AssistantPage(props: AssistantPageProps) { diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 957fab0..58394bf 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -128,6 +128,8 @@ export type Assistant = { runtimeMode: RuntimeMode; greeting: string; enableInterrupt: boolean; + visionEnabled: boolean; + visionModelResourceId: string | null; modelResourceIds: Partial>; knowledgeBaseId: string | null; prompt: string;