feat(workflow): add per-agent vision configuration

backend/services/pipecat/pipeline.py

@@ -29,6 +29,14 @@ from services.pipecat.service_factory import (
 )
 from db.session import SessionLocal
 from services.knowledge import search as search_knowledge
+from services.vision import (
+    VISION_ANALYSIS_SYSTEM_PROMPT,
+    VISION_SYSTEM_HINT,
+    VISION_TOOL_NAME,
+    config_with_main_llm_as_vision,
+    config_with_vision_resource,
+)
+from services.workflow_engine import WorkflowEngine
 
 from pipecat.adapters.schemas.function_schema import FunctionSchema
 from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -82,15 +90,6 @@ from services.pipecat.pipeline_events import (
 from pipecat.workers.runner import WorkerRunner
 
 
-VISION_TOOL_NAME = "fetch_user_image"
-VISION_SYSTEM_HINT = (
-    "当前会话打开了视觉理解。用户询问当前画面、摄像头里有什么、人物/物品/"
-    "环境状态或需要你看一眼时,调用 fetch_user_image 获取当前视频帧,再基于画面回答。"
-)
-VISION_ANALYSIS_SYSTEM_PROMPT = (
-    "你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。"
-    "如果画面不足以判断,请明确说明不确定。"
-)
 KNOWLEDGE_TOOL_NAME = "search_knowledge_base"
 AUTOMATIC_KNOWLEDGE_SYSTEM_HINT = (
     "你已连接内部知识库。系统会在每轮用户问题前自动提供相关资料；"
@@ -205,6 +204,9 @@ async def run_pipeline(
                    只要有 .input() / .output() / event_handler 即可。
         cfg:       助手配置(随请求内联传入)。
     """
+    if cfg.type == "workflow":
+        vision_enabled = WorkflowEngine(cfg.graph).uses_vision()
+
     logger.info(
         f"启动管线: assistant={cfg.name} type={cfg.type} "
         f"mode={cfg.runtimeMode} vision={vision_enabled}"
@@ -298,7 +300,7 @@ async def run_pipeline(
 
     def with_vision_hint(text: str) -> str:
         hints = []
-        if vision_enabled:
+        if vision_enabled and cfg.type != "workflow":
             hints.append(VISION_SYSTEM_HINT)
         if cfg.knowledge_base_id:
             hints.append(
@@ -354,8 +356,17 @@ async def run_pipeline(
         top_n=knowledge_top_n,
         score_threshold=knowledge_score_threshold,
     )
-    vision_native_mode = vision_enabled and _vision_uses_main_llm(cfg)
+    vision_native_mode = (
+        vision_enabled
+        and cfg.type != "workflow"
+        and _vision_uses_main_llm(cfg)
+    )
     vision_state: dict[str, str | None] = {"client_id": None}
+    workflow_vision_scope: dict[str, Any] = {
+        "enabled": False,
+        "vision_model_resource_id": None,
+        "llm_resource_id": None,
+    }
     vision_schema = FunctionSchema(
         name=VISION_TOOL_NAME,
         description=(
@@ -458,8 +469,14 @@ async def run_pipeline(
         )
 
     flow_global_functions = []
+    workflow_vision_function = None
     if cfg.type == "workflow" and vision_enabled:
         async def flow_fetch_user_image(args, _flow_manager):
+            if not workflow_vision_scope.get("enabled"):
+                return {
+                    "status": "disabled",
+                    "message": "当前 Agent 节点未开启视觉理解。",
+                }
             question = str((args or {}).get("question") or "请描述当前画面。")
             user_id = vision_state.get("client_id")
             if not user_id:
@@ -474,8 +491,26 @@ async def run_pipeline(
                 function_name=VISION_TOOL_NAME,
             )
             try:
+                vision_resource_id = str(
+                    workflow_vision_scope.get("vision_model_resource_id") or ""
+                )
+                llm_resource_id = str(
+                    workflow_vision_scope.get("llm_resource_id") or ""
+                )
+                resource_id = vision_resource_id or llm_resource_id
+                resource = cfg.workflow_model_resources.get(resource_id)
+                if resource:
+                    vision_cfg = config_with_vision_resource(cfg, resource)
+                elif not vision_resource_id:
+                    vision_cfg = config_with_main_llm_as_vision(cfg)
+                else:
+                    raise ValueError(f"视觉模型资源未加载:{vision_resource_id}")
                 frame = await vision_capture.request_image(llm, request)
-                observation = await _analyze_image_with_vision_model(cfg, frame, question)
+                observation = await _analyze_image_with_vision_model(
+                    vision_cfg,
+                    frame,
+                    question,
+                )
                 return {
                     "status": "ok",
                     "question": question,
@@ -487,15 +522,13 @@ async def run_pipeline(
                 logger.warning(f"Workflow 视觉理解失败:{exc}")
                 return {"status": "error", "message": "视觉理解暂时不可用。"}
 
-        flow_global_functions.append(
-            FlowsFunctionSchema(
-                name=VISION_TOOL_NAME,
-                description=vision_schema.description,
-                properties=vision_schema.properties,
-                required=vision_schema.required,
-                handler=flow_fetch_user_image,
-                cancel_on_interruption=True,
-            )
+        workflow_vision_function = FlowsFunctionSchema(
+            name=VISION_TOOL_NAME,
+            description=vision_schema.description,
+            properties=vision_schema.properties,
+            required=vision_schema.required,
+            handler=flow_fetch_user_image,
+            cancel_on_interruption=True,
         )
 
     if vision_enabled and cfg.type != "workflow":
@@ -612,6 +645,8 @@ async def run_pipeline(
             transport=transport,
             switch_services=service_controller.switch,
             set_knowledge_scope=knowledge_retrieval.set_scope,
+            set_vision_scope=lambda scope: workflow_vision_scope.update(scope),
+            vision_function=workflow_vision_function,
             set_input_enabled=lambda enabled: input_state.__setitem__("enabled", enabled),
             apply_turn_config=apply_workflow_turn_config,
             flow_global_functions=flow_global_functions,