feat: add realtime workflow vision tools

backend/services/workflow/realtime.py

@@ -28,6 +28,12 @@ from services.runtime_variables import DynamicVariableError, DynamicVariableStor
 from services.system_tools import state_update_properties, system_tool_kind
 from services.tool_executor import ToolExecutionError, ToolExecutor
 from services.tool_policy import policy_for_tool
+from services.vision import (
+    VISION_SYSTEM_HINT,
+    VISION_TOOL_NAME,
+    analyze_image_with_vision_model,
+    config_with_vision_resource,
+)
 from services.workflow.agent import EDGE_TOOL_STAGE_INSTRUCTION
 from services.workflow.models import WorkflowRuntimeState, WorkflowStatus
 from services.workflow.output import WorkflowOutput
@@ -385,10 +391,13 @@ class WorkflowRealtimeController:
         return RealtimeActivation(continue_response=generate)
 
     def _agent_prompt(self, node_id: str) -> str:
-        return (
+        prompt = (
             f"{self._engine.prompt_for(node_id, self._store)}\n\n"
             f"[工作流执行规则]\n{EDGE_TOOL_STAGE_INSTRUCTION}"
         )
+        if self._engine.agent_stage_config(node_id).vision_enabled:
+            prompt = f"{prompt}\n{VISION_SYSTEM_HINT}"
+        return prompt
 
     def _build_agent_tools(self, node_id: str) -> list[RealtimeTool]:
         stage = self._engine.agent_stage_config(node_id)
@@ -428,6 +437,9 @@ class WorkflowRealtimeController:
         knowledge = self._knowledge_tool(node_id, transition_id)
         if knowledge:
             add(*knowledge)
+        vision = self._vision_tool(node_id, transition_id)
+        if vision:
+            add(*vision)
         for edge in self._engine.edge_tool_edges(node_id):
             add(*self._transition_tool(edge, node_id, transition_id))
         self._handlers = handlers
@@ -519,6 +531,105 @@ class WorkflowRealtimeController:
             handler,
         )
 
+    def _vision_tool(
+        self,
+        node_id: str,
+        transition_id: int,
+    ) -> tuple[RealtimeTool, ToolHandler] | None:
+        """Build the on-demand camera tool for the active Realtime Agent."""
+        stage = self._engine.agent_stage_config(node_id)
+        if not stage.vision_enabled:
+            return None
+
+        vision_cfg: AssistantConfig | None = None
+        if stage.vision_model_resource_id:
+            resource = self._cfg.workflow_model_resources.get(
+                stage.vision_model_resource_id
+            )
+            if resource is not None:
+                vision_cfg = config_with_vision_resource(self._cfg, resource)
+
+        async def handler(arguments: dict[str, Any]) -> RealtimeToolResult:
+            if not self._is_current(node_id, transition_id):
+                return RealtimeToolResult(
+                    {"status": "stale", "message": "当前 Agent 已经切换。"},
+                    continue_response=False,
+                )
+            question = str(arguments.get("question") or "").strip()
+            if not question:
+                return RealtimeToolResult(
+                    {"status": "error", "message": "视觉问题为空"}
+                )
+            if self._runtime.capture_image is None:
+                return RealtimeToolResult(
+                    {"status": "no_video", "message": "当前会话未启用视频输入"}
+                )
+            try:
+                frame = await self._runtime.capture_image(question)
+            except TimeoutError:
+                return RealtimeToolResult(
+                    {"status": "timeout", "message": "等待摄像头画面超时"}
+                )
+            except Exception as exc:  # noqa: BLE001 - expose a stable tool error
+                logger.warning(f"Realtime Workflow 获取摄像头画面失败:{exc}")
+                return RealtimeToolResult(
+                    {"status": "error", "message": "暂时无法获取摄像头画面"}
+                )
+            if frame is None:
+                return RealtimeToolResult(
+                    {"status": "no_video", "message": "当前没有可用的视频客户端"}
+                )
+            if not self._is_current(node_id, transition_id):
+                return RealtimeToolResult(
+                    {"status": "stale", "message": "获取画面时 Agent 已经切换。"},
+                    continue_response=False,
+                )
+
+            try:
+                if vision_cfg is not None:
+                    observation = await analyze_image_with_vision_model(
+                        vision_cfg,
+                        frame,
+                        question,
+                    )
+                else:
+                    analyze = getattr(self._runtime.realtime, "analyze_image", None)
+                    if not callable(analyze):
+                        raise ValueError("当前 Realtime 适配器未实现原生视觉理解")
+                    observation = await analyze(frame, question)
+            except Exception as exc:  # noqa: BLE001 - provider errors become tool output
+                logger.warning(f"Realtime Workflow 视觉模型调用失败:{exc}")
+                return RealtimeToolResult(
+                    {"status": "error", "message": "视觉模型暂时不可用"}
+                )
+            if not self._is_current(node_id, transition_id):
+                return RealtimeToolResult(
+                    {"status": "stale", "message": "分析画面时 Agent 已经切换。"},
+                    continue_response=False,
+                )
+            return RealtimeToolResult(
+                {
+                    "status": "ok",
+                    "question": question,
+                    "observation": observation,
+                }
+            )
+
+        return (
+            RealtimeTool(
+                name=VISION_TOOL_NAME,
+                description="获取用户摄像头的当前画面，并回答一个视觉问题。",
+                properties={
+                    "question": {
+                        "type": "string",
+                        "description": "需要根据当前画面判断的具体问题",
+                    }
+                },
+                required=("question",),
+            ),
+            handler,
+        )
+
     def _system_tool(
         self,
         tool: RuntimeTool,