feat: add prompt startup actions and shared vision config

2026-08-01 23:31:14 +08:00
parent b747144ff1
commit 0331f8cd07
22 changed files with 1238 additions and 344 deletions
--- a/backend/services/brains/base.py
+++ b/backend/services/brains/base.py
@@ -118,6 +118,9 @@ class BaseBrain:
    async def setup(self, cfg: AssistantConfig, runtime: BrainRuntime) -> None:
        """Register tools and initialize per-call orchestration."""

+    async def run_preflight(self) -> None:
+        """Run deterministic server-side startup work before media starts."""
+
    async def on_connected(self, *, greeting_pending: bool = False) -> None:
        """Handle a connected client before an optional greeting is played.

@@ -198,6 +201,8 @@ class Brain(Protocol):

    async def setup(self, cfg: AssistantConfig, runtime: BrainRuntime) -> None: ...

+    async def run_preflight(self) -> None: ...
+
    async def on_connected(self, *, greeting_pending: bool = False) -> None: ...

    async def on_greeting_finished(self) -> None: ...
--- a/backend/services/brains/prompt_brain.py
+++ b/backend/services/brains/prompt_brain.py
@@ -2,9 +2,11 @@

 from __future__ import annotations

+import asyncio
 from typing import Any
 from uuid import uuid4

+from loguru import logger
 from models import AssistantConfig
 from pipecat.adapters.schemas.function_schema import FunctionSchema
 from pipecat.frames.frames import OutputTransportMessageUrgentFrame, TTSSpeakFrame
@@ -22,11 +24,19 @@ from services.brains.base import (
    BrainSpec,
    SessionVariableUpdate,
 )
+from services.action_runtime import (
+    ActionInvocationCancelled,
+    ActionRunner,
+    ActionStatus,
+)
 from services.runtime_variables import DynamicVariableStore
 from services.tool_executor import ToolExecutionError, ToolExecutor
 from services.tool_policy import policy_for_tool


+PREFLIGHT_TIMEOUT_SECONDS = 30
+
+
 class PromptBrain(BaseBrain):
    spec = BrainSpec(
        type="prompt",
@@ -39,8 +49,15 @@ class PromptBrain(BaseBrain):
        self._dynamic_enabled = True
        self._store = DynamicVariableStore.from_config(cfg)
        self._tools = ToolExecutor(self._store)
+        self._actions = ActionRunner(self._tools)
+        self._tool_by_id = {tool.id: tool for tool in cfg.tools}
        self._runtime: BrainRuntime | None = None
        self._waiting_for_generated_end_speech = False
+        self._greeting_finished = True
+        self._preflight_finished = False
+        self._opening_started = False
+        self._opening_finished = False
+        self._startup_failed = False

    async def greeting(self, cfg: AssistantConfig) -> str:
        return self._store.render(cfg.greeting) if self._dynamic_enabled else cfg.greeting
@@ -56,7 +73,17 @@ class PromptBrain(BaseBrain):
    async def setup(self, cfg: AssistantConfig, runtime: BrainRuntime) -> None:
        self._runtime = runtime
        self._tools.set_client_tools(runtime.client_tools)
+        self._actions = ActionRunner(
+            self._tools,
+            is_session_ending=lambda: runtime.call_end.ending,
+        )
+        self._tool_by_id = {tool.id: tool for tool in cfg.tools}
        self._waiting_for_generated_end_speech = False
+        self._greeting_finished = True
+        self._preflight_finished = False
+        self._opening_started = False
+        self._opening_finished = not bool(self._startup_actions("opening"))
+        self._startup_failed = False
        schemas: list[FunctionSchema] = []
        for tool in cfg.tools:
            if tool.type == "end_call":
@@ -74,6 +101,122 @@ class PromptBrain(BaseBrain):
            )
        runtime.set_tools(schemas)

+    async def run_preflight(self) -> None:
+        if self._preflight_finished:
+            return
+        try:
+            async with asyncio.timeout(PREFLIGHT_TIMEOUT_SECONDS):
+                succeeded = await self._run_startup_actions("preflight")
+        except TimeoutError as exc:
+            raise RuntimeError("Prompt preflight 超过 30 秒安全上限") from exc
+        if not succeeded:
+            raise RuntimeError("必需的 Prompt preflight Action 执行失败")
+        self._preflight_finished = True
+
+    async def on_connected(self, *, greeting_pending: bool = False) -> None:
+        self._greeting_finished = not greeting_pending
+        if (
+            self._startup_actions("opening")
+            and self._runtime is not None
+            and self._runtime.set_input_enabled is not None
+        ):
+            self._runtime.set_input_enabled(False)
+
+    async def on_client_ready(self) -> None:
+        if self._opening_started or self._opening_finished or self._startup_failed:
+            return
+        self._opening_started = True
+        try:
+            succeeded = await self._run_startup_actions("opening")
+        except ActionInvocationCancelled:
+            self._startup_failed = True
+            raise
+        if not succeeded:
+            await self._fail_opening("必需的开场 Action 执行失败")
+            return
+        self._opening_finished = True
+        self._release_startup_gate_if_ready()
+
+    async def on_greeting_finished(self) -> None:
+        self._greeting_finished = True
+        self._release_startup_gate_if_ready()
+
+    def _startup_actions(self, phase: str) -> list[dict[str, Any]]:
+        startup = self._cfg.startup if isinstance(self._cfg.startup, dict) else {}
+        return [
+            action
+            for action in startup.get("actions") or []
+            if isinstance(action, dict) and action.get("phase", "opening") == phase
+        ]
+
+    async def _run_startup_actions(self, phase: str) -> bool:
+        for action in self._startup_actions(phase):
+            action_id = str(action.get("id") or "startup_action")
+            tool_id = str(action.get("tool_id") or action.get("toolId") or "")
+            tool = self._tool_by_id.get(tool_id)
+            invocation_id = self._actions.new_invocation_id()
+            logger.info(
+                f"执行 Prompt {phase} Action: action={action_id} tool={tool_id}"
+            )
+            outcome = await self._actions.execute(
+                tool,
+                action.get("arguments") or {},
+                invocation_id=invocation_id,
+            )
+            if outcome.updated_variables:
+                self._refresh_prompt()
+            if phase == "opening" and self._runtime is not None:
+                await self._runtime.queue_frame(
+                    OutputTransportMessageUrgentFrame(
+                        message={
+                            "type": "startup-action-result",
+                            "actionId": action_id,
+                            "phase": phase,
+                            "outcome": outcome.trace_payload(),
+                        }
+                    )
+                )
+            if outcome.status == ActionStatus.SUCCESS:
+                continue
+            if outcome.status == ActionStatus.CANCELLED:
+                return False
+            if bool(action.get("required", True)):
+                logger.warning(
+                    f"必需的 Prompt {phase} Action 失败: "
+                    f"action={action_id} error={outcome.error}"
+                )
+                return False
+            logger.warning(
+                f"忽略可选 Prompt {phase} Action 失败: "
+                f"action={action_id} error={outcome.error}"
+            )
+        return True
+
+    def _release_startup_gate_if_ready(self) -> None:
+        runtime = self._runtime
+        if (
+            runtime is not None
+            and runtime.set_input_enabled is not None
+            and self._greeting_finished
+            and self._opening_finished
+            and not self._startup_failed
+            and not runtime.call_end.ending
+        ):
+            runtime.set_input_enabled(True)
+
+    async def _fail_opening(self, message: str) -> None:
+        self._startup_failed = True
+        runtime = self._runtime
+        if runtime is None or runtime.call_end.ending:
+            return
+        await runtime.queue_frame(
+            OutputTransportMessageUrgentFrame(
+                message={"type": "startup-action-error", "message": message}
+            )
+        )
+        runtime.call_end.begin("startup_action_failed")
+        await runtime.call_end.finish()
+
    def record_user_message(self, content: str) -> None:
        if not self._dynamic_enabled:
            return
--- a/backend/services/brains/workflow_brain.py
+++ b/backend/services/brains/workflow_brain.py
@@ -5,9 +5,7 @@ from __future__ import annotations
 import asyncio
 from copy import deepcopy
 from dataclasses import replace
-from time import monotonic
 from typing import Any
-from uuid import uuid4

 from loguru import logger
 from models import AssistantConfig, RuntimeTool
@@ -36,19 +34,18 @@ from services.brains.base import (
    BrainSpec,
    SessionVariableUpdate,
 )
+from services.action_runtime import (
+    ActionInvocationCancelled,
+    ActionOutcome,
+    ActionRunner,
+    ActionStatus,
+)
 from services.knowledge import search as search_knowledge
 from services.runtime_variables import DynamicVariableStore
 from services.tool_executor import ToolExecutionError, ToolExecutor
 from services.tool_policy import policy_for_tool
 from services.workflow.agent import WorkflowAgentStage
-from services.workflow.models import (
-    ActionError,
-    ActionOutcome,
-    ActionStatus,
-    RouteStatus,
-    WorkflowRuntimeState,
-    WorkflowStatus,
-)
+from services.workflow.models import RouteStatus, WorkflowRuntimeState, WorkflowStatus
 from services.workflow.output import WorkflowOutput
 from services.workflow.routing import WorkflowEdgeEvaluator
 from services.workflow_engine import WorkflowEngine
@@ -58,23 +55,6 @@ from services.workflow_router import WorkflowLLMRouter
 MAX_AUTOMATIC_HOPS = 50


-class _ActionFailure(RuntimeError):
-    """Internal adapter from heterogeneous tool failures to ActionOutcome."""
-
-    def __init__(
-        self,
-        message: str,
-        *,
-        code: str,
-        retryable: bool = False,
-        result: dict[str, Any] | None = None,
-    ) -> None:
-        super().__init__(message)
-        self.code = code
-        self.retryable = retryable
-        self.result = result
-
-
 class ConfiguredFlowManager(FlowManager):
    """Preserve Flow transitions while suppressing late async-tool replies."""

@@ -129,6 +109,7 @@ class WorkflowBrain(BaseBrain):
        self._cfg = cfg
        self._store = DynamicVariableStore.from_config(cfg or AssistantConfig(type="workflow"))
        self._tools = ToolExecutor(self._store)
+        self._actions = ActionRunner(self._tools)
        self._tool_by_id: dict[str, RuntimeTool] = {
            tool.id: tool for tool in (cfg.tools if cfg else [])
        }
@@ -166,6 +147,10 @@ class WorkflowBrain(BaseBrain):
        self._runtime = runtime
        self._store = DynamicVariableStore.from_config(cfg)
        self._tools = ToolExecutor(self._store, client_tools=runtime.client_tools)
+        self._actions = ActionRunner(
+            self._tools,
+            is_session_ending=lambda: runtime.call_end.ending,
+        )
        self._tool_by_id = {tool.id: tool for tool in cfg.tools}
        self._router = WorkflowLLMRouter(cfg)
        self._edge_evaluator = WorkflowEdgeEvaluator(
@@ -749,8 +734,7 @@ class WorkflowBrain(BaseBrain):
        await self._emit_node_active(node_id)
        data = self._engine.data(node_id)
        runtime = self._require_runtime()
-        invocation_id = f"act_{uuid4().hex[:20]}"
-        started_at = monotonic()
+        invocation_id = self._actions.new_invocation_id()
        block_user_input = data.get("userInputPolicy") == "block"
        if block_user_input and runtime.set_input_enabled:
            # Blocking only suppresses new audio/text input while the Action
@@ -760,7 +744,6 @@ class WorkflowBrain(BaseBrain):
            runtime.set_input_enabled(False)
        tool_id = str(data.get("toolId") or "")
        tool = self._tool_by_id.get(tool_id)
-        result: dict[str, Any] | None = None
        try:
            await self._emit_trace(
                "action_started",
@@ -769,82 +752,24 @@ class WorkflowBrain(BaseBrain):
                toolId=tool_id,
                toolType=tool.type if tool else None,
            )
-            if not tool:
-                raise _ActionFailure(
-                    f"工具不存在:{tool_id}",
-                    code="tool_not_found",
-                )
-            arguments = self._store.render_data(data.get("arguments") or {})
-            result = await self._tools.execute(
+            outcome = await self._actions.execute(
                tool,
-                arguments,
+                data.get("arguments") or {},
                result_assignments=self._action_result_assignments(data),
+                invocation_id=invocation_id,
            )
-            if result.get("status") != "ok":
-                returned_status = str(result.get("status") or "error")
-                raise _ActionFailure(
-                    str(result.get("message") or "工具返回执行失败状态"),
-                    code=str(result.get("code") or f"tool_{returned_status}"),
-                    retryable=bool(result.get("retryable", False)),
-                    result=result,
-                )
-            updated_variables = list(result.get("updated_variables") or [])
+            updated_variables = list(outcome.updated_variables)
            if updated_variables:
                await self._emit_variables(
                    reason="action",
                    node_id=node_id,
                    changed=updated_variables,
                )
-            outcome = ActionOutcome(
-                invocation_id=invocation_id,
-                status=ActionStatus.SUCCESS,
-                duration_ms=self._elapsed_ms(started_at),
-                result=result,
-                updated_variables=tuple(str(name) for name in updated_variables),
-            )
-        except asyncio.CancelledError:
-            outcome = ActionOutcome(
-                invocation_id=invocation_id,
-                status=ActionStatus.CANCELLED,
-                duration_ms=self._elapsed_ms(started_at),
-                result=result,
-                error=ActionError(
-                    code="action_cancelled",
-                    message="Action 随当前任务取消",
-                ),
-            )
+        except ActionInvocationCancelled as exc:
+            outcome = exc.outcome
            self._set_last_action(outcome)
            await self._emit_action_outcome(node_id, outcome)
            raise
-        except (_ActionFailure, ToolExecutionError, ValueError) as exc:
-            cancelled = self._action_was_cancelled(exc, runtime)
-            outcome = ActionOutcome(
-                invocation_id=invocation_id,
-                status=(
-                    ActionStatus.CANCELLED if cancelled else ActionStatus.FAILURE
-                ),
-                duration_ms=self._elapsed_ms(started_at),
-                result=(exc.result if isinstance(exc, _ActionFailure) else result),
-                error=ActionError(
-                    code=(
-                        "session_ended"
-                        if cancelled
-                        else (
-                            exc.code
-                            if isinstance(exc, _ActionFailure)
-                            else (
-                                "invalid_action_configuration"
-                                if isinstance(exc, ValueError)
-                                else "tool_execution_error"
-                            )
-                        )
-                    ),
-                    message=str(exc)[:2048],
-                    retryable=(
-                        exc.retryable if isinstance(exc, _ActionFailure) else False
-                    ),
-                ),
-            )
        finally:
            if block_user_input and runtime.set_input_enabled:
                runtime.set_input_enabled(True)
@@ -852,26 +777,6 @@ class WorkflowBrain(BaseBrain):
        await self._emit_action_outcome(node_id, outcome)
        return outcome

-    @staticmethod
-    def _elapsed_ms(started_at: float) -> int:
-        return max(0, round((monotonic() - started_at) * 1000))
-
-    @staticmethod
-    def _action_was_cancelled(exc: Exception, runtime: BrainRuntime) -> bool:
-        if getattr(runtime.call_end, "ending", False):
-            return True
-        message = str(exc)
-        return any(
-            marker in message
-            for marker in (
-                "会话已结束",
-                "会话已取消",
-                "管线已停止",
-                "通道已关闭",
-                "连接已断开",
-            )
-        )
-
    def _set_last_action(self, outcome: ActionOutcome) -> None:
        legacy_status = {
            ActionStatus.SUCCESS: "ok",