feat: add prompt startup actions and shared vision config

backend/services/action_runtime.py

@@ -0,0 +1,222 @@
+"""Deterministic Action execution shared by Prompt startup and Workflow."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable
+from dataclasses import dataclass
+from enum import StrEnum
+from time import monotonic
+from typing import Any
+from uuid import uuid4
+
+from models import RuntimeTool
+from services.tool_executor import ToolExecutionError, ToolExecutor
+
+
+class ActionStatus(StrEnum):
+    SUCCESS = "success"
+    FAILURE = "failure"
+    CANCELLED = "cancelled"
+
+
+@dataclass(frozen=True)
+class ActionError:
+    code: str
+    message: str
+    retryable: bool = False
+
+
+@dataclass(frozen=True)
+class ActionOutcome:
+    """One completed Action invocation.
+
+    Raw tool results stay in memory. Persisted or client-visible events should
+    use ``trace_payload`` so private business data is not copied accidentally.
+    """
+
+    invocation_id: str
+    status: ActionStatus
+    duration_ms: int
+    result: dict[str, Any] | None = None
+    updated_variables: tuple[str, ...] = ()
+    error: ActionError | None = None
+
+    @property
+    def should_route(self) -> bool:
+        return self.status != ActionStatus.CANCELLED
+
+    def trace_payload(self) -> dict[str, Any]:
+        payload: dict[str, Any] = {
+            "invocationId": self.invocation_id,
+            "status": self.status.value,
+            "durationMs": self.duration_ms,
+            "updatedVariables": list(self.updated_variables),
+        }
+        if self.result is not None:
+            payload["resultKeys"] = sorted(str(key) for key in self.result)
+        if self.error is not None:
+            payload["error"] = {
+                "code": self.error.code,
+                "message": self.error.message,
+                "retryable": self.error.retryable,
+            }
+        return payload
+
+
+class ActionInvocationCancelled(asyncio.CancelledError):
+    """Carry a structured outcome while preserving task cancellation."""
+
+    def __init__(self, outcome: ActionOutcome) -> None:
+        super().__init__(outcome.error.message if outcome.error else "Action cancelled")
+        self.outcome = outcome
+
+
+class ActionRunner:
+    """Normalize ToolExecutor's heterogeneous responses into ActionOutcome."""
+
+    _CANCELLED_MARKERS = (
+        "会话已结束",
+        "会话已取消",
+        "管线已停止",
+        "通道已关闭",
+        "连接已断开",
+    )
+
+    def __init__(
+        self,
+        executor: ToolExecutor,
+        *,
+        is_session_ending: Callable[[], bool] | None = None,
+    ) -> None:
+        self._executor = executor
+        self._is_session_ending = is_session_ending or (lambda: False)
+
+    @staticmethod
+    def new_invocation_id() -> str:
+        return f"act_{uuid4().hex[:20]}"
+
+    async def execute(
+        self,
+        tool: RuntimeTool | None,
+        arguments: dict[str, Any] | None = None,
+        *,
+        result_assignments: dict[str, str] | None = None,
+        invocation_id: str | None = None,
+    ) -> ActionOutcome:
+        invocation_id = invocation_id or self.new_invocation_id()
+        started_at = monotonic()
+        result: dict[str, Any] | None = None
+        if tool is None:
+            return self._failure(
+                invocation_id,
+                started_at,
+                code="tool_not_found",
+                message="Action 引用的工具不存在",
+            )
+
+        try:
+            rendered_arguments = self._executor.store.render_data(arguments or {})
+            result = await self._executor.execute(
+                tool,
+                rendered_arguments,
+                result_assignments=result_assignments,
+            )
+            if result.get("status") != "ok":
+                returned_status = str(result.get("status") or "error")
+                message = str(result.get("message") or "工具返回执行失败状态")
+                if self._is_cancelled(message):
+                    return ActionOutcome(
+                        invocation_id=invocation_id,
+                        status=ActionStatus.CANCELLED,
+                        duration_ms=self._elapsed_ms(started_at),
+                        result=result,
+                        error=ActionError(
+                            code="session_ended",
+                            message=message[:2048],
+                        ),
+                    )
+                return self._failure(
+                    invocation_id,
+                    started_at,
+                    code=str(result.get("code") or f"tool_{returned_status}"),
+                    message=message,
+                    retryable=bool(result.get("retryable", False)),
+                    result=result,
+                )
+            return ActionOutcome(
+                invocation_id=invocation_id,
+                status=ActionStatus.SUCCESS,
+                duration_ms=self._elapsed_ms(started_at),
+                result=result,
+                updated_variables=tuple(
+                    str(name) for name in result.get("updated_variables") or []
+                ),
+            )
+        except asyncio.CancelledError as exc:
+            outcome = ActionOutcome(
+                invocation_id=invocation_id,
+                status=ActionStatus.CANCELLED,
+                duration_ms=self._elapsed_ms(started_at),
+                result=result,
+                error=ActionError(
+                    code="action_cancelled",
+                    message="Action 随当前任务取消",
+                ),
+            )
+            raise ActionInvocationCancelled(outcome) from exc
+        except (ToolExecutionError, ValueError) as exc:
+            cancelled = self._is_cancelled(str(exc))
+            if cancelled:
+                return ActionOutcome(
+                    invocation_id=invocation_id,
+                    status=ActionStatus.CANCELLED,
+                    duration_ms=self._elapsed_ms(started_at),
+                    result=result,
+                    error=ActionError(
+                        code="session_ended",
+                        message=str(exc)[:2048],
+                    ),
+                )
+            return self._failure(
+                invocation_id,
+                started_at,
+                code=(
+                    "invalid_action_configuration"
+                    if isinstance(exc, ValueError)
+                    else "tool_execution_error"
+                ),
+                message=str(exc),
+                result=result,
+            )
+
+    def _is_cancelled(self, message: str) -> bool:
+        return self._is_session_ending() or any(
+            marker in message for marker in self._CANCELLED_MARKERS
+        )
+
+    def _failure(
+        self,
+        invocation_id: str,
+        started_at: float,
+        *,
+        code: str,
+        message: str,
+        retryable: bool = False,
+        result: dict[str, Any] | None = None,
+    ) -> ActionOutcome:
+        return ActionOutcome(
+            invocation_id=invocation_id,
+            status=ActionStatus.FAILURE,
+            duration_ms=self._elapsed_ms(started_at),
+            result=result,
+            error=ActionError(
+                code=code,
+                message=message[:2048],
+                retryable=retryable,
+            ),
+        )
+
+    @staticmethod
+    def _elapsed_ms(started_at: float) -> int:
+        return max(0, round((monotonic() - started_at) * 1000))

backend/services/brains/base.py

@@ -118,6 +118,9 @@ class BaseBrain:
     async def setup(self, cfg: AssistantConfig, runtime: BrainRuntime) -> None:
         """Register tools and initialize per-call orchestration."""
 
+    async def run_preflight(self) -> None:
+        """Run deterministic server-side startup work before media starts."""
+
     async def on_connected(self, *, greeting_pending: bool = False) -> None:
         """Handle a connected client before an optional greeting is played.
 
@@ -198,6 +201,8 @@ class Brain(Protocol):
 
     async def setup(self, cfg: AssistantConfig, runtime: BrainRuntime) -> None: ...
 
+    async def run_preflight(self) -> None: ...
+
     async def on_connected(self, *, greeting_pending: bool = False) -> None: ...
 
     async def on_greeting_finished(self) -> None: ...

backend/services/brains/prompt_brain.py

@@ -2,9 +2,11 @@
 
 from __future__ import annotations
 
+import asyncio
 from typing import Any
 from uuid import uuid4
 
+from loguru import logger
 from models import AssistantConfig
 from pipecat.adapters.schemas.function_schema import FunctionSchema
 from pipecat.frames.frames import OutputTransportMessageUrgentFrame, TTSSpeakFrame
@@ -22,11 +24,19 @@ from services.brains.base import (
     BrainSpec,
     SessionVariableUpdate,
 )
+from services.action_runtime import (
+    ActionInvocationCancelled,
+    ActionRunner,
+    ActionStatus,
+)
 from services.runtime_variables import DynamicVariableStore
 from services.tool_executor import ToolExecutionError, ToolExecutor
 from services.tool_policy import policy_for_tool
 
 
+PREFLIGHT_TIMEOUT_SECONDS = 30
+
+
 class PromptBrain(BaseBrain):
     spec = BrainSpec(
         type="prompt",
@@ -39,8 +49,15 @@ class PromptBrain(BaseBrain):
         self._dynamic_enabled = True
         self._store = DynamicVariableStore.from_config(cfg)
         self._tools = ToolExecutor(self._store)
+        self._actions = ActionRunner(self._tools)
+        self._tool_by_id = {tool.id: tool for tool in cfg.tools}
         self._runtime: BrainRuntime | None = None
         self._waiting_for_generated_end_speech = False
+        self._greeting_finished = True
+        self._preflight_finished = False
+        self._opening_started = False
+        self._opening_finished = False
+        self._startup_failed = False
 
     async def greeting(self, cfg: AssistantConfig) -> str:
         return self._store.render(cfg.greeting) if self._dynamic_enabled else cfg.greeting
@@ -56,7 +73,17 @@ class PromptBrain(BaseBrain):
     async def setup(self, cfg: AssistantConfig, runtime: BrainRuntime) -> None:
         self._runtime = runtime
         self._tools.set_client_tools(runtime.client_tools)
+        self._actions = ActionRunner(
+            self._tools,
+            is_session_ending=lambda: runtime.call_end.ending,
+        )
+        self._tool_by_id = {tool.id: tool for tool in cfg.tools}
         self._waiting_for_generated_end_speech = False
+        self._greeting_finished = True
+        self._preflight_finished = False
+        self._opening_started = False
+        self._opening_finished = not bool(self._startup_actions("opening"))
+        self._startup_failed = False
         schemas: list[FunctionSchema] = []
         for tool in cfg.tools:
             if tool.type == "end_call":
@@ -74,6 +101,122 @@ class PromptBrain(BaseBrain):
             )
         runtime.set_tools(schemas)
 
+    async def run_preflight(self) -> None:
+        if self._preflight_finished:
+            return
+        try:
+            async with asyncio.timeout(PREFLIGHT_TIMEOUT_SECONDS):
+                succeeded = await self._run_startup_actions("preflight")
+        except TimeoutError as exc:
+            raise RuntimeError("Prompt preflight 超过 30 秒安全上限") from exc
+        if not succeeded:
+            raise RuntimeError("必需的 Prompt preflight Action 执行失败")
+        self._preflight_finished = True
+
+    async def on_connected(self, *, greeting_pending: bool = False) -> None:
+        self._greeting_finished = not greeting_pending
+        if (
+            self._startup_actions("opening")
+            and self._runtime is not None
+            and self._runtime.set_input_enabled is not None
+        ):
+            self._runtime.set_input_enabled(False)
+
+    async def on_client_ready(self) -> None:
+        if self._opening_started or self._opening_finished or self._startup_failed:
+            return
+        self._opening_started = True
+        try:
+            succeeded = await self._run_startup_actions("opening")
+        except ActionInvocationCancelled:
+            self._startup_failed = True
+            raise
+        if not succeeded:
+            await self._fail_opening("必需的开场 Action 执行失败")
+            return
+        self._opening_finished = True
+        self._release_startup_gate_if_ready()
+
+    async def on_greeting_finished(self) -> None:
+        self._greeting_finished = True
+        self._release_startup_gate_if_ready()
+
+    def _startup_actions(self, phase: str) -> list[dict[str, Any]]:
+        startup = self._cfg.startup if isinstance(self._cfg.startup, dict) else {}
+        return [
+            action
+            for action in startup.get("actions") or []
+            if isinstance(action, dict) and action.get("phase", "opening") == phase
+        ]
+
+    async def _run_startup_actions(self, phase: str) -> bool:
+        for action in self._startup_actions(phase):
+            action_id = str(action.get("id") or "startup_action")
+            tool_id = str(action.get("tool_id") or action.get("toolId") or "")
+            tool = self._tool_by_id.get(tool_id)
+            invocation_id = self._actions.new_invocation_id()
+            logger.info(
+                f"执行 Prompt {phase} Action: action={action_id} tool={tool_id}"
+            )
+            outcome = await self._actions.execute(
+                tool,
+                action.get("arguments") or {},
+                invocation_id=invocation_id,
+            )
+            if outcome.updated_variables:
+                self._refresh_prompt()
+            if phase == "opening" and self._runtime is not None:
+                await self._runtime.queue_frame(
+                    OutputTransportMessageUrgentFrame(
+                        message={
+                            "type": "startup-action-result",
+                            "actionId": action_id,
+                            "phase": phase,
+                            "outcome": outcome.trace_payload(),
+                        }
+                    )
+                )
+            if outcome.status == ActionStatus.SUCCESS:
+                continue
+            if outcome.status == ActionStatus.CANCELLED:
+                return False
+            if bool(action.get("required", True)):
+                logger.warning(
+                    f"必需的 Prompt {phase} Action 失败: "
+                    f"action={action_id} error={outcome.error}"
+                )
+                return False
+            logger.warning(
+                f"忽略可选 Prompt {phase} Action 失败: "
+                f"action={action_id} error={outcome.error}"
+            )
+        return True
+
+    def _release_startup_gate_if_ready(self) -> None:
+        runtime = self._runtime
+        if (
+            runtime is not None
+            and runtime.set_input_enabled is not None
+            and self._greeting_finished
+            and self._opening_finished
+            and not self._startup_failed
+            and not runtime.call_end.ending
+        ):
+            runtime.set_input_enabled(True)
+
+    async def _fail_opening(self, message: str) -> None:
+        self._startup_failed = True
+        runtime = self._runtime
+        if runtime is None or runtime.call_end.ending:
+            return
+        await runtime.queue_frame(
+            OutputTransportMessageUrgentFrame(
+                message={"type": "startup-action-error", "message": message}
+            )
+        )
+        runtime.call_end.begin("startup_action_failed")
+        await runtime.call_end.finish()
+
     def record_user_message(self, content: str) -> None:
         if not self._dynamic_enabled:
             return

backend/services/brains/workflow_brain.py

@@ -5,9 +5,7 @@ from __future__ import annotations
 import asyncio
 from copy import deepcopy
 from dataclasses import replace
-from time import monotonic
 from typing import Any
-from uuid import uuid4
 
 from loguru import logger
 from models import AssistantConfig, RuntimeTool
@@ -36,19 +34,18 @@ from services.brains.base import (
     BrainSpec,
     SessionVariableUpdate,
 )
+from services.action_runtime import (
+    ActionInvocationCancelled,
+    ActionOutcome,
+    ActionRunner,
+    ActionStatus,
+)
 from services.knowledge import search as search_knowledge
 from services.runtime_variables import DynamicVariableStore
 from services.tool_executor import ToolExecutionError, ToolExecutor
 from services.tool_policy import policy_for_tool
 from services.workflow.agent import WorkflowAgentStage
-from services.workflow.models import (
-    ActionError,
-    ActionOutcome,
-    ActionStatus,
-    RouteStatus,
-    WorkflowRuntimeState,
-    WorkflowStatus,
-)
+from services.workflow.models import RouteStatus, WorkflowRuntimeState, WorkflowStatus
 from services.workflow.output import WorkflowOutput
 from services.workflow.routing import WorkflowEdgeEvaluator
 from services.workflow_engine import WorkflowEngine
@@ -58,23 +55,6 @@ from services.workflow_router import WorkflowLLMRouter
 MAX_AUTOMATIC_HOPS = 50
 
 
-class _ActionFailure(RuntimeError):
-    """Internal adapter from heterogeneous tool failures to ActionOutcome."""
-
-    def __init__(
-        self,
-        message: str,
-        *,
-        code: str,
-        retryable: bool = False,
-        result: dict[str, Any] | None = None,
-    ) -> None:
-        super().__init__(message)
-        self.code = code
-        self.retryable = retryable
-        self.result = result
-
-
 class ConfiguredFlowManager(FlowManager):
     """Preserve Flow transitions while suppressing late async-tool replies."""
 
@@ -129,6 +109,7 @@ class WorkflowBrain(BaseBrain):
         self._cfg = cfg
         self._store = DynamicVariableStore.from_config(cfg or AssistantConfig(type="workflow"))
         self._tools = ToolExecutor(self._store)
+        self._actions = ActionRunner(self._tools)
         self._tool_by_id: dict[str, RuntimeTool] = {
             tool.id: tool for tool in (cfg.tools if cfg else [])
         }
@@ -166,6 +147,10 @@ class WorkflowBrain(BaseBrain):
         self._runtime = runtime
         self._store = DynamicVariableStore.from_config(cfg)
         self._tools = ToolExecutor(self._store, client_tools=runtime.client_tools)
+        self._actions = ActionRunner(
+            self._tools,
+            is_session_ending=lambda: runtime.call_end.ending,
+        )
         self._tool_by_id = {tool.id: tool for tool in cfg.tools}
         self._router = WorkflowLLMRouter(cfg)
         self._edge_evaluator = WorkflowEdgeEvaluator(
@@ -749,8 +734,7 @@ class WorkflowBrain(BaseBrain):
         await self._emit_node_active(node_id)
         data = self._engine.data(node_id)
         runtime = self._require_runtime()
-        invocation_id = f"act_{uuid4().hex[:20]}"
-        started_at = monotonic()
+        invocation_id = self._actions.new_invocation_id()
         block_user_input = data.get("userInputPolicy") == "block"
         if block_user_input and runtime.set_input_enabled:
             # Blocking only suppresses new audio/text input while the Action
@@ -760,7 +744,6 @@ class WorkflowBrain(BaseBrain):
             runtime.set_input_enabled(False)
         tool_id = str(data.get("toolId") or "")
         tool = self._tool_by_id.get(tool_id)
-        result: dict[str, Any] | None = None
         try:
             await self._emit_trace(
                 "action_started",
@@ -769,82 +752,24 @@ class WorkflowBrain(BaseBrain):
                 toolId=tool_id,
                 toolType=tool.type if tool else None,
             )
-            if not tool:
-                raise _ActionFailure(
-                    f"工具不存在:{tool_id}",
-                    code="tool_not_found",
-                )
-            arguments = self._store.render_data(data.get("arguments") or {})
-            result = await self._tools.execute(
+            outcome = await self._actions.execute(
                 tool,
-                arguments,
+                data.get("arguments") or {},
                 result_assignments=self._action_result_assignments(data),
+                invocation_id=invocation_id,
             )
-            if result.get("status") != "ok":
-                returned_status = str(result.get("status") or "error")
-                raise _ActionFailure(
-                    str(result.get("message") or "工具返回执行失败状态"),
-                    code=str(result.get("code") or f"tool_{returned_status}"),
-                    retryable=bool(result.get("retryable", False)),
-                    result=result,
-                )
-            updated_variables = list(result.get("updated_variables") or [])
+            updated_variables = list(outcome.updated_variables)
             if updated_variables:
                 await self._emit_variables(
                     reason="action",
                     node_id=node_id,
                     changed=updated_variables,
                 )
-            outcome = ActionOutcome(
-                invocation_id=invocation_id,
-                status=ActionStatus.SUCCESS,
-                duration_ms=self._elapsed_ms(started_at),
-                result=result,
-                updated_variables=tuple(str(name) for name in updated_variables),
-            )
-        except asyncio.CancelledError:
-            outcome = ActionOutcome(
-                invocation_id=invocation_id,
-                status=ActionStatus.CANCELLED,
-                duration_ms=self._elapsed_ms(started_at),
-                result=result,
-                error=ActionError(
-                    code="action_cancelled",
-                    message="Action 随当前任务取消",
-                ),
-            )
+        except ActionInvocationCancelled as exc:
+            outcome = exc.outcome
             self._set_last_action(outcome)
             await self._emit_action_outcome(node_id, outcome)
             raise
-        except (_ActionFailure, ToolExecutionError, ValueError) as exc:
-            cancelled = self._action_was_cancelled(exc, runtime)
-            outcome = ActionOutcome(
-                invocation_id=invocation_id,
-                status=(
-                    ActionStatus.CANCELLED if cancelled else ActionStatus.FAILURE
-                ),
-                duration_ms=self._elapsed_ms(started_at),
-                result=(exc.result if isinstance(exc, _ActionFailure) else result),
-                error=ActionError(
-                    code=(
-                        "session_ended"
-                        if cancelled
-                        else (
-                            exc.code
-                            if isinstance(exc, _ActionFailure)
-                            else (
-                                "invalid_action_configuration"
-                                if isinstance(exc, ValueError)
-                                else "tool_execution_error"
-                            )
-                        )
-                    ),
-                    message=str(exc)[:2048],
-                    retryable=(
-                        exc.retryable if isinstance(exc, _ActionFailure) else False
-                    ),
-                ),
-            )
         finally:
             if block_user_input and runtime.set_input_enabled:
                 runtime.set_input_enabled(True)
@@ -852,26 +777,6 @@ class WorkflowBrain(BaseBrain):
         await self._emit_action_outcome(node_id, outcome)
         return outcome
 
-    @staticmethod
-    def _elapsed_ms(started_at: float) -> int:
-        return max(0, round((monotonic() - started_at) * 1000))
-
-    @staticmethod
-    def _action_was_cancelled(exc: Exception, runtime: BrainRuntime) -> bool:
-        if getattr(runtime.call_end, "ending", False):
-            return True
-        message = str(exc)
-        return any(
-            marker in message
-            for marker in (
-                "会话已结束",
-                "会话已取消",
-                "管线已停止",
-                "通道已关闭",
-                "连接已断开",
-            )
-        )
-
     def _set_last_action(self, outcome: ActionOutcome) -> None:
         legacy_status = {
             ActionStatus.SUCCESS: "ok",

backend/services/config_resolver.py

@@ -227,6 +227,7 @@ async def resolve_runtime_config(
         runtimeMode=assistant.runtime_mode,  # type: ignore[arg-type]
         enableInterrupt=assistant.enable_interrupt,
         turnConfig=assistant.turn_config or {},
+        startup=assistant.startup or {},
         tools=await _tools_for(session, assistant),
         knowledge_base_id=assistant.knowledge_base_id,
         knowledge_base_name=knowledge_base.name if knowledge_base else "",

backend/services/pipecat/pipeline.py

@@ -270,7 +270,6 @@ async def run_pipeline(
             build_workflow_voice_switcher(cfg, "TTS", tts)
         )
 
-    greeting = await brain.greeting(cfg)
     system_content = brain.system_prompt(cfg)
 
     worker_holder: dict = {}
@@ -683,6 +682,14 @@ async def run_pipeline(
             flow_global_functions=flow_global_functions,
         ),
     )
+    try:
+        await brain.run_preflight()
+    except Exception:
+        if recorder:
+            await recorder.finish(status="failed")
+        raise
+    # Preflight tools may assign variables used by the opening speech.
+    greeting = await brain.greeting(cfg)
 
     async def submit_user_input(value: UserInput) -> None:
         if not value.has_camera_frame:

backend/services/workflow/models.py

@@ -27,65 +27,6 @@ class RouteStatus(StrEnum):
     ERROR = "error"
 
 
-class ActionStatus(StrEnum):
-    """Stable Action outcomes used by routing and future debug tooling."""
-
-    SUCCESS = "success"
-    FAILURE = "failure"
-    CANCELLED = "cancelled"
-
-
-@dataclass(frozen=True)
-class ActionError:
-    """Machine-readable failure details without losing the operator message."""
-
-    code: str
-    message: str
-    retryable: bool = False
-
-
-@dataclass(frozen=True)
-class ActionOutcome:
-    """One completed Action invocation.
-
-    ``result`` remains an in-memory value because a tool response may contain
-    private business data. Trace events publish only its shape and variable
-    names, never the raw response.
-    """
-
-    invocation_id: str
-    status: ActionStatus
-    duration_ms: int
-    result: dict[str, Any] | None = None
-    updated_variables: tuple[str, ...] = ()
-    error: ActionError | None = None
-
-    @property
-    def should_route(self) -> bool:
-        """Cancellation is a lifecycle outcome, not a failure branch."""
-
-        return self.status != ActionStatus.CANCELLED
-
-    def trace_payload(self) -> dict[str, Any]:
-        """Return a persistence-safe summary of the execution result."""
-
-        payload: dict[str, Any] = {
-            "invocationId": self.invocation_id,
-            "status": self.status.value,
-            "durationMs": self.duration_ms,
-            "updatedVariables": list(self.updated_variables),
-        }
-        if self.result is not None:
-            payload["resultKeys"] = sorted(str(key) for key in self.result)
-        if self.error is not None:
-            payload["error"] = {
-                "code": self.error.code,
-                "message": self.error.message,
-                "retryable": self.error.retryable,
-            }
-        return payload
-
-
 @dataclass(frozen=True)
 class UserTurn:
     """One committed user turn that may cross automatic Workflow nodes."""