feat(workflow): add edge-tool routing and realtime runtime

backend/services/brains/base.py

@@ -101,6 +101,18 @@ class BrainRuntime:
     flow_global_functions: list[Any] = field(default_factory=list)
 
 
+@dataclass(frozen=True)
+class RealtimeBrainRuntime:
+    """Pipeline-owned capabilities for a speech-to-speech brain session."""
+
+    realtime: Any
+    queue_frame: Callable[[Frame], Awaitable[None]]
+    call_end: CallEndPort
+    session_id: str = ""
+    client_tools: ClientToolPort | None = None
+    set_input_enabled: Callable[[bool], None] | None = None
+
+
 class BaseBrain:
     """No-op lifecycle defaults for brains without local orchestration."""
 
@@ -118,6 +130,16 @@ class BaseBrain:
     async def setup(self, cfg: AssistantConfig, runtime: BrainRuntime) -> None:
         """Register tools and initialize per-call orchestration."""
 
+    async def setup_realtime(
+        self,
+        cfg: AssistantConfig,
+        runtime: RealtimeBrainRuntime,
+    ) -> None:
+        """Initialize optional speech-to-speech orchestration."""
+
+    async def on_realtime_user_speech_started(self) -> None:
+        """Allow a workflow to move past an interruptible fixed message."""
+
     async def run_preflight(self) -> None:
         """Run deterministic server-side startup work before media starts."""
 
@@ -211,6 +233,14 @@ class Brain(Protocol):
 
     async def setup(self, cfg: AssistantConfig, runtime: BrainRuntime) -> None: ...
 
+    async def setup_realtime(
+        self,
+        cfg: AssistantConfig,
+        runtime: RealtimeBrainRuntime,
+    ) -> None: ...
+
+    async def on_realtime_user_speech_started(self) -> None: ...
+
     async def run_preflight(self) -> None: ...
 
     async def on_connected(self, *, greeting_pending: bool = False) -> None: ...

backend/services/brains/workflow_brain.py

@@ -33,6 +33,7 @@ from services.brains.base import (
     BaseBrain,
     BrainRuntime,
     BrainSpec,
+    RealtimeBrainRuntime,
     SessionVariableUpdate,
 )
 from services.action_runtime import (
@@ -64,6 +65,7 @@ from services.workflow.agent import WorkflowAgentStage
 from services.workflow.models import RouteStatus, WorkflowRuntimeState, WorkflowStatus
 from services.workflow.output import WorkflowOutput
 from services.workflow.routing import WorkflowEdgeEvaluator
+from services.workflow.realtime import WorkflowRealtimeController
 from services.workflow_engine import WorkflowEngine
 from services.workflow_router import WorkflowLLMRouter
 
@@ -147,7 +149,7 @@ class ConfiguredFlowManager(FlowManager):
 class WorkflowBrain(BaseBrain):
     spec = BrainSpec(
         type="workflow",
-        supported_runtime_modes=frozenset({"pipeline"}),
+        supported_runtime_modes=frozenset({"pipeline", "realtime"}),
         owns_context=True,
     )
 
@@ -188,12 +190,15 @@ class WorkflowBrain(BaseBrain):
         self._waiting_for_generated_end_speech = False
         self._next_message_token = 1
         self._pending_message: _MessageContinuation | None = None
+        self._realtime_controller: WorkflowRealtimeController | None = None
 
     async def greeting(self, _cfg: AssistantConfig) -> str:
         """Workflow opening speech belongs to an explicit Message or Agent."""
         return ""
 
     def system_prompt(self, cfg: AssistantConfig) -> str:
+        if self._realtime_controller is not None:
+            return self._realtime_controller.system_prompt()
         return self._store.render(self._engine.global_prompt())
 
     def build_llm(self, cfg: AssistantConfig, context: LLMContext) -> FrameProcessor:
@@ -234,6 +239,7 @@ class WorkflowBrain(BaseBrain):
         self._waiting_for_generated_end_speech = False
         self._next_message_token = 1
         self._pending_message = None
+        self._realtime_controller = None
         self._manager = ConfiguredFlowManager(
             worker=runtime.worker,
             llm=runtime.llm,
@@ -243,7 +249,30 @@ class WorkflowBrain(BaseBrain):
         )
         self._manager.state["variables"] = self._store.values
 
+    async def setup_realtime(
+        self,
+        cfg: AssistantConfig,
+        runtime: RealtimeBrainRuntime,
+    ) -> None:
+        self._cfg = cfg
+        self._store = DynamicVariableStore.from_config(cfg)
+        self._realtime_controller = WorkflowRealtimeController(
+            cfg=cfg,
+            engine=self._engine,
+            store=self._store,
+            runtime=runtime,
+        )
+        runtime.realtime.set_tool_dispatcher(
+            self._realtime_controller.dispatch_tool
+        )
+        runtime.realtime.set_speech_started_handler(
+            self.on_realtime_user_speech_started
+        )
+
     async def on_connected(self, *, greeting_pending: bool = False) -> None:
+        if self._realtime_controller is not None:
+            await self._realtime_controller.start()
+            return
         self._state.enter(self._engine.start_id, WorkflowStatus.STARTING)
         await self._emit_node_active(self._engine.start_id)
         await self._emit_variables(
@@ -282,6 +311,9 @@ class WorkflowBrain(BaseBrain):
 
     async def on_client_ready(self) -> None:
         """Replay state that may have been emitted before WebRTC data was ready."""
+        if self._realtime_controller is not None:
+            await self._realtime_controller.on_client_ready()
+            return
         await self._require_output().mark_client_ready()
         current_node = (
             str(self._manager.current_node)
@@ -300,24 +332,41 @@ class WorkflowBrain(BaseBrain):
         self,
         dynamic_variables: dict[str, Any],
     ) -> SessionVariableUpdate:
+        if self._realtime_controller is not None:
+            return await self._realtime_controller.on_session_update(
+                dynamic_variables
+            )
         if self._ended:
             raise ValueError("工作流会话已经结束")
-        changed = self._store.assign_declared_many(dynamic_variables)
-        current = self._state.current_node_id
-        if changed and current and self._engine.node_type(current) == "agent":
-            await self._refresh_agent_prompt(current)
-        if changed:
-            await self._emit_variables(
-                reason="session_update",
-                node_id=current or None,
-                changed=changed,
+        async with self._turn_lock:
+            changed = self._store.assign_declared_many(dynamic_variables)
+            current = self._state.current_node_id
+            next_config = (
+                await self._after_variables_changed(
+                    current,
+                    changed,
+                    reason="session_update",
+                )
+                if current
+                else None
             )
+            if changed and not current:
+                await self._emit_variables(
+                    reason="session_update",
+                    node_id=None,
+                    changed=changed,
+                )
+            if next_config:
+                await self._activate_node_config(next_config)
         return SessionVariableUpdate(
             changed=changed,
             dynamic_variables=self._store.public_values(),
         )
 
     def record_user_message(self, content: str) -> None:
+        if self._realtime_controller is not None:
+            self._realtime_controller.record_user_message(content)
+            return
         if content and not self._ended:
             self._store.record("user", content)
 
@@ -327,6 +376,8 @@ class WorkflowBrain(BaseBrain):
         user_message: dict[str, Any] | None = None,
     ) -> bool:
         """Route a complete user turn before the active stage may reply."""
+        if self._realtime_controller is not None:
+            return await self._realtime_controller.handle_text_input(content)
         if not content or self._ended:
             return True
         async with self._turn_lock:
@@ -369,6 +420,26 @@ class WorkflowBrain(BaseBrain):
         self.record_user_message(content)
         self._state.begin_user_turn(content)
 
+        if self._engine.llm_routing_mode() == "edge_tool":
+            self._state.status = WorkflowStatus.ROUTING
+            edge = self._engine.deterministic_edge(
+                current,
+                self._store,
+                include_default=False,
+            )
+            if edge and manager.current_node == current:
+                next_config = await self._follow_edge(
+                    edge,
+                    triggering_user_text=content,
+                    triggering_user_message=user_message,
+                )
+                await self._activate_node_config(
+                    next_config,
+                    triggering_user_text=content,
+                )
+                return True
+            return await self._continue_current_node_after_no_transition(current)
+
         self._state.status = WorkflowStatus.ROUTING
         decision = await self._edge_evaluator.evaluate(
             current,
@@ -487,6 +558,12 @@ class WorkflowBrain(BaseBrain):
         node_id: str,
     ) -> dict | None:
         """Compatibility helper used by automatic-node traversal and tests."""
+        if self._engine.llm_routing_mode() == "edge_tool":
+            return self._engine.deterministic_edge(
+                node_id,
+                self._store,
+                include_default=True,
+            )
         decision = await self._edge_evaluator.evaluate(node_id)
         if decision.status == RouteStatus.ERROR:
             await self._require_output().emit_error(
@@ -507,6 +584,12 @@ class WorkflowBrain(BaseBrain):
         content: str,
         interrupted: bool,
     ) -> None:
+        if self._realtime_controller is not None:
+            await self._realtime_controller.on_assistant_text_end(
+                content,
+                interrupted,
+            )
+            return
         if content and not interrupted and not self._ended:
             self._store.record("agent", content, completed_agent_turn=True)
             self._state.consume_user_turn()
@@ -522,6 +605,10 @@ class WorkflowBrain(BaseBrain):
                 has_text=bool(content.strip()) and not interrupted
             )
 
+    async def on_realtime_user_speech_started(self) -> None:
+        if self._realtime_controller is not None:
+            await self._realtime_controller.on_user_speech_started()
+
     async def _refresh_agent_prompt(self, node_id: str) -> None:
         await self._require_agent_stage().refresh_prompt(node_id)
 
@@ -579,6 +666,9 @@ class WorkflowBrain(BaseBrain):
         append_function(self._knowledge_function(node_id))
         if stage.vision_enabled and self._require_runtime().vision_function:
             append_function(self._require_runtime().vision_function)
+        if self._engine.llm_routing_mode() == "edge_tool":
+            for edge in self._engine.edge_tool_edges(node_id):
+                append_function(self._edge_tool(edge, node_id))
         return self._require_agent_stage().node_config(
             node_id,
             functions=functions,
@@ -688,26 +778,13 @@ class WorkflowBrain(BaseBrain):
                 }
             updated_variables = list(result.get("updated_variables") or [])
             if updated_variables:
-                await self._emit_variables(
-                    reason="tool",
-                    node_id=node_id,
-                    changed=updated_variables,
-                )
-                await self._refresh_agent_prompt(node_id)
-                edge = self._engine.deterministic_edge(
-                    node_id,
-                    self._store,
-                    include_default=False,
-                )
-                if edge:
-                    next_config = await self._follow_edge(
-                        edge,
-                        triggering_user_text=(
-                            self._state.pending_user_turn.text
-                            if self._state.pending_user_turn
-                            else ""
-                        ),
+                async with self._turn_lock:
+                    next_config = await self._after_variables_changed(
+                        node_id,
+                        updated_variables,
+                        reason="tool",
                     )
+                if next_config:
                     return result, self._flow_managed_transition_config(
                         next_config,
                         triggering_user_text=(
@@ -735,6 +812,80 @@ class WorkflowBrain(BaseBrain):
             ),
         )
 
+    def _edge_tool(self, edge: dict, node_id: str) -> FlowsFunctionSchema:
+        """Expose one natural-language edge as an Agent-owned transition tool."""
+        registered_transition_id = self._state.transition_id
+
+        async def handler(_args, _flow_manager):
+            async with self._turn_lock:
+                if (
+                    self._state.current_node_id != node_id
+                    or self._state.transition_id != registered_transition_id
+                ):
+                    return {
+                        "status": "stale",
+                        "message": "当前 Agent 已经切换，本次跳转不再执行。",
+                    }
+                triggering_user_text = (
+                    self._state.pending_user_turn.text
+                    if self._state.pending_user_turn
+                    else ""
+                )
+                next_config = await self._follow_edge(
+                    edge,
+                    triggering_user_text=triggering_user_text,
+                )
+                return (
+                    {
+                        "status": "success",
+                        "targetNodeId": str(edge.get("target") or ""),
+                    },
+                    self._flow_managed_transition_config(
+                        next_config,
+                        triggering_user_text=triggering_user_text,
+                    ),
+                )
+
+        return FlowsFunctionSchema(
+            name=self._engine.edge_fn_name(edge),
+            description=self._engine.edge_description(edge),
+            properties={},
+            required=[],
+            handler=handler,
+            cancel_on_interruption=True,
+        )
+
+    async def _after_variables_changed(
+        self,
+        node_id: str,
+        changed: list[str],
+        *,
+        reason: str,
+    ) -> NodeConfig | None:
+        """Refresh the Agent and take a matching expression without an LLM."""
+        if not changed:
+            return None
+        await self._emit_variables(reason=reason, node_id=node_id, changed=changed)
+        if self._state.current_node_id != node_id:
+            return None
+        if self._engine.node_type(node_id) == "agent":
+            await self._refresh_agent_prompt(node_id)
+        edge = self._engine.deterministic_edge(
+            node_id,
+            self._store,
+            include_default=False,
+        )
+        if not edge:
+            return None
+        return await self._follow_edge(
+            edge,
+            triggering_user_text=(
+                self._state.pending_user_turn.text
+                if self._state.pending_user_turn
+                else ""
+            ),
+        )
+
     def _workflow_system_tool(
         self,
         tool: RuntimeTool,
@@ -779,18 +930,27 @@ class WorkflowBrain(BaseBrain):
                 changed = self._store.assign_declared_many(values)
             except DynamicVariableError as exc:
                 return {"status": "error", "message": f"状态更新失败: {exc}"}
-            if changed:
-                await self._emit_variables(
-                    reason="update_state",
-                    node_id=node_id,
-                    changed=changed,
-                )
-                await self._refresh_agent_prompt(node_id)
-            return {
+            result = {
                 "status": "success",
                 "changed": changed,
                 "variables": self._store.public_values(),
             }
+            async with self._turn_lock:
+                next_config = await self._after_variables_changed(
+                    node_id,
+                    changed,
+                    reason="update_state",
+                )
+            if next_config:
+                return result, self._flow_managed_transition_config(
+                    next_config,
+                    triggering_user_text=(
+                        self._state.pending_user_turn.text
+                        if self._state.pending_user_turn
+                        else ""
+                    ),
+                )
+            return result
 
         return FlowsFunctionSchema(
             name=tool.function_name,