wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add system tools and state updates

Browse Source
This commit is contained in:
Xin Wang
2026-08-04 15:27:39 +08:00
parent 5bf5987fe4
commit 731a372df9
33 changed files with 1787 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

180
backend/services/brains/prompt_brain.py
View File

@@ -43,14 +43,14 @@ from services.message_stage import (
MessageStageRunner,
MessageStageSpec,
)
from services.runtime_variables import DynamicVariableStore
from services.runtime_variables import DynamicVariableError, DynamicVariableStore
from services.system_tools import SYSTEM_TOOL_KINDS, state_update_properties
from services.tool_executor import ToolExecutionError, ToolExecutor
from services.tool_policy import policy_for_tool
PREFLIGHT_TIMEOUT_SECONDS = 30
class PromptBrain(BaseBrain):
spec = BrainSpec(
type="prompt",
@@ -123,22 +123,37 @@ class PromptBrain(BaseBrain):
set(cfg.llm_tool_ids) if cfg.llm_tool_ids is not None else None
)
schemas: list[FunctionSchema] = []
registered_names: set[str] = set()
for tool in cfg.tools:
if llm_tool_ids is not None and tool.id not in llm_tool_ids:
continue
if tool.type == "end_call":
if tool.type == "system":
schema, handler = self._make_end_call_tool(tool, runtime)
elif tool.type in {"http", "mcp", "client"}:
schema, handler = self._make_remote_tool(tool, runtime)
else:
continue
schemas.append(schema)
registered_names.add(schema.name)
policy = policy_for_tool(tool)
runtime.llm.register_function(
tool.function_name,
handler,
cancel_on_interruption=policy.cancel_on_interruption,
)
for kind in cfg.system_tools or []:
if kind not in SYSTEM_TOOL_KINDS:
logger.warning(f"忽略未知系统工具: {kind}")
continue
schema, handler = self._make_system_tool(kind, runtime)
if schema.name in registered_names:
logger.warning(
f"跳过系统工具 {schema.name}: 与已绑定工具函数名冲突"
)
continue
registered_names.add(schema.name)
schemas.append(schema)
runtime.llm.register_function(schema.name, handler)
runtime.set_tools(schemas)
async def run_preflight(self) -> None:
@@ -582,3 +597,162 @@ class PromptBrain(BaseBrain):
required=["reason"] if capture_reason else [],
)
return schema, end_call
# ---------- 内置系统工具(助手配置 system_tools 开关) ----------
def _make_system_tool(self, kind: str, runtime: BrainRuntime):
if kind == "end_conversation":
return self._make_end_conversation_tool(runtime)
if kind == "update_state":
return self._make_update_state_tool(runtime)
if kind == "skip_turn":
return self._make_skip_turn_tool()
if kind == "request_human_handoff":
return self._make_handoff_tool(runtime)
raise ValueError(f"未知系统工具: {kind}")
def _make_end_conversation_tool(self, runtime: BrainRuntime):
"""结束本次对话,等待模型已生成的告别语播完后再挂断。"""
async def end_conversation(params: FunctionCallParams) -> None:
reason = str(
params.arguments.get("reason") or "end_conversation"
).strip()
self._waiting_for_generated_end_speech = True
runtime.call_end.begin(reason)
await params.result_callback(
{"status": "success", "action": "ending_call"},
properties=FunctionCallResultProperties(run_llm=False),
)
schema = FunctionSchema(
name="end_conversation",
description=(
"礼貌地结束本次对话。当用户明确告别、表示任务已完成"
"或要求挂断时调用。"
),
properties={
"reason": {
"type": "string",
"description": "结束对话的简短原因。",
}
},
required=[],
)
return schema, end_conversation
def _make_update_state_tool(self, runtime: BrainRuntime):
"""更新已声明的动态变量(会话状态),并让模型继续当前回答。"""
writable = state_update_properties(self._cfg.dynamic_variable_definitions)
async def update_state(params: FunctionCallParams) -> None:
state = dict(params.arguments or {})
try:
changed = self._store.assign_declared_many(state)
except DynamicVariableError as exc:
await params.result_callback(
{"status": "error", "message": f"状态更新失败: {exc}"}
)
return
if changed:
self._refresh_prompt()
await runtime.queue_frame(
OutputTransportMessageUrgentFrame(
message={
"type": "session-variables",
"reason": "update_state",
"variables": self._store.public_values(),
"changed": changed,
}
)
)
await params.result_callback(
{
"status": "success",
"changed": changed,
"variables": self._store.public_values(),
}
)
schema = FunctionSchema(
name="update_state",
description=(
"静默更新本次对话中已经声明并明确列出的动态变量。"
"只提交本轮获得或确认的信息，更新后继续当前回答。"
),
properties=writable,
required=[],
)
return schema, update_state
def _make_skip_turn_tool(self):
"""跳过当前轮次,不生成任何语音回复。"""
async def skip_turn(params: FunctionCallParams) -> None:
reason = str(params.arguments.get("reason") or "").strip()
result = {"status": "success", "action": "skip_turn"}
if reason:
result["reason"] = reason
await params.result_callback(
result,
properties=FunctionCallResultProperties(run_llm=False),
)
schema = FunctionSchema(
name="skip_turn",
description=(
"跳过当前轮次,不生成任何语音回复。"
"仅当用户明确要求稍等、话还没说完，或输入可确认只是噪音时调用。"
),
properties={
"reason": {
"type": "string",
"description": "跳过本轮的原因(可选)。",
}
},
required=[],
)
return schema, skip_turn
def _make_handoff_tool(self, runtime: BrainRuntime):
"""提交人工接管请求；请求完成前保持当前 AI 会话可用。"""
async def request_human_handoff(params: FunctionCallParams) -> None:
reason = str(
params.arguments.get("reason") or "human_handoff"
).strip()
await runtime.queue_frame(
OutputTransportMessageUrgentFrame(
message={
"type": "handoff-requested",
"source": "prompt-system-tool",
"reason": reason,
"message": "用户请求转接人工服务。",
}
)
)
await params.result_callback(
{
"status": "requested",
"action": "human_handoff_requested",
"message": "人工接管请求已提交，请告知用户正在等待人工响应。",
}
)
schema = FunctionSchema(
name="request_human_handoff",
description=(
"提交人工接管请求。当用户明确要求人工服务、投诉升级或 AI 无法"
"解决时调用。该工具只提交请求，不代表人工已经接通；调用后继续"
"回复用户并说明正在等待人工响应。"
),
properties={
"reason": {
"type": "string",
"description": "转接人工的原因。",
}
},
required=[],
)
return schema, request_human_handoff

238
backend/services/brains/workflow_brain.py
View File

@@ -56,7 +56,8 @@ from services.message_stage import (
MessageStageRunner,
MessageStageSpec,
)
from services.runtime_variables import DynamicVariableStore
from services.runtime_variables import DynamicVariableError, DynamicVariableStore
from services.system_tools import SYSTEM_TOOL_KINDS, state_update_properties
from services.tool_executor import ToolExecutionError, ToolExecutor
from services.tool_policy import policy_for_tool
from services.workflow.agent import WorkflowAgentStage
@@ -184,6 +185,7 @@ class WorkflowBrain(BaseBrain):
self._output: WorkflowOutput | None = None
self._agent_stage: WorkflowAgentStage | None = None
self._ended = False
self._waiting_for_generated_end_speech = False
self._next_message_token = 1
self._pending_message: _MessageContinuation | None = None
@@ -229,6 +231,7 @@ class WorkflowBrain(BaseBrain):
runtime=runtime,
)
self._ended = False
self._waiting_for_generated_end_speech = False
self._next_message_token = 1
self._pending_message = None
self._manager = ConfiguredFlowManager(
@@ -494,18 +497,30 @@ class WorkflowBrain(BaseBrain):
return None
return decision.edge
async def on_assistant_text_start(self, _turn_id: str) -> None:
if self._runtime is not None:
self._runtime.call_end.begin_response()
async def on_assistant_text_end(
self,
_turn_id: str,
content: str,
interrupted: bool,
) -> None:
if not content or interrupted or self._ended:
return
self._store.record("agent", content, completed_agent_turn=True)
self._state.consume_user_turn()
if self._engine.node_type(self._state.current_node_id) == "agent":
self._state.status = WorkflowStatus.WAITING_USER
if content and not interrupted and not self._ended:
self._store.record("agent", content, completed_agent_turn=True)
self._state.consume_user_turn()
if self._engine.node_type(self._state.current_node_id) == "agent":
self._state.status = WorkflowStatus.WAITING_USER
if (
self._waiting_for_generated_end_speech
and self._runtime is not None
and self._runtime.call_end.ending
):
self._waiting_for_generated_end_speech = False
await self._runtime.call_end.finish_after_current_speech(
has_text=bool(content.strip()) and not interrupted
)
async def _refresh_agent_prompt(self, node_id: str) -> None:
await self._require_agent_stage().refresh_prompt(node_id)
@@ -528,15 +543,43 @@ class WorkflowBrain(BaseBrain):
) -> NodeConfig:
stage = self._engine.agent_stage_config(node_id)
functions: list[FlowsFunctionSchema] = []
registered_names: set[str] = set()
def append_function(function: FlowsFunctionSchema | None) -> None:
if function is None:
return
function_name = getattr(function, "name", "")
if not function_name:
# Runtime-provided global functions are opaque in a few
# adapters; FlowManager remains responsible for those names.
functions.append(function)
return
if function_name in registered_names:
logger.warning(
f"跳过 Agent {node_id} 的函数 {function_name}: 函数名冲突"
)
return
registered_names.add(function_name)
functions.append(function)
for tool_id in stage.tool_ids:
tool = self._tool_by_id.get(str(tool_id))
if tool and tool.type in {"http", "mcp", "client"}:
functions.append(self._flow_tool(tool, node_id))
knowledge_function = self._knowledge_function(node_id)
if knowledge_function:
functions.append(knowledge_function)
append_function(self._flow_tool(tool, node_id))
append_function(self._knowledge_function(node_id))
if stage.vision_enabled and self._require_runtime().vision_function:
functions.append(self._require_runtime().vision_function)
append_function(self._require_runtime().vision_function)
for kind in stage.system_tools:
if kind not in SYSTEM_TOOL_KINDS:
logger.warning(f"忽略 Agent {node_id} 的未知系统工具: {kind}")
continue
append_function(
self._workflow_system_tool(
kind,
node_id=node_id,
state_variable_names=stage.state_variable_names,
)
)
return self._require_agent_stage().node_config(
node_id,
functions=functions,
@@ -693,6 +736,159 @@ class WorkflowBrain(BaseBrain):
),
)
def _workflow_system_tool(
self,
kind: str,
*,
node_id: str,
state_variable_names: tuple[str, ...],
) -> FlowsFunctionSchema:
"""Build one platform-owned tool scoped to the active Agent node."""
if kind == "update_state":
return self._workflow_update_state_tool(
node_id,
state_variable_names=state_variable_names,
)
if kind == "skip_turn":
return self._workflow_skip_turn_tool()
if kind == "request_human_handoff":
return self._workflow_handoff_tool(node_id)
if kind == "end_conversation":
return self._workflow_end_conversation_tool()
raise ValueError(f"未知系统工具: {kind}")
def _workflow_update_state_tool(
self,
node_id: str,
*,
state_variable_names: tuple[str, ...],
) -> FlowsFunctionSchema:
allowed = frozenset(state_variable_names)
async def handler(args, _flow_manager):
values = dict(args or {})
unauthorized = sorted(set(values) - allowed)
if unauthorized:
return {
"status": "error",
"message": "状态变量未获当前节点授权: " + ",".join(unauthorized),
}
try:
changed = self._store.assign_declared_many(values)
except DynamicVariableError as exc:
return {"status": "error", "message": f"状态更新失败: {exc}"}
if changed:
await self._emit_variables(
reason="update_state",
node_id=node_id,
changed=changed,
)
await self._refresh_agent_prompt(node_id)
return {
"status": "success",
"changed": changed,
"variables": self._store.public_values(),
}
return FlowsFunctionSchema(
name="update_state",
description=(
"静默更新当前阶段明确授权的动态变量。"
"只提交本轮获得或确认的信息，更新后继续当前回答。"
),
properties=state_update_properties(
self._cfg.dynamic_variable_definitions if self._cfg else {},
allowed_names=state_variable_names,
),
required=[],
handler=handler,
)
@staticmethod
def _workflow_skip_turn_tool() -> FlowsFunctionSchema:
async def handler(args, _flow_manager):
reason = str((args or {}).get("reason") or "").strip()
result = {"status": "success", "action": "skip_turn"}
if reason:
result["reason"] = reason
return result
setattr(handler, "_suppress_followup_llm", True)
return FlowsFunctionSchema(
name="skip_turn",
description=(
"跳过当前轮次，不生成任何语音回复。"
"仅当用户明确要求稍等、话还没说完，或输入可确认只是噪音时调用。"
),
properties={
"reason": {
"type": "string",
"description": "跳过本轮的原因（可选）。",
}
},
required=[],
handler=handler,
)
def _workflow_handoff_tool(self, node_id: str) -> FlowsFunctionSchema:
async def handler(args, _flow_manager):
reason = str((args or {}).get("reason") or "human_handoff").strip()
await self._require_runtime().queue_frame(
OutputTransportMessageUrgentFrame(
message={
"type": "handoff-requested",
"source": "workflow-system-tool",
"nodeId": node_id,
"reason": reason,
"message": "用户请求转接人工服务。",
}
)
)
self._store.values["system__handoff_status"] = "requested"
return {
"status": "requested",
"action": "human_handoff_requested",
"message": "人工接管请求已提交，请告知用户正在等待人工响应。",
}
return FlowsFunctionSchema(
name="request_human_handoff",
description=(
"提交人工接管请求。当用户明确要求人工服务、投诉升级或 AI 无法"
"解决时调用。该工具只提交请求，不代表人工已经接通；调用后继续"
"回复用户并说明正在等待人工响应。"
),
properties={
"reason": {"type": "string", "description": "转接人工的原因。"}
},
required=[],
handler=handler,
)
def _workflow_end_conversation_tool(self) -> FlowsFunctionSchema:
async def handler(args, _flow_manager):
reason = str((args or {}).get("reason") or "end_conversation").strip()
self._waiting_for_generated_end_speech = True
self._require_runtime().call_end.begin(reason)
return {"status": "success", "action": "ending_call"}
setattr(handler, "_suppress_followup_llm", True)
return FlowsFunctionSchema(
name="end_conversation",
description=(
"礼貌地结束本次对话。当用户明确告别、表示任务已完成"
"或要求挂断时调用。"
),
properties={
"reason": {
"type": "string",
"description": "结束对话的简短原因。",
}
},
required=[],
handler=handler,
)
def _flow_managed_transition_config(
self,
node_config: NodeConfig,
@@ -838,6 +1034,8 @@ class WorkflowBrain(BaseBrain):
outcome = await self._enter_action(node_id)
if not outcome.should_route:
return self._passive_node_config(node_id, context_messages)
elif node_type == "update_state":
await self._enter_update_state(node_id)
elif node_type == "message":
self._prepare_message_continuation(
node_id,
@@ -862,6 +1060,22 @@ class WorkflowBrain(BaseBrain):
node_id = str(edge.get("target") or "")
raise RuntimeError("工作流连续自动跳转超过安全上限")
async def _enter_update_state(self, node_id: str) -> None:
"""Apply one deterministic, atomic dynamic-variable update."""
self._state.enter(node_id, WorkflowStatus.RUNNING_ACTION)
await self._emit_node_active(node_id)
raw_assignments = self._engine.data(node_id).get("assignments") or {}
rendered = self._store.render_data(raw_assignments)
if not isinstance(rendered, dict):
raise DynamicVariableError("Update State 节点赋值必须是对象")
changed = self._store.assign_declared_many(rendered)
if changed:
await self._emit_variables(
reason="update_state",
node_id=node_id,
changed=changed,
)
def _prepare_message_continuation(
self,
node_id: str,