wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add configurable client tools and photo input

Browse Source
This commit is contained in:
Xin Wang
2026-07-30 19:06:03 +08:00
parent 510a277b5a
commit 913435785e
24 changed files with 1802 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
backend/services/brains/prompt_brain.py
View File

@@ -20,6 +20,7 @@ from services.runtime_variables import (
DynamicVariableStore,
)
from services.tool_executor import ToolExecutionError, ToolExecutor
from services.tool_policy import policy_for_tool
class PromptBrain(BaseBrain):
@@ -50,17 +51,23 @@ class PromptBrain(BaseBrain):
async def setup(self, cfg: AssistantConfig, runtime: BrainRuntime) -> None:
self._runtime = runtime
self._tools.set_client_tools(runtime.client_tools)
self._waiting_for_generated_end_speech = False
schemas: list[FunctionSchema] = []
for tool in cfg.tools:
if tool.type == "end_call":
schema, handler = self._make_end_call_tool(tool, runtime)
elif tool.type in {"http", "mcp"}:
elif tool.type in {"http", "mcp", "client"}:
schema, handler = self._make_remote_tool(tool, runtime)
else:
continue
schemas.append(schema)
runtime.llm.register_function(tool.function_name, handler)
policy = policy_for_tool(tool)
runtime.llm.register_function(
tool.function_name,
handler,
cancel_on_interruption=policy.cancel_on_interruption,
)
runtime.set_tools(schemas)
def record_user_message(self, content: str) -> None:
@@ -99,16 +106,27 @@ class PromptBrain(BaseBrain):
def _make_remote_tool(self, tool, runtime: BrainRuntime):
properties, required = self._tools.schema_parts(tool)
self._tools.register_secrets(tool)
policy = policy_for_tool(tool)
async def call_http(params: FunctionCallParams) -> None:
async def return_result(params: FunctionCallParams, result: dict) -> None:
if not policy.runs_llm_after_result:
await params.result_callback(
result,
properties=FunctionCallResultProperties(run_llm=False),
)
else:
await params.result_callback(result)
async def call_tool(params: FunctionCallParams) -> None:
try:
result = await self._tools.execute(tool, dict(params.arguments or {}))
if result["updated_variables"]:
self._refresh_prompt()
await params.result_callback(result)
await return_result(params, result)
except (ToolExecutionError, ValueError) as exc:
await params.result_callback(
{"status": "error", "message": f"工具调用失败: {exc}"}
await return_result(
params,
{"status": "error", "message": f"工具调用失败: {exc}"},
)
schema = FunctionSchema(
@@ -117,7 +135,7 @@ class PromptBrain(BaseBrain):
properties=properties,
required=required,
)
return schema, call_http
return schema, call_tool
def _make_end_call_tool(self, tool, runtime: BrainRuntime):
config = (tool.definition or {}).get("config") or {}