wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add configurable client tools and photo input

Browse Source
This commit is contained in:
Xin Wang
2026-07-30 19:06:03 +08:00
parent 510a277b5a
commit 913435785e
24 changed files with 1802 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/services/brains/base.py
View File

@@ -17,6 +17,7 @@ from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.frames.frames import Frame
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.frame_processor import FrameProcessor
from services.client_tools import ClientToolPort
GREETING_CONTEXT_MARKER = "[会话事实：助手开场白已播放]"
@@ -74,6 +75,7 @@ class BrainRuntime:
set_system_prompt: Callable[[str], None]
set_tools: Callable[[list[FunctionSchema] | None], None]
call_end: CallEndPort
client_tools: ClientToolPort | None = None
worker: Any = None
context_aggregator: Any = None
transport: Any = None

32
backend/services/brains/prompt_brain.py
View File

@@ -20,6 +20,7 @@ from services.runtime_variables import (
DynamicVariableStore,
)
from services.tool_executor import ToolExecutionError, ToolExecutor
from services.tool_policy import policy_for_tool
class PromptBrain(BaseBrain):
@@ -50,17 +51,23 @@ class PromptBrain(BaseBrain):
async def setup(self, cfg: AssistantConfig, runtime: BrainRuntime) -> None:
self._runtime = runtime
self._tools.set_client_tools(runtime.client_tools)
self._waiting_for_generated_end_speech = False
schemas: list[FunctionSchema] = []
for tool in cfg.tools:
if tool.type == "end_call":
schema, handler = self._make_end_call_tool(tool, runtime)
elif tool.type in {"http", "mcp"}:
elif tool.type in {"http", "mcp", "client"}:
schema, handler = self._make_remote_tool(tool, runtime)
else:
continue
schemas.append(schema)
runtime.llm.register_function(tool.function_name, handler)
policy = policy_for_tool(tool)
runtime.llm.register_function(
tool.function_name,
handler,
cancel_on_interruption=policy.cancel_on_interruption,
)
runtime.set_tools(schemas)
def record_user_message(self, content: str) -> None:
@@ -99,16 +106,27 @@ class PromptBrain(BaseBrain):
def _make_remote_tool(self, tool, runtime: BrainRuntime):
properties, required = self._tools.schema_parts(tool)
self._tools.register_secrets(tool)
policy = policy_for_tool(tool)
async def call_http(params: FunctionCallParams) -> None:
async def return_result(params: FunctionCallParams, result: dict) -> None:
if not policy.runs_llm_after_result:
await params.result_callback(
result,
properties=FunctionCallResultProperties(run_llm=False),
)
else:
await params.result_callback(result)
async def call_tool(params: FunctionCallParams) -> None:
try:
result = await self._tools.execute(tool, dict(params.arguments or {}))
if result["updated_variables"]:
self._refresh_prompt()
await params.result_callback(result)
await return_result(params, result)
except (ToolExecutionError, ValueError) as exc:
await params.result_callback(
{"status": "error", "message": f"工具调用失败: {exc}"}
await return_result(
params,
{"status": "error", "message": f"工具调用失败: {exc}"},
)
schema = FunctionSchema(
@@ -117,7 +135,7 @@ class PromptBrain(BaseBrain):
properties=properties,
required=required,
)
return schema, call_http
return schema, call_tool
def _make_end_call_tool(self, tool, runtime: BrainRuntime):
config = (tool.definition or {}).get("config") or {}

55
backend/services/brains/workflow_brain.py
View File

@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio
from copy import deepcopy
from dataclasses import replace
from typing import Any
from loguru import logger
@@ -23,10 +24,15 @@ from pipecat.frames.frames import (
)
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.frame_processor import FrameProcessor
from pipecat.services.llm_service import (
FunctionCallParams,
FunctionCallResultProperties,
)
from services.brains.base import BaseBrain, BrainRuntime, BrainSpec
from services.knowledge import search as search_knowledge
from services.runtime_variables import DynamicVariableStore
from services.tool_executor import ToolExecutionError, ToolExecutor
from services.tool_policy import policy_for_tool
from services.workflow.agent import WorkflowAgentStage
from services.workflow.models import (
RouteStatus,
@@ -42,6 +48,38 @@ from services.workflow_router import WorkflowLLMRouter
MAX_AUTOMATIC_HOPS = 50
class ConfiguredFlowManager(FlowManager):
"""Preserve Flow transitions while suppressing late async-tool replies."""
async def _create_transition_func(self, name, handler):
transition = await super()._create_transition_func(name, handler)
if not getattr(handler, "_suppress_followup_llm", False):
return transition
async def configured_transition(params: FunctionCallParams) -> None:
original_callback = params.result_callback
async def result_callback(result, *, properties=None):
if properties and properties.on_context_updated:
# Deterministic Workflow transitions already use run_llm=False
# and must retain their context-updated callback.
configured_properties = properties
elif properties:
configured_properties = replace(properties, run_llm=False)
else:
configured_properties = FunctionCallResultProperties(
run_llm=False
)
await original_callback(
result,
properties=configured_properties,
)
await transition(replace(params, result_callback=result_callback))
return configured_transition
class WorkflowBrain(BaseBrain):
spec = BrainSpec(
type="workflow",
@@ -100,7 +138,7 @@ class WorkflowBrain(BaseBrain):
self._cfg = cfg
self._runtime = runtime
self._store = DynamicVariableStore.from_config(cfg)
self._tools = ToolExecutor(self._store)
self._tools = ToolExecutor(self._store, client_tools=runtime.client_tools)
self._tool_by_id = {tool.id: tool for tool in cfg.tools}
self._router = WorkflowLLMRouter(cfg)
self._edge_evaluator = WorkflowEdgeEvaluator(
@@ -120,7 +158,7 @@ class WorkflowBrain(BaseBrain):
self._ended = False
self._greeting_context_message = None
self._startup_waiting_for_greeting = False
self._manager = FlowManager(
self._manager = ConfiguredFlowManager(
worker=runtime.worker,
llm=runtime.llm,
context_aggregator=runtime.context_aggregator,
@@ -328,7 +366,7 @@ class WorkflowBrain(BaseBrain):
functions: list[FlowsFunctionSchema] = []
for tool_id in stage.tool_ids:
tool = self._tool_by_id.get(str(tool_id))
if tool and tool.type in {"http", "mcp"}:
if tool and tool.type in {"http", "mcp", "client"}:
functions.append(self._flow_tool(tool, node_id))
knowledge_function = self._knowledge_function(node_id)
if knowledge_function:
@@ -409,6 +447,7 @@ class WorkflowBrain(BaseBrain):
def _flow_tool(self, tool: RuntimeTool, node_id: str) -> FlowsFunctionSchema:
properties, required = self._tools.schema_parts(tool)
self._tools.register_secrets(tool)
policy = policy_for_tool(tool)
async def handler(args, _flow_manager):
transition_id = self._state.transition_id
@@ -456,13 +495,21 @@ class WorkflowBrain(BaseBrain):
)
return result
if not policy.runs_llm_after_result:
setattr(handler, "_suppress_followup_llm", True)
return FlowsFunctionSchema(
name=tool.function_name,
description=tool.description or f"调用 {tool.name}",
properties=properties,
required=required,
handler=handler,
cancel_on_interruption=True,
cancel_on_interruption=policy.cancel_on_interruption,
timeout_secs=(
float(((tool.definition or {}).get("config") or {}).get("timeout_seconds") or 3)
if tool.type == "client"
else None
),
)
def _flow_managed_transition_config(