From cdd8275e355e3e41dd37b07cbf1053e6526aa599 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Fri, 27 Feb 2026 16:04:49 +0800 Subject: [PATCH] Add voice_message_prompt tool to API and UI components. Update DuplexPipeline, Assistants, and DebugDrawer to support new tool functionality, including parameter validation and speech synthesis integration. Ensure existing tools are preserved during seeding process in the database. --- api/app/routers/tools.py | 28 ++++++++-- engine/core/duplex_pipeline.py | 12 +++++ web/pages/Assistants.tsx | 96 +++++++++++++++++++++++++++++++--- 3 files changed, 125 insertions(+), 11 deletions(-) diff --git a/api/app/routers/tools.py b/api/app/routers/tools.py index eab3f91..b0087a7 100644 --- a/api/app/routers/tools.py +++ b/api/app/routers/tools.py @@ -87,6 +87,17 @@ TOOL_REGISTRY = { "required": [] } }, + "voice_message_prompt": { + "name": "语音消息提示", + "description": "播报一条语音提示消息", + "parameters": { + "type": "object", + "properties": { + "msg": {"type": "string", "description": "要播报的消息文本"} + }, + "required": ["msg"] + } + }, } TOOL_CATEGORY_MAP = { @@ -97,6 +108,7 @@ TOOL_CATEGORY_MAP = { "turn_off_camera": "system", "increase_volume": "system", "decrease_volume": "system", + "voice_message_prompt": "system", } TOOL_ICON_MAP = { @@ -107,6 +119,7 @@ TOOL_ICON_MAP = { "turn_off_camera": "CameraOff", "increase_volume": "Volume2", "decrease_volume": "Volume2", + "voice_message_prompt": "Volume2", } TOOL_HTTP_DEFAULTS = { @@ -185,11 +198,16 @@ def _validate_query_http_config(*, category: str, tool_id: Optional[str], http_u def _seed_default_tools_if_empty(db: Session) -> None: - """Seed built-in tools only when tool_resources is empty.""" + """Ensure built-in tools exist in tool_resources without overriding custom edits.""" _ensure_tool_resource_schema(db) - if db.query(ToolResource).count() > 0: - return + existing_ids = { + str(item[0]) + for item in db.query(ToolResource.id).all() + } + changed = False for tool_id, payload in TOOL_REGISTRY.items(): + if tool_id in existing_ids: + continue http_defaults = TOOL_HTTP_DEFAULTS.get(tool_id, {}) db.add(ToolResource( id=tool_id, @@ -207,7 +225,9 @@ def _seed_default_tools_if_empty(db: Session) -> None: enabled=True, is_system=True, )) - db.commit() + changed = True + if changed: + db.commit() def recreate_tool_resources(db: Session) -> None: diff --git a/engine/core/duplex_pipeline.py b/engine/core/duplex_pipeline.py index 3f5be85..858e5cd 100644 --- a/engine/core/duplex_pipeline.py +++ b/engine/core/duplex_pipeline.py @@ -146,12 +146,24 @@ class DuplexPipeline: "required": [], }, }, + "voice_message_prompt": { + "name": "voice_message_prompt", + "description": "Speak a message prompt on client side", + "parameters": { + "type": "object", + "properties": { + "msg": {"type": "string", "description": "Message text to speak"}, + }, + "required": ["msg"], + }, + }, } _DEFAULT_CLIENT_EXECUTORS = frozenset({ "turn_on_camera", "turn_off_camera", "increase_volume", "decrease_volume", + "voice_message_prompt", }) def __init__( diff --git a/web/pages/Assistants.tsx b/web/pages/Assistants.tsx index 7a1ba6f..f3122e0 100644 --- a/web/pages/Assistants.tsx +++ b/web/pages/Assistants.tsx @@ -1682,6 +1682,13 @@ const TOOL_PARAMETER_HINTS: Record = { }, required: [], }, + voice_message_prompt: { + type: 'object', + properties: { + msg: { type: 'string', description: 'Message text to speak' }, + }, + required: ['msg'], + }, code_interpreter: { type: 'object', properties: { @@ -1694,6 +1701,15 @@ const TOOL_PARAMETER_HINTS: Record = { const getDefaultToolParameters = (toolId: string) => TOOL_PARAMETER_HINTS[toolId] || { type: 'object', properties: {} }; +const DEBUG_CLIENT_TOOLS = [ + { id: 'turn_on_camera', name: 'turn_on_camera', description: '打开摄像头' }, + { id: 'turn_off_camera', name: 'turn_off_camera', description: '关闭摄像头' }, + { id: 'increase_volume', name: 'increase_volume', description: '调高音量' }, + { id: 'decrease_volume', name: 'decrease_volume', description: '调低音量' }, + { id: 'voice_message_prompt', name: 'voice_message_prompt', description: '语音消息提示' }, +] as const; +const DEBUG_CLIENT_TOOL_ID_SET = new Set(DEBUG_CLIENT_TOOLS.map((item) => item.id)); + type DynamicVariableEntry = { id: string; key: string; @@ -2034,6 +2050,13 @@ export const DebugDrawer: React.FC<{ const [aecEnabled, setAecEnabled] = useState(() => localStorage.getItem('debug_audio_aec') !== '0'); const [nsEnabled, setNsEnabled] = useState(() => localStorage.getItem('debug_audio_ns') !== '0'); const [agcEnabled, setAgcEnabled] = useState(() => localStorage.getItem('debug_audio_agc') !== '0'); + const [clientToolEnabledMap, setClientToolEnabledMap] = useState>(() => { + const result: Record = {}; + for (const tool of DEBUG_CLIENT_TOOLS) { + result[tool.id] = localStorage.getItem(`debug_client_tool_enabled_${tool.id}`) !== '0'; + } + return result; + }); const micAudioCtxRef = useRef(null); const micSourceRef = useRef(null); @@ -2043,14 +2066,18 @@ export const DebugDrawer: React.FC<{ const userDraftIndexRef = useRef(null); const lastUserFinalRef = useRef(''); const debugVolumePercentRef = useRef(50); + const isClientToolEnabled = (toolId: string) => clientToolEnabledMap[toolId] !== false; const selectedToolSchemas = useMemo(() => { - const ids = assistant.tools || []; - if (!ids.length) return []; + const ids = Array.from(new Set([...(assistant.tools || []), ...DEBUG_CLIENT_TOOLS.map((item) => item.id)])); const byId = new Map(tools.map((t) => [t.id, t])); return ids.map((id) => { const item = byId.get(id); const toolId = item?.id || id; - const isClientTool = (item?.category || 'query') === 'system'; + const debugClientTool = DEBUG_CLIENT_TOOLS.find((tool) => tool.id === toolId); + if (debugClientTool && !isClientToolEnabled(toolId)) { + return null; + } + const isClientTool = debugClientTool ? true : (item?.category || 'query') === 'system'; const parameterSchema = (item?.parameterSchema && typeof item.parameterSchema === 'object') ? item.parameterSchema : getDefaultToolParameters(toolId); @@ -2063,12 +2090,12 @@ export const DebugDrawer: React.FC<{ ...(parameterDefaults && Object.keys(parameterDefaults).length > 0 ? { defaultArgs: parameterDefaults } : {}), function: { name: toolId, - description: item?.description || item?.name || id, + description: item?.description || item?.name || debugClientTool?.description || id, parameters: parameterSchema, }, }; - }); - }, [assistant.tools, tools]); + }).filter(Boolean) as Array>; + }, [assistant.tools, tools, clientToolEnabledMap]); const clearResponseTracking = () => { assistantDraftIndexRef.current = null; @@ -2129,6 +2156,12 @@ export const DebugDrawer: React.FC<{ localStorage.setItem('debug_audio_agc', agcEnabled ? '1' : '0'); }, [agcEnabled]); + useEffect(() => { + for (const tool of DEBUG_CLIENT_TOOLS) { + localStorage.setItem(`debug_client_tool_enabled_${tool.id}`, isClientToolEnabled(tool.id) ? '1' : '0'); + } + }, [clientToolEnabledMap]); + // Auto-scroll logic useEffect(() => { if (scrollRef.current) { @@ -2940,6 +2973,10 @@ export const DebugDrawer: React.FC<{ output: { message: `Unhandled client tool '${toolName}'` }, status: { code: 501, message: 'not_implemented' }, }; + if (DEBUG_CLIENT_TOOL_ID_SET.has(toolName) && !isClientToolEnabled(toolName)) { + resultPayload.output = { message: `Client tool '${toolName}' is disabled in debug settings` }; + resultPayload.status = { code: 503, message: 'tool_disabled' }; + } const sendToolResult = () => { ws.send( JSON.stringify({ @@ -2962,7 +2999,9 @@ export const DebugDrawer: React.FC<{ ]); }; try { - if (toolName === 'turn_on_camera') { + if (resultPayload.status.code === 503) { + // Keep disabled result as-is. + } else if (toolName === 'turn_on_camera') { navigator.mediaDevices .getUserMedia({ video: selectedCamera ? { deviceId: { exact: selectedCamera } } : true, @@ -3010,6 +3049,22 @@ export const DebugDrawer: React.FC<{ level: debugVolumePercentRef.current, }; resultPayload.status = { code: 200, message: 'ok' }; + } else if (toolName === 'voice_message_prompt') { + const msg = String(parsedArgs?.msg || '').trim(); + if (!msg) { + resultPayload.output = { message: "Missing required argument 'msg'" }; + resultPayload.status = { code: 422, message: 'invalid_arguments' }; + } else if (typeof window !== 'undefined' && 'speechSynthesis' in window) { + const utterance = new SpeechSynthesisUtterance(msg); + utterance.lang = 'zh-CN'; + window.speechSynthesis.cancel(); + window.speechSynthesis.speak(utterance); + resultPayload.output = { message: 'voice_prompt_sent', msg }; + resultPayload.status = { code: 200, message: 'ok' }; + } else { + resultPayload.output = { message: 'speech_synthesis_unavailable', msg }; + resultPayload.status = { code: 503, message: 'speech_output_unavailable' }; + } } } catch (err) { resultPayload.output = { @@ -3381,6 +3436,33 @@ export const DebugDrawer: React.FC<{ Auto Gain Control (AGC) +
+

Client Tools

+

滑块控制调试会话内客户端工具是否启用。

+
+ {DEBUG_CLIENT_TOOLS.map((tool) => { + const enabled = isClientToolEnabled(tool.id); + return ( +
+
+
{tool.name}
+
{tool.description}
+
+ +
+ ); + })} +
+

Dynamic Variables