diff --git a/api/app/routers/tools.py b/api/app/routers/tools.py index eab3f91..b0087a7 100644 --- a/api/app/routers/tools.py +++ b/api/app/routers/tools.py @@ -87,6 +87,17 @@ TOOL_REGISTRY = { "required": [] } }, + "voice_message_prompt": { + "name": "语音消息提示", + "description": "播报一条语音提示消息", + "parameters": { + "type": "object", + "properties": { + "msg": {"type": "string", "description": "要播报的消息文本"} + }, + "required": ["msg"] + } + }, } TOOL_CATEGORY_MAP = { @@ -97,6 +108,7 @@ TOOL_CATEGORY_MAP = { "turn_off_camera": "system", "increase_volume": "system", "decrease_volume": "system", + "voice_message_prompt": "system", } TOOL_ICON_MAP = { @@ -107,6 +119,7 @@ TOOL_ICON_MAP = { "turn_off_camera": "CameraOff", "increase_volume": "Volume2", "decrease_volume": "Volume2", + "voice_message_prompt": "Volume2", } TOOL_HTTP_DEFAULTS = { @@ -185,11 +198,16 @@ def _validate_query_http_config(*, category: str, tool_id: Optional[str], http_u def _seed_default_tools_if_empty(db: Session) -> None: - """Seed built-in tools only when tool_resources is empty.""" + """Ensure built-in tools exist in tool_resources without overriding custom edits.""" _ensure_tool_resource_schema(db) - if db.query(ToolResource).count() > 0: - return + existing_ids = { + str(item[0]) + for item in db.query(ToolResource.id).all() + } + changed = False for tool_id, payload in TOOL_REGISTRY.items(): + if tool_id in existing_ids: + continue http_defaults = TOOL_HTTP_DEFAULTS.get(tool_id, {}) db.add(ToolResource( id=tool_id, @@ -207,7 +225,9 @@ def _seed_default_tools_if_empty(db: Session) -> None: enabled=True, is_system=True, )) - db.commit() + changed = True + if changed: + db.commit() def recreate_tool_resources(db: Session) -> None: diff --git a/engine/core/duplex_pipeline.py b/engine/core/duplex_pipeline.py index 3f5be85..858e5cd 100644 --- a/engine/core/duplex_pipeline.py +++ b/engine/core/duplex_pipeline.py @@ -146,12 +146,24 @@ class DuplexPipeline: "required": [], }, }, + "voice_message_prompt": { + "name": "voice_message_prompt", + "description": "Speak a message prompt on client side", + "parameters": { + "type": "object", + "properties": { + "msg": {"type": "string", "description": "Message text to speak"}, + }, + "required": ["msg"], + }, + }, } _DEFAULT_CLIENT_EXECUTORS = frozenset({ "turn_on_camera", "turn_off_camera", "increase_volume", "decrease_volume", + "voice_message_prompt", }) def __init__( diff --git a/web/pages/Assistants.tsx b/web/pages/Assistants.tsx index 7a1ba6f..f3122e0 100644 --- a/web/pages/Assistants.tsx +++ b/web/pages/Assistants.tsx @@ -1682,6 +1682,13 @@ const TOOL_PARAMETER_HINTS: Record = { }, required: [], }, + voice_message_prompt: { + type: 'object', + properties: { + msg: { type: 'string', description: 'Message text to speak' }, + }, + required: ['msg'], + }, code_interpreter: { type: 'object', properties: { @@ -1694,6 +1701,15 @@ const TOOL_PARAMETER_HINTS: Record = { const getDefaultToolParameters = (toolId: string) => TOOL_PARAMETER_HINTS[toolId] || { type: 'object', properties: {} }; +const DEBUG_CLIENT_TOOLS = [ + { id: 'turn_on_camera', name: 'turn_on_camera', description: '打开摄像头' }, + { id: 'turn_off_camera', name: 'turn_off_camera', description: '关闭摄像头' }, + { id: 'increase_volume', name: 'increase_volume', description: '调高音量' }, + { id: 'decrease_volume', name: 'decrease_volume', description: '调低音量' }, + { id: 'voice_message_prompt', name: 'voice_message_prompt', description: '语音消息提示' }, +] as const; +const DEBUG_CLIENT_TOOL_ID_SET = new Set(DEBUG_CLIENT_TOOLS.map((item) => item.id)); + type DynamicVariableEntry = { id: string; key: string; @@ -2034,6 +2050,13 @@ export const DebugDrawer: React.FC<{ const [aecEnabled, setAecEnabled] = useState(() => localStorage.getItem('debug_audio_aec') !== '0'); const [nsEnabled, setNsEnabled] = useState(() => localStorage.getItem('debug_audio_ns') !== '0'); const [agcEnabled, setAgcEnabled] = useState(() => localStorage.getItem('debug_audio_agc') !== '0'); + const [clientToolEnabledMap, setClientToolEnabledMap] = useState>(() => { + const result: Record = {}; + for (const tool of DEBUG_CLIENT_TOOLS) { + result[tool.id] = localStorage.getItem(`debug_client_tool_enabled_${tool.id}`) !== '0'; + } + return result; + }); const micAudioCtxRef = useRef(null); const micSourceRef = useRef(null); @@ -2043,14 +2066,18 @@ export const DebugDrawer: React.FC<{ const userDraftIndexRef = useRef(null); const lastUserFinalRef = useRef(''); const debugVolumePercentRef = useRef(50); + const isClientToolEnabled = (toolId: string) => clientToolEnabledMap[toolId] !== false; const selectedToolSchemas = useMemo(() => { - const ids = assistant.tools || []; - if (!ids.length) return []; + const ids = Array.from(new Set([...(assistant.tools || []), ...DEBUG_CLIENT_TOOLS.map((item) => item.id)])); const byId = new Map(tools.map((t) => [t.id, t])); return ids.map((id) => { const item = byId.get(id); const toolId = item?.id || id; - const isClientTool = (item?.category || 'query') === 'system'; + const debugClientTool = DEBUG_CLIENT_TOOLS.find((tool) => tool.id === toolId); + if (debugClientTool && !isClientToolEnabled(toolId)) { + return null; + } + const isClientTool = debugClientTool ? true : (item?.category || 'query') === 'system'; const parameterSchema = (item?.parameterSchema && typeof item.parameterSchema === 'object') ? item.parameterSchema : getDefaultToolParameters(toolId); @@ -2063,12 +2090,12 @@ export const DebugDrawer: React.FC<{ ...(parameterDefaults && Object.keys(parameterDefaults).length > 0 ? { defaultArgs: parameterDefaults } : {}), function: { name: toolId, - description: item?.description || item?.name || id, + description: item?.description || item?.name || debugClientTool?.description || id, parameters: parameterSchema, }, }; - }); - }, [assistant.tools, tools]); + }).filter(Boolean) as Array>; + }, [assistant.tools, tools, clientToolEnabledMap]); const clearResponseTracking = () => { assistantDraftIndexRef.current = null; @@ -2129,6 +2156,12 @@ export const DebugDrawer: React.FC<{ localStorage.setItem('debug_audio_agc', agcEnabled ? '1' : '0'); }, [agcEnabled]); + useEffect(() => { + for (const tool of DEBUG_CLIENT_TOOLS) { + localStorage.setItem(`debug_client_tool_enabled_${tool.id}`, isClientToolEnabled(tool.id) ? '1' : '0'); + } + }, [clientToolEnabledMap]); + // Auto-scroll logic useEffect(() => { if (scrollRef.current) { @@ -2940,6 +2973,10 @@ export const DebugDrawer: React.FC<{ output: { message: `Unhandled client tool '${toolName}'` }, status: { code: 501, message: 'not_implemented' }, }; + if (DEBUG_CLIENT_TOOL_ID_SET.has(toolName) && !isClientToolEnabled(toolName)) { + resultPayload.output = { message: `Client tool '${toolName}' is disabled in debug settings` }; + resultPayload.status = { code: 503, message: 'tool_disabled' }; + } const sendToolResult = () => { ws.send( JSON.stringify({ @@ -2962,7 +2999,9 @@ export const DebugDrawer: React.FC<{ ]); }; try { - if (toolName === 'turn_on_camera') { + if (resultPayload.status.code === 503) { + // Keep disabled result as-is. + } else if (toolName === 'turn_on_camera') { navigator.mediaDevices .getUserMedia({ video: selectedCamera ? { deviceId: { exact: selectedCamera } } : true, @@ -3010,6 +3049,22 @@ export const DebugDrawer: React.FC<{ level: debugVolumePercentRef.current, }; resultPayload.status = { code: 200, message: 'ok' }; + } else if (toolName === 'voice_message_prompt') { + const msg = String(parsedArgs?.msg || '').trim(); + if (!msg) { + resultPayload.output = { message: "Missing required argument 'msg'" }; + resultPayload.status = { code: 422, message: 'invalid_arguments' }; + } else if (typeof window !== 'undefined' && 'speechSynthesis' in window) { + const utterance = new SpeechSynthesisUtterance(msg); + utterance.lang = 'zh-CN'; + window.speechSynthesis.cancel(); + window.speechSynthesis.speak(utterance); + resultPayload.output = { message: 'voice_prompt_sent', msg }; + resultPayload.status = { code: 200, message: 'ok' }; + } else { + resultPayload.output = { message: 'speech_synthesis_unavailable', msg }; + resultPayload.status = { code: 503, message: 'speech_output_unavailable' }; + } } } catch (err) { resultPayload.output = { @@ -3381,6 +3436,33 @@ export const DebugDrawer: React.FC<{ Auto Gain Control (AGC) +
+

Client Tools

+

滑块控制调试会话内客户端工具是否启用。

+
+ {DEBUG_CLIENT_TOOLS.map((tool) => { + const enabled = isClientToolEnabled(tool.id); + return ( +
+
+
{tool.name}
+
{tool.description}
+
+ +
+ ); + })} +
+

Dynamic Variables