Add voice_message_prompt tool to API and UI components. Update DuplexPipeline, Assistants, and DebugDrawer to support new tool functionality, including parameter validation and speech synthesis integration. Ensure existing tools are preserved during seeding process in the database.

2026-02-27 16:04:49 +08:00
parent b035e023c4
commit cdd8275e35
3 changed files with 125 additions and 11 deletions
--- a/api/app/routers/tools.py
+++ b/api/app/routers/tools.py
@@ -87,6 +87,17 @@ TOOL_REGISTRY = {
            "required": []
        }
    },
+    "voice_message_prompt": {
+        "name": "语音消息提示",
+        "description": "播报一条语音提示消息",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "msg": {"type": "string", "description": "要播报的消息文本"}
+            },
+            "required": ["msg"]
+        }
+    },
 }

 TOOL_CATEGORY_MAP = {
@@ -97,6 +108,7 @@ TOOL_CATEGORY_MAP = {
    "turn_off_camera": "system",
    "increase_volume": "system",
    "decrease_volume": "system",
+    "voice_message_prompt": "system",
 }

 TOOL_ICON_MAP = {
@@ -107,6 +119,7 @@ TOOL_ICON_MAP = {
    "turn_off_camera": "CameraOff",
    "increase_volume": "Volume2",
    "decrease_volume": "Volume2",
+    "voice_message_prompt": "Volume2",
 }

 TOOL_HTTP_DEFAULTS = {
@@ -185,11 +198,16 @@ def _validate_query_http_config(*, category: str, tool_id: Optional[str], http_u


 def _seed_default_tools_if_empty(db: Session) -> None:
-    """Seed built-in tools only when tool_resources is empty."""
+    """Ensure built-in tools exist in tool_resources without overriding custom edits."""
    _ensure_tool_resource_schema(db)
-    if db.query(ToolResource).count() > 0:
-        return
+    existing_ids = {
+        str(item[0])
+        for item in db.query(ToolResource.id).all()
+    }
+    changed = False
    for tool_id, payload in TOOL_REGISTRY.items():
+        if tool_id in existing_ids:
+            continue
        http_defaults = TOOL_HTTP_DEFAULTS.get(tool_id, {})
        db.add(ToolResource(
            id=tool_id,
@@ -207,7 +225,9 @@ def _seed_default_tools_if_empty(db: Session) -> None:
            enabled=True,
            is_system=True,
        ))
-    db.commit()
+        changed = True
+    if changed:
+        db.commit()


 def recreate_tool_resources(db: Session) -> None:
--- a/engine/core/duplex_pipeline.py
+++ b/engine/core/duplex_pipeline.py
@@ -146,12 +146,24 @@ class DuplexPipeline:
                "required": [],
            },
        },
+        "voice_message_prompt": {
+            "name": "voice_message_prompt",
+            "description": "Speak a message prompt on client side",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "msg": {"type": "string", "description": "Message text to speak"},
+                },
+                "required": ["msg"],
+            },
+        },
    }
    _DEFAULT_CLIENT_EXECUTORS = frozenset({
        "turn_on_camera",
        "turn_off_camera",
        "increase_volume",
        "decrease_volume",
+        "voice_message_prompt",
    })

    def __init__(
--- a/web/pages/Assistants.tsx
+++ b/web/pages/Assistants.tsx
@@ -1682,6 +1682,13 @@ const TOOL_PARAMETER_HINTS: Record<string, any> = {
    },
    required: [],
  },
+  voice_message_prompt: {
+    type: 'object',
+    properties: {
+      msg: { type: 'string', description: 'Message text to speak' },
+    },
+    required: ['msg'],
+  },
  code_interpreter: {
    type: 'object',
    properties: {
@@ -1694,6 +1701,15 @@ const TOOL_PARAMETER_HINTS: Record<string, any> = {
 const getDefaultToolParameters = (toolId: string) =>
  TOOL_PARAMETER_HINTS[toolId] || { type: 'object', properties: {} };

+const DEBUG_CLIENT_TOOLS = [
+  { id: 'turn_on_camera', name: 'turn_on_camera', description: '打开摄像头' },
+  { id: 'turn_off_camera', name: 'turn_off_camera', description: '关闭摄像头' },
+  { id: 'increase_volume', name: 'increase_volume', description: '调高音量' },
+  { id: 'decrease_volume', name: 'decrease_volume', description: '调低音量' },
+  { id: 'voice_message_prompt', name: 'voice_message_prompt', description: '语音消息提示' },
+] as const;
+const DEBUG_CLIENT_TOOL_ID_SET = new Set<string>(DEBUG_CLIENT_TOOLS.map((item) => item.id));
+
 type DynamicVariableEntry = {
  id: string;
  key: string;
@@ -2034,6 +2050,13 @@ export const DebugDrawer: React.FC<{
  const [aecEnabled, setAecEnabled] = useState<boolean>(() => localStorage.getItem('debug_audio_aec') !== '0');
  const [nsEnabled, setNsEnabled] = useState<boolean>(() => localStorage.getItem('debug_audio_ns') !== '0');
  const [agcEnabled, setAgcEnabled] = useState<boolean>(() => localStorage.getItem('debug_audio_agc') !== '0');
+  const [clientToolEnabledMap, setClientToolEnabledMap] = useState<Record<string, boolean>>(() => {
+    const result: Record<string, boolean> = {};
+    for (const tool of DEBUG_CLIENT_TOOLS) {
+      result[tool.id] = localStorage.getItem(`debug_client_tool_enabled_${tool.id}`) !== '0';
+    }
+    return result;
+  });

  const micAudioCtxRef = useRef<AudioContext | null>(null);
  const micSourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
@@ -2043,14 +2066,18 @@ export const DebugDrawer: React.FC<{
  const userDraftIndexRef = useRef<number | null>(null);
  const lastUserFinalRef = useRef<string>('');
  const debugVolumePercentRef = useRef<number>(50);
+  const isClientToolEnabled = (toolId: string) => clientToolEnabledMap[toolId] !== false;
  const selectedToolSchemas = useMemo(() => {
-    const ids = assistant.tools || [];
-    if (!ids.length) return [];
+    const ids = Array.from(new Set([...(assistant.tools || []), ...DEBUG_CLIENT_TOOLS.map((item) => item.id)]));
    const byId = new Map(tools.map((t) => [t.id, t]));
    return ids.map((id) => {
      const item = byId.get(id);
      const toolId = item?.id || id;
-      const isClientTool = (item?.category || 'query') === 'system';
+      const debugClientTool = DEBUG_CLIENT_TOOLS.find((tool) => tool.id === toolId);
+      if (debugClientTool && !isClientToolEnabled(toolId)) {
+        return null;
+      }
+      const isClientTool = debugClientTool ? true : (item?.category || 'query') === 'system';
      const parameterSchema = (item?.parameterSchema && typeof item.parameterSchema === 'object')
        ? item.parameterSchema
        : getDefaultToolParameters(toolId);
@@ -2063,12 +2090,12 @@ export const DebugDrawer: React.FC<{
        ...(parameterDefaults && Object.keys(parameterDefaults).length > 0 ? { defaultArgs: parameterDefaults } : {}),
        function: {
          name: toolId,
-          description: item?.description || item?.name || id,
+          description: item?.description || item?.name || debugClientTool?.description || id,
          parameters: parameterSchema,
        },
      };
-    });
-  }, [assistant.tools, tools]);
+    }).filter(Boolean) as Array<Record<string, any>>;
+  }, [assistant.tools, tools, clientToolEnabledMap]);

  const clearResponseTracking = () => {
    assistantDraftIndexRef.current = null;
@@ -2129,6 +2156,12 @@ export const DebugDrawer: React.FC<{
    localStorage.setItem('debug_audio_agc', agcEnabled ? '1' : '0');
  }, [agcEnabled]);

+  useEffect(() => {
+    for (const tool of DEBUG_CLIENT_TOOLS) {
+      localStorage.setItem(`debug_client_tool_enabled_${tool.id}`, isClientToolEnabled(tool.id) ? '1' : '0');
+    }
+  }, [clientToolEnabledMap]);
+
  // Auto-scroll logic
  useEffect(() => {
      if (scrollRef.current) {
@@ -2940,6 +2973,10 @@ export const DebugDrawer: React.FC<{
              output: { message: `Unhandled client tool '${toolName}'` },
              status: { code: 501, message: 'not_implemented' },
            };
+            if (DEBUG_CLIENT_TOOL_ID_SET.has(toolName) && !isClientToolEnabled(toolName)) {
+              resultPayload.output = { message: `Client tool '${toolName}' is disabled in debug settings` };
+              resultPayload.status = { code: 503, message: 'tool_disabled' };
+            }
            const sendToolResult = () => {
              ws.send(
                JSON.stringify({
@@ -2962,7 +2999,9 @@ export const DebugDrawer: React.FC<{
              ]);
            };
            try {
-              if (toolName === 'turn_on_camera') {
+              if (resultPayload.status.code === 503) {
+                // Keep disabled result as-is.
+              } else if (toolName === 'turn_on_camera') {
                navigator.mediaDevices
                  .getUserMedia({
                    video: selectedCamera ? { deviceId: { exact: selectedCamera } } : true,
@@ -3010,6 +3049,22 @@ export const DebugDrawer: React.FC<{
                  level: debugVolumePercentRef.current,
                };
                resultPayload.status = { code: 200, message: 'ok' };
+              } else if (toolName === 'voice_message_prompt') {
+                const msg = String(parsedArgs?.msg || '').trim();
+                if (!msg) {
+                  resultPayload.output = { message: "Missing required argument 'msg'" };
+                  resultPayload.status = { code: 422, message: 'invalid_arguments' };
+                } else if (typeof window !== 'undefined' && 'speechSynthesis' in window) {
+                  const utterance = new SpeechSynthesisUtterance(msg);
+                  utterance.lang = 'zh-CN';
+                  window.speechSynthesis.cancel();
+                  window.speechSynthesis.speak(utterance);
+                  resultPayload.output = { message: 'voice_prompt_sent', msg };
+                  resultPayload.status = { code: 200, message: 'ok' };
+                } else {
+                  resultPayload.output = { message: 'speech_synthesis_unavailable', msg };
+                  resultPayload.status = { code: 503, message: 'speech_output_unavailable' };
+                }
              }
            } catch (err) {
              resultPayload.output = {
@@ -3381,6 +3436,33 @@ export const DebugDrawer: React.FC<{
            Auto Gain Control (AGC)
          </label>
        </div>
+        <div className="rounded-md border border-white/10 bg-black/20 p-2 space-y-2">
+          <p className="text-[10px] uppercase tracking-widest text-muted-foreground">Client Tools</p>
+          <p className="text-[11px] text-muted-foreground">滑块控制调试会话内客户端工具是否启用。</p>
+          <div className="space-y-2">
+            {DEBUG_CLIENT_TOOLS.map((tool) => {
+              const enabled = isClientToolEnabled(tool.id);
+              return (
+                <div key={tool.id} className="flex items-center justify-between gap-3 rounded-md border border-white/10 bg-black/20 px-2 py-1.5">
+                  <div className="min-w-0">
+                    <div className="text-[11px] font-mono text-foreground truncate">{tool.name}</div>
+                    <div className="text-[10px] text-muted-foreground truncate">{tool.description}</div>
+                  </div>
+                  <button
+                    type="button"
+                    onClick={() => setClientToolEnabledMap((prev) => ({ ...prev, [tool.id]: !enabled }))}
+                    className={`relative h-6 w-11 rounded-full transition-colors ${enabled ? 'bg-emerald-500/80' : 'bg-white/20'}`}
+                    title={enabled ? '点击关闭' : '点击开启'}
+                  >
+                    <span
+                      className={`absolute top-0.5 h-5 w-5 rounded-full bg-white shadow transition-transform ${enabled ? 'translate-x-5' : 'translate-x-0.5'}`}
+                    />
+                  </button>
+                </div>
+              );
+            })}
+          </div>
+        </div>
        <div className="rounded-md border border-white/10 bg-black/20 p-2 space-y-2">
          <div className="flex items-center justify-between gap-2">
            <p className="text-[10px] uppercase tracking-widest text-muted-foreground">Dynamic Variables</p>