Add wait_for_response functionality to ToolResource and related components. Update API models, schemas, and routers to support new parameter. Enhance UI components to manage wait_for_response state, ensuring proper integration across the application.

2026-02-27 16:54:39 +08:00
parent 95c6e93a9c
commit 229243e832
9 changed files with 303 additions and 44 deletions
--- a/web/pages/Assistants.tsx
+++ b/web/pages/Assistants.tsx
@@ -1717,6 +1717,9 @@ const DEBUG_CLIENT_TOOLS = [
  { id: 'text_msg_prompt', name: 'text_msg_prompt', description: '文本消息提示' },
 ] as const;
 const DEBUG_CLIENT_TOOL_ID_SET = new Set<string>(DEBUG_CLIENT_TOOLS.map((item) => item.id));
+const DEBUG_CLIENT_TOOL_WAIT_DEFAULTS: Record<string, boolean> = {
+  text_msg_prompt: true,
+};

 type DynamicVariableEntry = {
  id: string;
@@ -1981,7 +1984,16 @@ export const DebugDrawer: React.FC<{
  const [inputText, setInputText] = useState('');
  const [isLoading, setIsLoading] = useState(false);
  const [callStatus, setCallStatus] = useState<'idle' | 'calling' | 'active'>('idle');
-  const [textPromptDialog, setTextPromptDialog] = useState<{ open: boolean; message: string }>({ open: false, message: '' });
+  const [textPromptDialog, setTextPromptDialog] = useState<{
+    open: boolean;
+    message: string;
+    pendingResult?: {
+      toolCallId: string;
+      toolName: string;
+      toolDisplayName: string;
+      waitForResponse: boolean;
+    };
+  }>({ open: false, message: '' });
  const [textSessionStarted, setTextSessionStarted] = useState(false);
  const [wsStatus, setWsStatus] = useState<'disconnected' | 'connecting' | 'ready' | 'error'>('disconnected');
  const [wsError, setWsError] = useState('');
@@ -2043,6 +2055,7 @@ export const DebugDrawer: React.FC<{
  const assistantResponseIndexByIdRef = useRef<Map<string, number>>(new Map());
  const pendingTtfbByResponseIdRef = useRef<Map<string, number>>(new Map());
  const interruptedResponseIdsRef = useRef<Set<string>>(new Set());
+  const interruptedDropNoticeKeysRef = useRef<Set<string>>(new Set());
  const audioCtxRef = useRef<AudioContext | null>(null);
  const playbackTimeRef = useRef<number>(0);
  const activeAudioSourcesRef = useRef<Set<AudioBufferSourceNode>>(new Set());
@@ -2088,6 +2101,9 @@ export const DebugDrawer: React.FC<{
        return null;
      }
      const isClientTool = debugClientTool ? true : (item?.category || 'query') === 'system';
+      const waitForResponse = isClientTool
+        ? (item?.waitForResponse ?? DEBUG_CLIENT_TOOL_WAIT_DEFAULTS[toolId] ?? false)
+        : false;
      const parameterSchema = (item?.parameterSchema && typeof item.parameterSchema === 'object')
        ? item.parameterSchema
        : getDefaultToolParameters(toolId);
@@ -2097,6 +2113,7 @@ export const DebugDrawer: React.FC<{
      return {
        type: 'function',
        executor: isClientTool ? 'client' : 'server',
+        waitForResponse,
        ...(parameterDefaults && Object.keys(parameterDefaults).length > 0 ? { defaultArgs: parameterDefaults } : {}),
        function: {
          name: toolId,
@@ -2112,6 +2129,7 @@ export const DebugDrawer: React.FC<{
    assistantResponseIndexByIdRef.current.clear();
    pendingTtfbByResponseIdRef.current.clear();
    interruptedResponseIdsRef.current.clear();
+    interruptedDropNoticeKeysRef.current.clear();
  };

  const extractResponseId = (payload: any): string | undefined => {
@@ -2120,6 +2138,23 @@ export const DebugDrawer: React.FC<{
    return responseId || undefined;
  };

+  const noteInterruptedDrop = (responseId: string, kind: 'ttfb' | 'delta' | 'final') => {
+    const key = `${responseId}:${kind}`;
+    if (interruptedDropNoticeKeysRef.current.has(key)) return;
+    interruptedDropNoticeKeysRef.current.add(key);
+    if (interruptedDropNoticeKeysRef.current.size > 256) {
+      const oldest = interruptedDropNoticeKeysRef.current.values().next().value as string | undefined;
+      if (oldest) interruptedDropNoticeKeysRef.current.delete(oldest);
+    }
+    setMessages((prev) => [
+      ...prev,
+      {
+        role: 'tool',
+        text: `drop stale ${kind} from interrupted response ${responseId}`,
+      },
+    ]);
+  };
+
  // Initialize
  useEffect(() => {
    if (isOpen) {
@@ -2364,6 +2399,64 @@ export const DebugDrawer: React.FC<{
    clearPlaybackQueue();
  };

+  const emitClientToolResult = (resultPayload: any, toolDisplayName?: string) => {
+    const ws = wsRef.current;
+    if (ws && ws.readyState === WebSocket.OPEN) {
+      ws.send(
+        JSON.stringify({
+          type: 'tool_call.results',
+          results: [resultPayload],
+        })
+      );
+    }
+    const statusCode = Number(resultPayload?.status?.code || 500);
+    const statusMessage = String(resultPayload?.status?.message || 'error');
+    const displayName = toolDisplayName || String(resultPayload?.name || 'unknown_tool');
+    const resultText =
+      statusCode === 200 && typeof resultPayload?.output?.result === 'number'
+        ? `result ${displayName} = ${resultPayload.output.result}`
+        : `result ${displayName} status=${statusCode} ${statusMessage}`;
+    setMessages((prev) => [
+      ...prev,
+      {
+        role: 'tool',
+        text: resultText,
+      },
+    ]);
+  };
+
+  const closeTextPromptDialog = (action: 'confirm' | 'dismiss') => {
+    let pending:
+      | {
+          toolCallId: string;
+          toolName: string;
+          toolDisplayName: string;
+          waitForResponse: boolean;
+        }
+      | undefined;
+    let message = '';
+    setTextPromptDialog((prev) => {
+      pending = prev.pendingResult;
+      message = prev.message;
+      return { open: false, message: '' };
+    });
+    if (pending?.waitForResponse) {
+      emitClientToolResult(
+        {
+          tool_call_id: pending.toolCallId,
+          name: pending.toolName,
+          output: {
+            message: 'text_prompt_closed',
+            action,
+            msg: message,
+          },
+          status: { code: 200, message: 'ok' },
+        },
+        pending.toolDisplayName
+      );
+    }
+  };
+
  const scheduleQueuedPlayback = (ctx: AudioContext) => {
    const queue = queuedAudioBuffersRef.current;
    if (queue.length === 0) return;
@@ -2492,6 +2585,9 @@ export const DebugDrawer: React.FC<{
  };

  const handleHangup = () => {
+    if (textPromptDialog.open) {
+      closeTextPromptDialog('dismiss');
+    }
    stopVoiceCapture();
    stopMedia();
    closeWs();
@@ -2941,7 +3037,10 @@ export const DebugDrawer: React.FC<{
          if (!Number.isFinite(maybeTtfb) || maybeTtfb < 0) return;
          const ttfbMs = Math.round(maybeTtfb);
          const responseId = extractResponseId(payload);
-          if (responseId && interruptedResponseIdsRef.current.has(responseId)) return;
+          if (responseId && interruptedResponseIdsRef.current.has(responseId)) {
+            noteInterruptedDrop(responseId, 'ttfb');
+            return;
+          }
          if (responseId) {
            const indexed = assistantResponseIndexByIdRef.current.get(responseId);
            if (typeof indexed === 'number') {
@@ -2994,6 +3093,9 @@ export const DebugDrawer: React.FC<{
                parsedArgs = {};
              }
            }
+            const waitForResponse = Boolean(
+              payload?.wait_for_response ?? toolCall?.wait_for_response ?? toolCall?.waitForResponse ?? false
+            );
            const resultPayload: any = {
              tool_call_id: toolCallId,
              name: toolName,
@@ -3004,31 +3106,21 @@ export const DebugDrawer: React.FC<{
              resultPayload.output = { message: `Client tool '${toolName}' is disabled in debug settings` };
              resultPayload.status = { code: 503, message: 'tool_disabled' };
            }
-            const sendToolResult = () => {
-              ws.send(
-                JSON.stringify({
-                  type: 'tool_call.results',
-                  results: [resultPayload],
-                })
-              );
-              const statusCode = Number(resultPayload?.status?.code || 500);
-              const statusMessage = String(resultPayload?.status?.message || 'error');
-              const resultText =
-                statusCode === 200 && typeof resultPayload?.output?.result === 'number'
-                  ? `result ${toolDisplayName} = ${resultPayload.output.result}`
-                  : `result ${toolDisplayName} status=${statusCode} ${statusMessage}`;
-              setMessages((prev) => [
-                ...prev,
-                {
-                  role: 'tool',
-                  text: resultText,
-                },
-              ]);
-            };
            try {
              if (resultPayload.status.code === 503) {
                // Keep disabled result as-is.
              } else if (toolName === 'turn_on_camera') {
+                if (!waitForResponse) {
+                  emitClientToolResult(
+                    {
+                      tool_call_id: toolCallId,
+                      name: toolName,
+                      output: { message: 'camera_on_dispatched' },
+                      status: { code: 200, message: 'ok' },
+                    },
+                    toolDisplayName
+                  );
+                }
                navigator.mediaDevices
                  .getUserMedia({
                    video: selectedCamera ? { deviceId: { exact: selectedCamera } } : true,
@@ -3037,20 +3129,36 @@ export const DebugDrawer: React.FC<{
                  .then((stream) => {
                    if (videoRef.current) videoRef.current.srcObject = stream;
                    streamRef.current = stream;
-                    resultPayload.output = {
-                      message: 'camera_on',
-                      tracks: stream.getVideoTracks().length,
-                    };
-                    resultPayload.status = { code: 200, message: 'ok' };
-                    sendToolResult();
+                    if (waitForResponse) {
+                      emitClientToolResult(
+                        {
+                          tool_call_id: toolCallId,
+                          name: toolName,
+                          output: {
+                            message: 'camera_on',
+                            tracks: stream.getVideoTracks().length,
+                          },
+                          status: { code: 200, message: 'ok' },
+                        },
+                        toolDisplayName
+                      );
+                    }
                  })
                  .catch((err) => {
-                    resultPayload.output = {
-                      message: `Client tool '${toolName}' failed`,
-                      error: err instanceof Error ? err.message : String(err),
-                    };
-                    resultPayload.status = { code: 500, message: 'client_tool_failed' };
-                    sendToolResult();
+                    if (waitForResponse) {
+                      emitClientToolResult(
+                        {
+                          tool_call_id: toolCallId,
+                          name: toolName,
+                          output: {
+                            message: `Client tool '${toolName}' failed`,
+                            error: err instanceof Error ? err.message : String(err),
+                          },
+                          status: { code: 500, message: 'client_tool_failed' },
+                        },
+                        toolDisplayName
+                      );
+                    }
                  });
                return;
              } else if (toolName === 'turn_off_camera') {
@@ -3085,6 +3193,36 @@ export const DebugDrawer: React.FC<{
                  const utterance = new SpeechSynthesisUtterance(msg);
                  utterance.lang = 'zh-CN';
                  window.speechSynthesis.cancel();
+                  if (waitForResponse) {
+                    utterance.onend = () => {
+                      emitClientToolResult(
+                        {
+                          tool_call_id: toolCallId,
+                          name: toolName,
+                          output: { message: 'voice_prompt_completed', msg },
+                          status: { code: 200, message: 'ok' },
+                        },
+                        toolDisplayName
+                      );
+                    };
+                    utterance.onerror = (event) => {
+                      emitClientToolResult(
+                        {
+                          tool_call_id: toolCallId,
+                          name: toolName,
+                          output: {
+                            message: 'voice_prompt_failed',
+                            msg,
+                            error: String(event.error || 'speech_error'),
+                          },
+                          status: { code: 500, message: 'client_tool_failed' },
+                        },
+                        toolDisplayName
+                      );
+                    };
+                    window.speechSynthesis.speak(utterance);
+                    return;
+                  }
                  window.speechSynthesis.speak(utterance);
                  resultPayload.output = { message: 'voice_prompt_sent', msg };
                  resultPayload.status = { code: 200, message: 'ok' };
@@ -3098,9 +3236,22 @@ export const DebugDrawer: React.FC<{
                  resultPayload.output = { message: "Missing required argument 'msg'" };
                  resultPayload.status = { code: 422, message: 'invalid_arguments' };
                } else {
-                  setTextPromptDialog({ open: true, message: msg });
-                  resultPayload.output = { message: 'text_prompt_shown', msg };
-                  resultPayload.status = { code: 200, message: 'ok' };
+                  setTextPromptDialog({
+                    open: true,
+                    message: msg,
+                    pendingResult: {
+                      toolCallId: toolCallId,
+                      toolName,
+                      toolDisplayName,
+                      waitForResponse,
+                    },
+                  });
+                  if (!waitForResponse) {
+                    resultPayload.output = { message: 'text_prompt_shown', msg };
+                    resultPayload.status = { code: 200, message: 'ok' };
+                  } else {
+                    return;
+                  }
                }
              }
            } catch (err) {
@@ -3110,7 +3261,7 @@ export const DebugDrawer: React.FC<{
              };
              resultPayload.status = { code: 500, message: 'client_tool_failed' };
            }
-            sendToolResult();
+            emitClientToolResult(resultPayload, toolDisplayName);
          }
          return;
        }
@@ -3221,7 +3372,10 @@ export const DebugDrawer: React.FC<{
          const delta = String(payload.text || '');
          if (!delta) return;
          const responseId = extractResponseId(payload);
-          if (responseId && interruptedResponseIdsRef.current.has(responseId)) return;
+          if (responseId && interruptedResponseIdsRef.current.has(responseId)) {
+            noteInterruptedDrop(responseId, 'delta');
+            return;
+          }
          setMessages((prev) => {
            let idx = assistantDraftIndexRef.current;
            if (idx === null || !prev[idx] || prev[idx].role !== 'model') {
@@ -3288,7 +3442,10 @@ export const DebugDrawer: React.FC<{
        if (type === 'assistant.response.final') {
          const finalText = String(payload.text || '');
          const responseId = extractResponseId(payload);
-          if (responseId && interruptedResponseIdsRef.current.has(responseId)) return;
+          if (responseId && interruptedResponseIdsRef.current.has(responseId)) {
+            noteInterruptedDrop(responseId, 'final');
+            return;
+          }
          setMessages((prev) => {
            let idx = assistantDraftIndexRef.current;
            assistantDraftIndexRef.current = null;
@@ -3813,7 +3970,7 @@ export const DebugDrawer: React.FC<{
              <div className="relative w-[92%] max-w-md rounded-xl border border-white/15 bg-card/95 p-4 shadow-2xl animate-in zoom-in-95 duration-200">
                <button
                  type="button"
-                  onClick={() => setTextPromptDialog({ open: false, message: '' })}
+                  onClick={() => closeTextPromptDialog('dismiss')}
                  className="absolute right-3 top-3 rounded-sm opacity-70 hover:opacity-100 text-muted-foreground hover:text-foreground transition-opacity"
                  title="关闭"
                >
@@ -3824,7 +3981,7 @@ export const DebugDrawer: React.FC<{
                  <p className="mt-2 text-sm leading-6 text-foreground whitespace-pre-wrap break-words">{textPromptDialog.message}</p>
                </div>
                <div className="flex justify-end">
-                  <Button size="sm" onClick={() => setTextPromptDialog({ open: false, message: '' })}>
+                  <Button size="sm" onClick={() => closeTextPromptDialog('confirm')}>
                    确认
                  </Button>
                </div>