feat: add configurable client tools and photo input

frontend/src/hooks/use-voice-preview.ts

@@ -47,6 +47,25 @@ export type ChatMessage = {
 
 type AppMessage = Record<string, unknown> & { type?: string };
 type DynamicVariableValue = string | number | boolean;
+export type UserInputPart =
+  | { type: "input_text"; text: string }
+  | {
+      type: "input_image";
+      source: { type: "camera_frame"; frame: "current" };
+    };
+export type UserInputResult = {
+  inputId: string;
+  status: "accepted";
+};
+export type ClientToolHandler = (
+  argumentsValue: Record<string, unknown>,
+) => unknown | Promise<unknown>;
+
+type PendingUserInput = {
+  resolve: (result: UserInputResult) => void;
+  reject: (error: Error) => void;
+  timeout: number;
+};
 
 function publicVariableSnapshot(
   value: unknown,
@@ -116,6 +135,13 @@ function encodeHeaderJson(value: unknown): string {
   return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
 }
 
+function newInputId(): string {
+  if (typeof crypto !== "undefined" && "randomUUID" in crypto) {
+    return `input_${crypto.randomUUID()}`;
+  }
+  return `input_${Date.now()}_${Math.random().toString(16).slice(2)}`;
+}
+
 function readNetworkMetrics(report: RTCStatsReport): NetworkMetrics | null {
   const outbound: Partial<Record<"audio" | "video", number>> = {};
   const lost: Partial<Record<"audio" | "video", number>> = {};
@@ -219,6 +245,8 @@ export function useVoicePreview(
   const pendingAssistantTurnsRef = useRef(
     new Map<string, { timestamp: string }>(),
   );
+  const pendingUserInputsRef = useRef(new Map<string, PendingUserInput>());
+  const clientToolHandlersRef = useRef(new Map<string, ClientToolHandler>());
   const endedByServerRef = useRef(false);
   const selectedDeviceIdRef = useRef("");
   const selectedOutputDeviceIdRef = useRef("");
@@ -273,6 +301,11 @@ export function useVoicePreview(
     if (audioRef.current) audioRef.current.srcObject = null;
     startingRef.current = false;
     pendingAssistantTurnsRef.current.clear();
+    pendingUserInputsRef.current.forEach(({ reject, timeout }) => {
+      window.clearTimeout(timeout);
+      reject(new Error("连接已关闭，用户输入未完成"));
+    });
+    pendingUserInputsRef.current.clear();
     networkStatsRef.current = null;
     onNodeActiveRef.current?.(null);
   }, []);
@@ -301,8 +334,78 @@ export function useVoicePreview(
     setStatus("failed");
   }, [releaseResources]);
 
+  const dispatchClientTool = useCallback(async (msg: AppMessage) => {
+    const toolCallId =
+      typeof msg.tool_call_id === "string" ? msg.tool_call_id : "";
+    const functionName =
+      typeof msg.function_name === "string" ? msg.function_name : "";
+    const waitForResponse = msg.wait_for_response !== false;
+    if (!toolCallId || !functionName) return;
+
+    const handler = clientToolHandlersRef.current.get(functionName);
+    const transport = transportRef.current;
+    if (!transport || transport.state !== "connected") return;
+    if (!handler) {
+      if (waitForResponse) {
+        transport.sendAppMessage({
+          type: "client-tool-result",
+          tool_call_id: toolCallId,
+          status: "error",
+          message: `客户端未注册工具: ${functionName}`,
+        });
+      }
+      return;
+    }
+
+    try {
+      const argumentsValue =
+        msg.arguments &&
+        typeof msg.arguments === "object" &&
+        !Array.isArray(msg.arguments)
+          ? (msg.arguments as Record<string, unknown>)
+          : {};
+      const data = await handler(argumentsValue);
+      if (transportRef.current !== transport) return;
+      if (!waitForResponse) return;
+      transport.sendAppMessage({
+        type: "client-tool-result",
+        tool_call_id: toolCallId,
+        status: "ok",
+        data,
+      });
+    } catch (toolError) {
+      if (transportRef.current !== transport) return;
+      if (!waitForResponse) return;
+      transport.sendAppMessage({
+        type: "client-tool-result",
+        tool_call_id: toolCallId,
+        status: "error",
+        message: errorMessage(toolError, "客户端工具执行失败"),
+      });
+    }
+  }, []);
+
   const handleAppMessage = useCallback((msg: AppMessage) => {
-    if (
+    if (msg.type === "client-tool-call") {
+      void dispatchClientTool(msg);
+    } else if (
+      msg.type === "user-input-result" &&
+      typeof msg.input_id === "string"
+    ) {
+      const pending = pendingUserInputsRef.current.get(msg.input_id);
+      if (!pending) return;
+      pendingUserInputsRef.current.delete(msg.input_id);
+      window.clearTimeout(pending.timeout);
+      if (msg.status === "accepted") {
+        pending.resolve({ inputId: msg.input_id, status: "accepted" });
+      } else {
+        pending.reject(
+          new Error(
+            typeof msg.message === "string" ? msg.message : "用户输入处理失败",
+          ),
+        );
+      }
+    } else if (
       msg.type === "assistant-text-start" &&
       typeof msg.turn_id === "string"
     ) {
@@ -413,7 +516,7 @@ export function useVoicePreview(
       setCallEnded(true);
       disconnect();
     }
-  }, [disconnect]);
+  }, [disconnect, dispatchClientTool]);
 
   const connect = useCallback(async (options: ConnectOptions = {}) => {
     if (startingRef.current || transportRef.current) return;
@@ -601,10 +704,68 @@ export function useVoicePreview(
     const trimmed = text.trim();
     const transport = transportRef.current;
     if (!trimmed || !transport || transport.state !== "connected") return false;
-    transport.sendAppMessage({ type: "user-text", text: trimmed });
+    const inputId = newInputId();
+    transport.sendAppMessage({
+      type: "user-input",
+      schema_version: 1,
+      input_id: inputId,
+      parts: [{ type: "input_text", text: trimmed }],
+      options: { run_immediately: true, interrupt: true },
+    });
     return true;
   }, []);
 
+  const sendUserInput = useCallback(
+    (
+      parts: UserInputPart[],
+      options: { runImmediately?: boolean; interrupt?: boolean } = {},
+    ): Promise<UserInputResult> => {
+      const transport = transportRef.current;
+      if (!transport || transport.state !== "connected") {
+        return Promise.reject(new Error("当前未连接语音服务"));
+      }
+      const inputId = newInputId();
+      return new Promise<UserInputResult>((resolve, reject) => {
+        const timeout = window.setTimeout(() => {
+          pendingUserInputsRef.current.delete(inputId);
+          reject(new Error("等待用户输入处理结果超时"));
+        }, 30_000);
+        pendingUserInputsRef.current.set(inputId, { resolve, reject, timeout });
+        try {
+          transport.sendAppMessage({
+            type: "user-input",
+            schema_version: 1,
+            input_id: inputId,
+            parts,
+            options: {
+              run_immediately: options.runImmediately ?? true,
+              interrupt: options.interrupt ?? true,
+            },
+          });
+        } catch (sendError) {
+          window.clearTimeout(timeout);
+          pendingUserInputsRef.current.delete(inputId);
+          reject(
+            new Error(errorMessage(sendError, "发送用户输入失败")),
+          );
+        }
+      });
+    },
+    [],
+  );
+
+  const registerClientTool = useCallback(
+    (functionName: string, handler: ClientToolHandler): (() => void) => {
+      clientToolHandlersRef.current.set(functionName, handler);
+      return () => {
+        if (clientToolHandlersRef.current.get(functionName) === handler) {
+          clientToolHandlersRef.current.delete(functionName);
+        }
+      };
+    },
+    [],
+  );
+
   useEffect(() => releaseResources, [releaseResources]);
 
   return {
@@ -627,6 +788,8 @@ export function useVoicePreview(
     selectOutputDevice,
     supportsOutputSelection,
     sendText,
+    sendUserInput,
+    registerClientTool,
     connect,
     replaceVideoStream,
     selectCamera,