feat: add configurable client tools and photo input

This commit is contained in:
Xin Wang
2026-07-30 19:06:03 +08:00
parent 510a277b5a
commit 913435785e
24 changed files with 1802 additions and 139 deletions

View File

@@ -0,0 +1,66 @@
"use client";
import { useCallback, useEffect, useState } from "react";
import type { VoicePreview } from "@/hooks/use-voice-preview";
const PHOTO_BUTTON_TOOL = "set_photo_button_visible";
export function usePhotoCaptureTool(preview: VoicePreview) {
const { registerClientTool, sendUserInput, status } = preview;
const [visible, setVisible] = useState(false);
const [capturing, setCapturing] = useState(false);
const [error, setError] = useState<string | null>(null);
useEffect(
() =>
registerClientTool(PHOTO_BUTTON_TOOL, ({ visible: nextValue }) => {
if (typeof nextValue !== "boolean") {
throw new Error("visible 参数必须是布尔值");
}
setVisible(nextValue);
setError(null);
return { visible: nextValue };
}),
[registerClientTool],
);
useEffect(() => {
if (status === "connected") return;
// eslint-disable-next-line react-hooks/set-state-in-effect
setVisible(false);
setCapturing(false);
}, [status]);
const capture = useCallback(async () => {
if (capturing || status !== "connected") return;
setCapturing(true);
setError(null);
try {
await sendUserInput(
[
{
type: "input_image",
source: { type: "camera_frame", frame: "current" },
},
],
{ runImmediately: true, interrupt: true },
);
} catch (captureError) {
setError(
captureError instanceof Error ? captureError.message : "拍照提交失败",
);
} finally {
setCapturing(false);
}
}, [capturing, sendUserInput, status]);
return {
visible,
capturing,
error,
capture,
};
}
export type PhotoCaptureTool = ReturnType<typeof usePhotoCaptureTool>;