wx44wx/AI-VideoAssistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add new tools to DuplexPipeline: calculator, code_interpreter, turn_on_camera, turn_off_camera, increase_volume, and decrease_volume. Implement fallback schema for unknown string tools and assign default client executors for specific tools. Update tests to validate new functionality and ensure correct tool handling in the pipeline.

Browse Source
This commit is contained in:
Xin Wang
2026-02-27 13:59:37 +08:00
parent 8d453e10da
commit d942c85eff
3 changed files with 214 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
web/pages/Assistants.tsx
View File

@@ -2042,6 +2042,7 @@ export const DebugDrawer: React.FC<{
const micFrameBufferRef = useRef<Uint8Array>(new Uint8Array(0));
const userDraftIndexRef = useRef<number | null>(null);
const lastUserFinalRef = useRef<string>('');
const debugVolumePercentRef = useRef<number>(50);
const selectedToolSchemas = useMemo(() => {
const ids = assistant.tools || [];
if (!ids.length) return [];
@@ -2916,33 +2917,100 @@ export const DebugDrawer: React.FC<{
},
]);
if (executor === 'client' && toolCallId && ws.readyState === WebSocket.OPEN) {
let parsedArgs: Record<string, any> = {};
if (rawArgs) {
try {
const candidate = JSON.parse(rawArgs);
parsedArgs = candidate && typeof candidate === 'object' ? candidate : {};
} catch {
parsedArgs = {};
}
}
const resultPayload: any = {
tool_call_id: toolCallId,
name: toolName,
output: {
message: 'Client tool execution is not implemented in debug web client',
},
output: { message: `Unhandled client tool '${toolName}'` },
status: { code: 501, message: 'not_implemented' },
};
ws.send(
JSON.stringify({
type: 'tool_call.results',
results: [resultPayload],
})
);
const statusCode = Number(resultPayload?.status?.code || 500);
const statusMessage = String(resultPayload?.status?.message || 'error');
const resultText =
statusCode === 200 && typeof resultPayload?.output?.result === 'number'
? `result ${toolName} = ${resultPayload.output.result}`
: `result ${toolName} status=${statusCode} ${statusMessage}`;
setMessages((prev) => [
...prev,
{
role: 'tool',
text: resultText,
},
]);
const sendToolResult = () => {
ws.send(
JSON.stringify({
type: 'tool_call.results',
results: [resultPayload],
})
);
const statusCode = Number(resultPayload?.status?.code || 500);
const statusMessage = String(resultPayload?.status?.message || 'error');
const resultText =
statusCode === 200 && typeof resultPayload?.output?.result === 'number'
? `result ${toolName} = ${resultPayload.output.result}`
: `result ${toolName} status=${statusCode} ${statusMessage}`;
setMessages((prev) => [
...prev,
{
role: 'tool',
text: resultText,
},
]);
};
try {
if (toolName === 'turn_on_camera') {
navigator.mediaDevices
.getUserMedia({
video: selectedCamera ? { deviceId: { exact: selectedCamera } } : true,
audio: false,
})
.then((stream) => {
if (videoRef.current) videoRef.current.srcObject = stream;
streamRef.current = stream;
resultPayload.output = {
message: 'camera_on',
tracks: stream.getVideoTracks().length,
};
resultPayload.status = { code: 200, message: 'ok' };
sendToolResult();
})
.catch((err) => {
resultPayload.output = {
message: `Client tool '${toolName}' failed`,
error: err instanceof Error ? err.message : String(err),
};
resultPayload.status = { code: 500, message: 'client_tool_failed' };
sendToolResult();
});
return;
} else if (toolName === 'turn_off_camera') {
stopMedia();
if (videoRef.current) videoRef.current.srcObject = null;
resultPayload.output = { message: 'camera_off' };
resultPayload.status = { code: 200, message: 'ok' };
} else if (toolName === 'increase_volume') {
const rawStep = Number(parsedArgs?.step);
const step = Number.isFinite(rawStep) ? Math.max(1, Math.floor(rawStep)) : 1;
debugVolumePercentRef.current = Math.min(100, debugVolumePercentRef.current + step);
resultPayload.output = {
message: 'volume_increased',
level: debugVolumePercentRef.current,
};
resultPayload.status = { code: 200, message: 'ok' };
} else if (toolName === 'decrease_volume') {
const rawStep = Number(parsedArgs?.step);
const step = Number.isFinite(rawStep) ? Math.max(1, Math.floor(rawStep)) : 1;
debugVolumePercentRef.current = Math.max(0, debugVolumePercentRef.current - step);
resultPayload.output = {
message: 'volume_decreased',
level: debugVolumePercentRef.current,
};
resultPayload.status = { code: 200, message: 'ok' };
}
} catch (err) {
resultPayload.output = {
message: `Client tool '${toolName}' failed`,
error: err instanceof Error ? err.message : String(err),
};
resultPayload.status = { code: 500, message: 'client_tool_failed' };
}
sendToolResult();
}
return;
}