wx44wx/AI-VideoAssistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add calculator tool

Browse Source
This commit is contained in:
Xin Wang
2026-02-10 16:39:56 +08:00
parent 4b8da32787
commit 54eb48fb74
1 changed files with 179 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

185
web/pages/Assistants.tsx
View File

@@ -1,5 +1,5 @@
import React, { useState, useEffect, useRef } from 'react';
import React, { useState, useEffect, useMemo, useRef } from 'react';
import { Plus, Search, Play, Copy, Trash2, Mic, MessageSquare, Save, Video, PhoneOff, Camera, ArrowLeftRight, Send, Phone, Rocket, AlertTriangle, PhoneCall, CameraOff, Image, Images, CloudSun, Calendar, TrendingUp, Coins, Wrench, Globe, Terminal, X, ClipboardCheck, Sparkles, Volume2, Timer, ChevronDown, Database, Server, Zap, ExternalLink, Key, BrainCircuit, Ear, Book, Filter } from 'lucide-react';
import { Button, Input, Badge, Drawer, Dialog } from '../components/UI';
import { ASRModel, Assistant, KnowledgeBase, LLMModel, TabValue, Tool, Voice } from '../types';
@@ -915,6 +915,7 @@ export const AssistantsPage: React.FC = () => {
voices={voices}
llmModels={llmModels}
asrModels={asrModels}
tools={tools}
/>
)}
@@ -962,10 +963,78 @@ const BotIcon = ({className}: {className?: string}) => (
</svg>
);
const TOOL_PARAMETER_HINTS: Record<string, any> = {
search: {
type: 'object',
properties: { query: { type: 'string', description: 'Search query' } },
required: ['query'],
},
calculator: {
type: 'object',
properties: { expression: { type: 'string', description: 'Math expression' } },
required: ['expression'],
},
weather: {
type: 'object',
properties: { city: { type: 'string', description: 'City name' } },
required: ['city'],
},
translate: {
type: 'object',
properties: {
text: { type: 'string', description: 'Text to translate' },
target_lang: { type: 'string', description: 'Target language' },
},
required: ['text', 'target_lang'],
},
knowledge: {
type: 'object',
properties: {
query: { type: 'string', description: 'Knowledge query' },
kb_id: { type: 'string', description: 'Knowledge base id' },
},
required: ['query'],
},
};
const getDefaultToolParameters = (toolId: string) =>
TOOL_PARAMETER_HINTS[toolId] || { type: 'object', properties: {} };
const parseToolArgs = (raw: unknown): Record<string, any> => {
if (raw && typeof raw === 'object') return raw as Record<string, any>;
if (typeof raw !== 'string') return {};
const text = raw.trim();
if (!text) return {};
try {
const parsed = JSON.parse(text);
return parsed && typeof parsed === 'object' ? parsed : {};
} catch {
return {};
}
};
const evaluateCalculatorExpression = (expression: string): { ok: true; result: number } | { ok: false; error: string } => {
const expr = String(expression || '').trim();
if (!expr) return { ok: false, error: 'empty expression' };
if (expr.length > 200) return { ok: false, error: 'expression too long' };
if (!/^[0-9+\-*/().%\s]+$/.test(expr)) return { ok: false, error: 'invalid characters' };
if (expr.includes('**') || expr.includes('//')) return { ok: false, error: 'unsupported operator' };
try {
const value = Function(`"use strict"; return (${expr});`)();
if (typeof value !== 'number' || !Number.isFinite(value)) {
return { ok: false, error: 'expression is not finite number' };
}
return { ok: true, result: value };
} catch {
return { ok: false, error: 'invalid expression' };
}
};
// Stable transcription log so the scroll container is not recreated on every render (avoids scroll jumping)
const TranscriptionLog: React.FC<{
scrollRef: React.RefObject<HTMLDivElement | null>;
messages: { role: 'user' | 'model'; text: string }[];
messages: { role: 'user' | 'model' | 'tool'; text: string }[];
isLoading: boolean;
className?: string;
}> = ({ scrollRef, messages, isLoading, className = '' }) => (
@@ -973,8 +1042,8 @@ const TranscriptionLog: React.FC<{
{messages.length === 0 && <div className="text-center text-muted-foreground text-xs py-4">暂无转写记录</div>}
{messages.map((m, i) => (
<div key={i} className={`flex ${m.role === 'user' ? 'justify-end' : 'justify-start'}`}>
<div className={`max-w-[85%] rounded-lg px-3 py-2 text-sm ${m.role === 'user' ? 'bg-primary text-primary-foreground' : 'bg-card border border-white/10 shadow-sm text-foreground'}`}>
<span className="text-[10px] opacity-70 block mb-0.5 uppercase tracking-wider">{m.role === 'user' ? 'Me' : 'AI'}</span>
<div className={`max-w-[85%] rounded-lg px-3 py-2 text-sm ${m.role === 'user' ? 'bg-primary text-primary-foreground' : m.role === 'tool' ? 'bg-amber-500/10 border border-amber-400/30 text-amber-100' : 'bg-card border border-white/10 shadow-sm text-foreground'}`}>
<span className="text-[10px] opacity-70 block mb-0.5 uppercase tracking-wider">{m.role === 'user' ? 'Me' : m.role === 'tool' ? 'Tool' : 'AI'}</span>
{m.text}
</div>
</div>
@@ -991,6 +1060,7 @@ export const DebugDrawer: React.FC<{
voices?: Voice[];
llmModels?: LLMModel[];
asrModels?: ASRModel[];
tools?: Tool[];
sessionMetadataExtras?: Record<string, any>;
onProtocolEvent?: (event: Record<string, any>) => void;
}> = ({
@@ -1000,6 +1070,7 @@ export const DebugDrawer: React.FC<{
voices = [],
llmModels = [],
asrModels = [],
tools = [],
sessionMetadataExtras,
onProtocolEvent,
}) => {
@@ -1035,7 +1106,7 @@ export const DebugDrawer: React.FC<{
};
const [mode, setMode] = useState<'text' | 'voice' | 'video'>('text');
const [messages, setMessages] = useState<{role: 'user' | 'model', text: string}[]>([]);
const [messages, setMessages] = useState<{role: 'user' | 'model' | 'tool', text: string}[]>([]);
const [inputText, setInputText] = useState('');
const [isLoading, setIsLoading] = useState(false);
const [callStatus, setCallStatus] = useState<'idle' | 'calling' | 'active'>('idle');
@@ -1088,6 +1159,22 @@ export const DebugDrawer: React.FC<{
const micGainRef = useRef<GainNode | null>(null);
const userDraftIndexRef = useRef<number | null>(null);
const lastUserFinalRef = useRef<string>('');
const selectedToolSchemas = useMemo(() => {
const ids = assistant.tools || [];
if (!ids.length) return [];
const byId = new Map(tools.map((t) => [t.id, t]));
return ids.map((id) => {
const item = byId.get(id);
return {
type: 'function',
function: {
name: item?.id || id,
description: item?.description || item?.name || id,
parameters: getDefaultToolParameters(item?.id || id),
},
};
});
}, [assistant.tools, tools]);
// Initialize
useEffect(() => {
@@ -1558,6 +1645,7 @@ export const DebugDrawer: React.FC<{
greeting: assistant.opener || '',
knowledgeBaseId,
knowledge,
tools: selectedToolSchemas,
services,
history: {
assistantId: assistant.id,
@@ -1668,6 +1756,91 @@ export const DebugDrawer: React.FC<{
return;
}
if (type === 'assistant.tool_call') {
const toolCall = payload?.tool_call || {};
const toolCallId = String(toolCall?.id || '').trim();
const toolName = String(toolCall?.function?.name || toolCall?.name || 'unknown_tool');
const rawArgs = String(toolCall?.function?.arguments || '');
const argText = rawArgs.length > 160 ? `${rawArgs.slice(0, 160)}...` : rawArgs;
setMessages((prev) => [
...prev,
{
role: 'tool',
text: `call ${toolName}${argText ? ` args=${argText}` : ''}`,
},
]);
if (toolCallId && ws.readyState === WebSocket.OPEN) {
const argsObj = parseToolArgs(toolCall?.function?.arguments);
const normalizedToolName = toolName.trim().toLowerCase();
let resultPayload: any = {
tool_call_id: toolCallId,
name: toolName,
output: {
message: 'Tool execution is not implemented in debug web client',
},
status: { code: 501, message: 'not_implemented' },
};
const isCalculatorTool =
normalizedToolName === 'calculator' ||
normalizedToolName.endsWith('.calculator') ||
normalizedToolName.includes('calculator');
if (isCalculatorTool) {
const expression = String(argsObj.expression || argsObj.input || '').trim();
if (!expression) {
resultPayload = {
tool_call_id: toolCallId,
name: toolName,
output: { error: 'missing expression' },
status: { code: 400, message: 'bad_request' },
};
} else {
const calc = evaluateCalculatorExpression(expression);
if (calc.ok) {
resultPayload = {
tool_call_id: toolCallId,
name: toolName,
output: {
expression,
result: calc.result,
},
status: { code: 200, message: 'ok' },
};
} else {
resultPayload = {
tool_call_id: toolCallId,
name: toolName,
output: { expression, error: calc.error },
status: { code: 422, message: 'invalid_expression' },
};
}
}
}
ws.send(
JSON.stringify({
type: 'tool_call.results',
results: [resultPayload],
})
);
const statusCode = Number(resultPayload?.status?.code || 500);
const statusMessage = String(resultPayload?.status?.message || 'error');
const resultText =
statusCode === 200 && typeof resultPayload?.output?.result === 'number'
? `result ${toolName} = ${resultPayload.output.result}`
: `result ${toolName} status=${statusCode} ${statusMessage}`;
setMessages((prev) => [
...prev,
{
role: 'tool',
text: resultText,
},
]);
}
return;
}
if (type === 'session.started') {
wsReadyRef.current = true;
setWsStatus('ready');
@@ -1846,7 +2019,7 @@ export const DebugDrawer: React.FC<{
if (!isOpen) return;
const localResolved = buildLocalResolvedRuntime();
setResolvedConfigView(JSON.stringify(localResolved, null, 2));
}, [isOpen, assistant, voices, llmModels, asrModels]);
}, [isOpen, assistant, voices, llmModels, asrModels, tools]);
const renderLocalVideo = (isSmall: boolean) => (
<div className={`relative w-full h-full bg-black overflow-hidden ${isSmall ? 'rounded-lg border border-white/20 shadow-lg' : ''}`}>