diff --git a/web/pages/Assistants.tsx b/web/pages/Assistants.tsx index 6b64919..1da3915 100644 --- a/web/pages/Assistants.tsx +++ b/web/pages/Assistants.tsx @@ -1329,6 +1329,7 @@ export const DebugDrawer: React.FC<{ onProtocolEvent, }) => { const TARGET_SAMPLE_RATE = 16000; + const PCM_FRAME_BYTES = 640; // WS v1 fixed 20ms frame for 16k mono pcm_s16le const downsampleTo16k = (input: Float32Array, inputSampleRate: number): Float32Array => { if (inputSampleRate === TARGET_SAMPLE_RATE) return input; if (inputSampleRate < TARGET_SAMPLE_RATE) return input; @@ -1367,6 +1368,7 @@ export const DebugDrawer: React.FC<{ const [textSessionStarted, setTextSessionStarted] = useState(false); const [wsStatus, setWsStatus] = useState<'disconnected' | 'connecting' | 'ready' | 'error'>('disconnected'); const [wsError, setWsError] = useState(''); + const wsStatusRef = useRef<'disconnected' | 'connecting' | 'ready' | 'error'>('disconnected'); const [resolvedConfigOpen, setResolvedConfigOpen] = useState(false); const [resolvedConfigView, setResolvedConfigView] = useState(''); const [captureConfigOpen, setCaptureConfigOpen] = useState(false); @@ -1410,6 +1412,7 @@ export const DebugDrawer: React.FC<{ const micSourceRef = useRef(null); const micProcessorRef = useRef(null); const micGainRef = useRef(null); + const micFrameBufferRef = useRef(new Uint8Array(0)); const userDraftIndexRef = useRef(null); const lastUserFinalRef = useRef(''); const selectedToolSchemas = useMemo(() => { @@ -1461,6 +1464,10 @@ export const DebugDrawer: React.FC<{ localStorage.setItem('debug_ws_url', wsUrl); }, [wsUrl]); + useEffect(() => { + wsStatusRef.current = wsStatus; + }, [wsStatus]); + useEffect(() => { localStorage.setItem('debug_audio_aec', aecEnabled ? '1' : '0'); }, [aecEnabled]); @@ -1540,10 +1547,31 @@ export const DebugDrawer: React.FC<{ void micAudioCtxRef.current.close(); micAudioCtxRef.current = null; } + micFrameBufferRef.current = new Uint8Array(0); setCaptureConfigView(''); stopMedia(); }; + const sendFramedMicAudio = (pcm16: Int16Array) => { + const ws = wsRef.current; + if (!ws || ws.readyState !== WebSocket.OPEN || !wsReadyRef.current) return; + if (pcm16.byteLength <= 0) return; + + const chunkBytes = new Uint8Array(pcm16.buffer, pcm16.byteOffset, pcm16.byteLength); + const pending = micFrameBufferRef.current; + const merged = new Uint8Array(pending.length + chunkBytes.length); + merged.set(pending, 0); + merged.set(chunkBytes, pending.length); + + let offset = 0; + while (merged.length - offset >= PCM_FRAME_BYTES) { + ws.send(merged.subarray(offset, offset + PCM_FRAME_BYTES)); + offset += PCM_FRAME_BYTES; + } + + micFrameBufferRef.current = offset >= merged.length ? new Uint8Array(0) : merged.slice(offset); + }; + const buildMicConstraints = (): MediaTrackConstraints => ({ deviceId: selectedMic ? { exact: selectedMic } : undefined, echoCancellation: aecEnabled, @@ -1596,7 +1624,7 @@ export const DebugDrawer: React.FC<{ const inChannel = event.inputBuffer.getChannelData(0); const downsampled = downsampleTo16k(inChannel, event.inputBuffer.sampleRate); const pcm16 = float32ToPcm16(downsampled); - wsRef.current.send(pcm16.buffer); + sendFramedMicAudio(pcm16); }; micSourceRef.current = source; @@ -1949,6 +1977,7 @@ export const DebugDrawer: React.FC<{ assistantDraftIndexRef.current = null; userDraftIndexRef.current = null; lastUserFinalRef.current = ''; + micFrameBufferRef.current = new Uint8Array(0); setTextSessionStarted(false); stopPlaybackImmediately(); if (isOpen) setWsStatus('disconnected'); @@ -2096,6 +2125,14 @@ export const DebugDrawer: React.FC<{ return; } + if (type === 'config.resolved') { + const resolved = payload?.config || payload?.data?.config; + if (resolved) { + setResolvedConfigView(JSON.stringify({ config: resolved }, null, 2)); + } + return; + } + if (type === 'input.speech_started') { setIsLoading(true); return; @@ -2225,7 +2262,7 @@ export const DebugDrawer: React.FC<{ } if (type === 'error') { - const message = String(payload.message || 'Unknown error'); + const message = String(payload?.message || payload?.data?.error?.message || 'Unknown error'); setWsStatus('error'); setWsError(message); setIsLoading(false); @@ -2251,7 +2288,7 @@ export const DebugDrawer: React.FC<{ setTextSessionStarted(false); userDraftIndexRef.current = null; stopPlaybackImmediately(); - if (wsStatus !== 'error') setWsStatus('disconnected'); + if (wsStatusRef.current !== 'error') setWsStatus('disconnected'); }; }); };