diff --git a/examples/webpage/README.md b/examples/webpage/README.md index 8bcc835..239a09a 100644 --- a/examples/webpage/README.md +++ b/examples/webpage/README.md @@ -6,7 +6,8 @@ A self-contained browser client for the engine's product websocket ## Features - **Connect / Disconnect** to any `ws://` or `wss://` URL. -- **Mic on/off toggle** — getUserMedia is requested with +- **Microphone selector + mic on/off toggle** — available input devices + are listed with `enumerateDevices`, and getUserMedia is requested with `echoCancellation`, `noiseSuppression`, and `autoGainControl` so the browser handles AEC against the bot's voice. - **Text composer** — type a message and press Enter to send @@ -17,6 +18,8 @@ A self-contained browser client for the engine's product websocket (assistant — deltas arrive ahead of the synthesized audio), and locally for text you submit (the engine doesn't echo text input back as a transcript). +- **WebSocket log** panel for connection state and compact send/receive + events. Audio chunks are summarized so the UI does not flood. - **Gapless TTS playback** by scheduling each `response.audio.delta` chunk back-to-back on the AudioContext. - **Live VU meter** + mic and bot activity indicators. @@ -56,8 +59,9 @@ examples/webpage/ 3. Open in Chrome, Edge, or Safari. - Click **Connect** (uses `ws://127.0.0.1:8001/ws-product` by default). - - Click **Enable mic** and start speaking. The browser will prompt - for microphone access on first use. + - Pick a microphone if needed, click **Enable mic**, and start + speaking. The browser will prompt for microphone access on first + use. Device names may appear only after permission is granted. > The browser's mic API requires a secure context. `http://localhost` > qualifies; if you serve from another host, use HTTPS and a `wss://` diff --git a/examples/webpage/app.js b/examples/webpage/app.js index 3c63592..d5a46bb 100644 --- a/examples/webpage/app.js +++ b/examples/webpage/app.js @@ -4,7 +4,7 @@ * * Responsibilities: * - Open/close the websocket and run the session handshake. - * - Capture mic audio with echoCancellation, noiseSuppression, autoGainControl. + * - List/select microphones and capture mic audio with browser AEC enabled. * - Downsample to PCM16 mono @ 16 kHz in an AudioWorklet and stream frames * as binary websocket messages. * - Play `response.audio.delta` frames gaplessly through Web Audio. @@ -15,6 +15,8 @@ const SAMPLE_RATE = 16000; const CHANNELS = 1; const FRAME_MS = 20; const PROTOCOL = "va.ws.v1"; +const MAX_WS_LOG_LINES = 120; +const AUDIO_DELTA_LOG_INTERVAL_MS = 1000; const els = { url: document.getElementById("ws-url"), @@ -23,10 +25,13 @@ const els = { statusText: document.getElementById("status-text"), chatLog: document.getElementById("chat-log"), micBtn: document.getElementById("mic-btn"), + micSelect: document.getElementById("mic-select"), micLabel: document.querySelector(".mic-btn__label"), micIndicator: document.getElementById("mic-indicator"), botIndicator: document.getElementById("bot-indicator"), clearBtn: document.getElementById("clear-btn"), + clearWsLogBtn: document.getElementById("clear-ws-log-btn"), + wsLog: document.getElementById("ws-log"), meterFill: document.getElementById("meter-fill"), composer: document.getElementById("composer"), textInput: document.getElementById("text-input"), @@ -44,6 +49,8 @@ const state = { recorderNode: null, micEnabled: false, + micDevices: [], + selectedMicDeviceId: "", // Output scheduling. nextPlaybackTime: 0, @@ -57,6 +64,11 @@ const state = { // VU meter smoothing. meterLevel: 0, + + // Compact websocket logging. + audioDeltaLogCount: 0, + audioDeltaLogBytes: 0, + lastAudioDeltaLogAt: 0, }; /* ------------------------------------------------------------------ UI */ @@ -90,6 +102,10 @@ function setMicButton() { els.micIndicator.classList.toggle("is-active", state.micEnabled); } +function setMicSelectEnabled() { + els.micSelect.disabled = !state.connected || !navigator.mediaDevices; +} + function setComposerEnabled(enabled) { els.textInput.disabled = !enabled; els.sendBtn.disabled = !enabled || els.textInput.value.trim().length === 0; @@ -139,6 +155,100 @@ function clearChat() { els.chatLog.appendChild(empty); } +function truncateLogValue(value, maxLength = 160) { + const text = String(value); + if (text.length <= maxLength) return text; + return `${text.slice(0, maxLength - 1)}…`; +} + +function compactWsPayload(payload) { + if (!payload || typeof payload !== "object") return String(payload); + const compact = { ...payload }; + + if (typeof compact.audio === "string") { + compact.audio = ``; + } + if (typeof compact.data === "string" && compact.data.length > 160) { + compact.data = ``; + } + if (typeof compact.text === "string") { + compact.text = truncateLogValue(compact.text); + } + + try { + return JSON.stringify(compact); + } catch (_) { + return payload.type || "unserializable websocket payload"; + } +} + +function addWsLog(direction, detail, kind = direction) { + if (els.wsLog.querySelector(".ws-log__empty")) { + els.wsLog.innerHTML = ""; + } + + const entry = document.createElement("div"); + entry.className = `ws-log__entry ws-log__entry--${kind}`; + + const time = document.createElement("span"); + time.className = "ws-log__time"; + time.textContent = new Date().toLocaleTimeString([], { + hour12: false, + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }); + + const dir = document.createElement("span"); + dir.className = "ws-log__direction"; + dir.textContent = direction; + + const body = document.createElement("span"); + body.className = "ws-log__detail"; + body.textContent = detail; + + entry.append(time, dir, body); + els.wsLog.appendChild(entry); + + while (els.wsLog.children.length > MAX_WS_LOG_LINES) { + els.wsLog.firstElementChild.remove(); + } + els.wsLog.scrollTop = els.wsLog.scrollHeight; +} + +function flushAudioDeltaLog() { + if (state.audioDeltaLogCount === 0) return; + addWsLog( + "recv", + `response.audio.delta x${state.audioDeltaLogCount} (${state.audioDeltaLogBytes} bytes)`, + ); + state.audioDeltaLogCount = 0; + state.audioDeltaLogBytes = 0; + state.lastAudioDeltaLogAt = performance.now(); +} + +function logWsPayload(direction, payload) { + if (payload?.type === "response.audio.delta") { + state.audioDeltaLogCount += 1; + state.audioDeltaLogBytes += payload.bytes || payload.audio?.length || 0; + const now = performance.now(); + if (now - state.lastAudioDeltaLogAt >= AUDIO_DELTA_LOG_INTERVAL_MS) { + flushAudioDeltaLog(); + } + return; + } + + flushAudioDeltaLog(); + addWsLog(direction, compactWsPayload(payload)); +} + +function clearWsLog() { + state.audioDeltaLogCount = 0; + state.audioDeltaLogBytes = 0; + els.wsLog.innerHTML = + '
No websocket events yet.
'; +} + /* ---------------------------------------------------------------- Audio */ async function ensureAudioContext() { @@ -153,18 +263,63 @@ async function ensureAudioContext() { return state.audioContext; } +function renderMicDevices() { + const previousValue = state.selectedMicDeviceId || els.micSelect.value; + els.micSelect.innerHTML = ""; + + const defaultOption = document.createElement("option"); + defaultOption.value = ""; + defaultOption.textContent = "Default microphone"; + els.micSelect.appendChild(defaultOption); + + state.micDevices.forEach((device, index) => { + const option = document.createElement("option"); + option.value = device.deviceId; + option.textContent = device.label || `Microphone ${index + 1}`; + els.micSelect.appendChild(option); + }); + + const hasPrevious = state.micDevices.some( + (device) => device.deviceId === previousValue, + ); + state.selectedMicDeviceId = hasPrevious ? previousValue : ""; + els.micSelect.value = state.selectedMicDeviceId; + setMicSelectEnabled(); +} + +async function refreshMicDevices() { + if (!navigator.mediaDevices?.enumerateDevices) { + setMicSelectEnabled(); + return; + } + + try { + const devices = await navigator.mediaDevices.enumerateDevices(); + state.micDevices = devices.filter((device) => device.kind === "audioinput"); + renderMicDevices(); + } catch (err) { + console.warn("Could not enumerate microphones", err); + setMicSelectEnabled(); + } +} + async function startMic() { const ctx = await ensureAudioContext(); + const audioConstraints = { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + channelCount: 1, + }; + if (state.selectedMicDeviceId) { + audioConstraints.deviceId = { exact: state.selectedMicDeviceId }; + } state.micStream = await navigator.mediaDevices.getUserMedia({ - audio: { - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true, - channelCount: 1, - }, + audio: audioConstraints, video: false, }); + await refreshMicDevices(); state.micSourceNode = ctx.createMediaStreamSource(state.micStream); state.recorderNode = new AudioWorkletNode(ctx, "pcm-recorder", { @@ -187,10 +342,12 @@ async function startMic() { state.micSourceNode.connect(state.recorderNode); state.micEnabled = true; + addWsLog("send", "input.audio binary stream started"); setMicButton(); } function stopMic() { + const wasEnabled = state.micEnabled; if (state.recorderNode) { try { state.recorderNode.port.onmessage = null; @@ -220,6 +377,9 @@ function stopMic() { } state.micEnabled = false; updateMeter(0); + if (wasEnabled) { + addWsLog("send", "input.audio binary stream stopped"); + } setMicButton(); } @@ -310,6 +470,11 @@ function sendText(text) { const value = (text || "").trim(); if (!value) return false; if (!state.ws || state.ws.readyState !== WebSocket.OPEN) return false; + const message = { + type: "input.text", + text: value, + interrupt: true, + }; // The engine does not echo text input back as a transcript event, so we // render the user bubble locally. Also interrupt any in-flight bot audio @@ -318,13 +483,8 @@ function sendText(text) { // `response.text.final(interrupted=true)` for the in-flight assistant // turn, which finalizes that bubble in place. A brand-new bubble for the // reply will be created when `response.text.started` arrives. - state.ws.send( - JSON.stringify({ - type: "input.text", - text: value, - interrupt: true, - }), - ); + state.ws.send(JSON.stringify(message)); + logWsPayload("send", message); stopPlaybackQueue(); addBubble("user", value); return true; @@ -420,6 +580,7 @@ async function connect() { state.connecting = true; setStatus("connecting", "Connecting…"); setConnectButton(); + addWsLog("system", `connecting ${url}`); try { // Pre-warm audio context on user gesture so playback works on Safari. @@ -429,6 +590,7 @@ async function connect() { state.connecting = false; setStatus("error", "Audio init failed"); setConnectButton(); + addWsLog("error", `audio init failed: ${err.message || err}`, "error"); return; } @@ -440,30 +602,35 @@ async function connect() { state.connecting = false; setStatus("error", "Bad URL"); setConnectButton(); + addWsLog("error", `bad websocket URL: ${err.message || err}`, "error"); return; } ws.binaryType = "arraybuffer"; state.ws = ws; ws.addEventListener("open", () => { + const startMessage = { + type: "session.start", + protocol: PROTOCOL, + audio: { + encoding: "pcm_s16le", + sample_rate: SAMPLE_RATE, + channels: CHANNELS, + }, + }; + state.connecting = false; state.connected = true; resetPlaybackClock(); + addWsLog("system", "websocket open"); setStatus("connected", "Connected"); setConnectButton(); setMicButton(); + setMicSelectEnabled(); + refreshMicDevices(); - ws.send( - JSON.stringify({ - type: "session.start", - protocol: PROTOCOL, - audio: { - encoding: "pcm_s16le", - sample_rate: SAMPLE_RATE, - channels: CHANNELS, - }, - }), - ); + ws.send(JSON.stringify(startMessage)); + logWsPayload("send", startMessage); addBubble("system", "Session started."); setComposerEnabled(true); els.textInput.focus(); @@ -477,11 +644,18 @@ async function connect() { parsed = JSON.parse(data); } catch (err) { console.warn("Bad JSON from server", err, data); + addWsLog( + "error", + `invalid JSON from server: ${truncateLogValue(data)}`, + "error", + ); return; } + logWsPayload("recv", parsed); handleEvent(parsed); } else if (data instanceof ArrayBuffer) { // Server doesn't currently send binary, but handle it just in case. + addWsLog("recv", `binary audio ${data.byteLength} bytes`); schedulePlayback(new Int16Array(data)); } }); @@ -489,6 +663,7 @@ async function connect() { ws.addEventListener("error", (err) => { console.error("WebSocket error", err); setStatus("error", "Connection error"); + addWsLog("error", "websocket error", "error"); }); ws.addEventListener("close", (event) => { @@ -500,8 +675,16 @@ async function connect() { stopPlaybackQueue(); setConnectButton(); setMicButton(); + setMicSelectEnabled(); setComposerEnabled(false); setBotIndicator(false); + flushAudioDeltaLog(); + addWsLog( + "system", + `websocket close code=${event.code}${ + event.reason ? ` reason=${event.reason}` : "" + }`, + ); if (wasConnected) { addBubble( "system", @@ -518,9 +701,9 @@ function disconnect() { if (!state.ws) return; try { if (state.ws.readyState === WebSocket.OPEN) { - state.ws.send( - JSON.stringify({ type: "session.stop", reason: "client_disconnect" }), - ); + const stopMessage = { type: "session.stop", reason: "client_disconnect" }; + state.ws.send(JSON.stringify(stopMessage)); + logWsPayload("send", stopMessage); } } catch (_) { /* ignore */ @@ -556,10 +739,36 @@ els.micBtn.addEventListener("click", async () => { } }); +els.micSelect.addEventListener("change", async () => { + state.selectedMicDeviceId = els.micSelect.value; + if (!state.micEnabled) return; + + els.micSelect.disabled = true; + els.micBtn.disabled = true; + try { + stopMic(); + await startMic(); + } catch (err) { + console.error("Mic switch error", err); + addBubble("system", `Mic switch error: ${err.message || err}`); + } finally { + setMicButton(); + setMicSelectEnabled(); + } +}); + +if (navigator.mediaDevices?.addEventListener) { + navigator.mediaDevices.addEventListener("devicechange", refreshMicDevices); +} + els.clearBtn.addEventListener("click", () => { clearChat(); }); +els.clearWsLogBtn.addEventListener("click", () => { + clearWsLog(); +}); + function autosizeTextarea() { const ta = els.textInput; ta.style.height = "auto"; @@ -611,4 +820,5 @@ window.addEventListener("beforeunload", () => { setStatus("idle", "Disconnected"); setConnectButton(); setMicButton(); +setMicSelectEnabled(); setComposerEnabled(false); diff --git a/examples/webpage/index.html b/examples/webpage/index.html index 29b06dc..440e078 100644 --- a/examples/webpage/index.html +++ b/examples/webpage/index.html @@ -48,6 +48,18 @@ +
+
+

WebSocket Log

+ +
+
+
No websocket events yet.
+
+
+