wx44wx/engine-v5-pipecat-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add audio logging and WebSocket enhancements to webpage

Browse Source 
- Introduced new state properties for audio send logging.
- Implemented functions to flush audio send logs and handle binary data sending.
- Updated WebSocket logging to differentiate between send and receive actions.
- Enhanced UI layout and styles for better responsiveness and usability.
- Improved overall structure and readability of the code.
This commit is contained in:
Xin Wang
2026-05-21 14:25:20 +08:00
parent 092f532f51
commit f3eab5b272
3 changed files with 347 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
examples/webpage/app.js
View File

@@ -69,6 +69,9 @@ const state = {
audioDeltaLogCount: 0, audioDeltaLogCount: 0,
audioDeltaLogBytes: 0, audioDeltaLogBytes: 0,
lastAudioDeltaLogAt: 0, lastAudioDeltaLogAt: 0,
audioSendLogCount: 0,
audioSendLogBytes: 0,
lastAudioSendLogAt: 0,
}; };
/* ------------------------------------------------------------------ UI */ /* ------------------------------------------------------------------ UI */
@@ -201,7 +204,12 @@ function addWsLog(direction, detail, kind = direction) {
const dir = document.createElement("span"); const dir = document.createElement("span");
dir.className = "ws-log__direction"; dir.className = "ws-log__direction";
dir.textContent = direction; dir.textContent =
direction === "send"
? "SEND"
: direction === "recv"
? "RECV"
: direction.toUpperCase();
const body = document.createElement("span"); const body = document.createElement("span");
body.className = "ws-log__detail"; body.className = "ws-log__detail";
@@ -227,8 +235,30 @@ function flushAudioDeltaLog() {
state.lastAudioDeltaLogAt = performance.now(); state.lastAudioDeltaLogAt = performance.now();
} }
function flushAudioSendLog() {
if (state.audioSendLogCount === 0) return;
addWsLog(
"send",
`input.audio binary x${state.audioSendLogCount} (${state.audioSendLogBytes} bytes)`,
);
state.audioSendLogCount = 0;
state.audioSendLogBytes = 0;
state.lastAudioSendLogAt = performance.now();
}
function flushPendingWsLogs() {
flushAudioDeltaLog();
flushAudioSendLog();
}
function logWsPayload(direction, payload) { function logWsPayload(direction, payload) {
if (payload?.type === "response.audio.delta") { if (direction === "send") {
flushAudioSendLog();
} else {
flushAudioDeltaLog();
}
if (direction === "recv" && payload?.type === "response.audio.delta") {
state.audioDeltaLogCount += 1; state.audioDeltaLogCount += 1;
state.audioDeltaLogBytes += payload.bytes || payload.audio?.length || 0; state.audioDeltaLogBytes += payload.bytes || payload.audio?.length || 0;
const now = performance.now(); const now = performance.now();
@@ -238,13 +268,50 @@ function logWsPayload(direction, payload) {
return; return;
} }
flushAudioDeltaLog();
addWsLog(direction, compactWsPayload(payload)); addWsLog(direction, compactWsPayload(payload));
} }
function logBinarySend(byteLength) {
state.audioSendLogCount += 1;
state.audioSendLogBytes += byteLength;
const now = performance.now();
if (now - state.lastAudioSendLogAt >= AUDIO_DELTA_LOG_INTERVAL_MS) {
flushAudioSendLog();
}
}
function wsSend(data) {
if (!state.ws || state.ws.readyState !== WebSocket.OPEN) return false;
if (typeof data === "string") {
try {
logWsPayload("send", JSON.parse(data));
} catch (_) {
flushAudioSendLog();
flushAudioDeltaLog();
addWsLog("send", truncateLogValue(data));
}
} else {
const byteLength =
data instanceof ArrayBuffer
? data.byteLength
: ArrayBuffer.isView(data)
? data.byteLength
: 0;
if (byteLength > 0) {
logBinarySend(byteLength);
}
}
state.ws.send(data);
return true;
}
function clearWsLog() { function clearWsLog() {
state.audioDeltaLogCount = 0; state.audioDeltaLogCount = 0;
state.audioDeltaLogBytes = 0; state.audioDeltaLogBytes = 0;
state.audioSendLogCount = 0;
state.audioSendLogBytes = 0;
els.wsLog.innerHTML = els.wsLog.innerHTML =
'<div class="ws-log__empty">No websocket events yet.</div>'; '<div class="ws-log__empty">No websocket events yet.</div>';
} }
@@ -335,14 +402,14 @@ async function startMic() {
const data = event.data; const data = event.data;
if (!data || data.type !== "frame") return; if (!data || data.type !== "frame") return;
updateMeter(data.rms || 0); updateMeter(data.rms || 0);
if (state.connected && state.ws && state.ws.readyState === WebSocket.OPEN) { if (state.connected) {
state.ws.send(data.buffer); wsSend(data.buffer);
} }
}; };
state.micSourceNode.connect(state.recorderNode); state.micSourceNode.connect(state.recorderNode);
state.micEnabled = true; state.micEnabled = true;
addWsLog("send", "input.audio binary stream started"); addWsLog("system", "mic capture started (binary input.audio frames)");
setMicButton(); setMicButton();
} }
@@ -378,7 +445,8 @@ function stopMic() {
state.micEnabled = false; state.micEnabled = false;
updateMeter(0); updateMeter(0);
if (wasEnabled) { if (wasEnabled) {
addWsLog("send", "input.audio binary stream stopped"); flushAudioSendLog();
addWsLog("system", "mic capture stopped");
} }
setMicButton(); setMicButton();
} }
@@ -483,8 +551,7 @@ function sendText(text) {
// `response.text.final(interrupted=true)` for the in-flight assistant // `response.text.final(interrupted=true)` for the in-flight assistant
// turn, which finalizes that bubble in place. A brand-new bubble for the // turn, which finalizes that bubble in place. A brand-new bubble for the
// reply will be created when `response.text.started` arrives. // reply will be created when `response.text.started` arrives.
state.ws.send(JSON.stringify(message)); wsSend(JSON.stringify(message));
logWsPayload("send", message);
stopPlaybackQueue(); stopPlaybackQueue();
addBubble("user", value); addBubble("user", value);
return true; return true;
@@ -629,8 +696,7 @@ async function connect() {
setMicSelectEnabled(); setMicSelectEnabled();
refreshMicDevices(); refreshMicDevices();
ws.send(JSON.stringify(startMessage)); wsSend(JSON.stringify(startMessage));
logWsPayload("send", startMessage);
addBubble("system", "Session started."); addBubble("system", "Session started.");
setComposerEnabled(true); setComposerEnabled(true);
els.textInput.focus(); els.textInput.focus();
@@ -678,7 +744,7 @@ async function connect() {
setMicSelectEnabled(); setMicSelectEnabled();
setComposerEnabled(false); setComposerEnabled(false);
setBotIndicator(false); setBotIndicator(false);
flushAudioDeltaLog(); flushPendingWsLogs();
addWsLog( addWsLog(
"system", "system",
`websocket close code=${event.code}${ `websocket close code=${event.code}${
@@ -702,8 +768,7 @@ function disconnect() {
try { try {
if (state.ws.readyState === WebSocket.OPEN) { if (state.ws.readyState === WebSocket.OPEN) {
const stopMessage = { type: "session.stop", reason: "client_disconnect" }; const stopMessage = { type: "session.stop", reason: "client_disconnect" };
state.ws.send(JSON.stringify(stopMessage)); wsSend(JSON.stringify(stopMessage));
logWsPayload("send", stopMessage);
} }
} catch (_) { } catch (_) {
/* ignore */ /* ignore */

214
examples/webpage/index.html
View File

@@ -36,111 +36,121 @@
</div> </div>
</header> </header>
<section class="chat" aria-label="Conversation history"> <div class="app__body">
<div id="chat-log" class="chat__log" role="log" aria-live="polite"> <div class="app__main">
<div class="chat__empty"> <section class="chat" aria-label="Conversation history">
<p>Connect to the engine, enable your mic, and start talking.</p> <div id="chat-log" class="chat__log" role="log" aria-live="polite">
<p class="chat__hint"> <div class="chat__empty">
Audio is streamed as PCM16 mono @ 16&nbsp;kHz over <p>Connect to the engine, enable your mic, and start talking.</p>
<code>/ws-product</code>. <p class="chat__hint">
Audio is streamed as PCM16 mono @ 16&nbsp;kHz over
<code>/ws-product</code>.
</p>
</div>
</div>
</section>
<footer class="controls" aria-label="Chat controls">
<div class="meter" aria-hidden="true">
<div id="meter-fill" class="meter__fill"></div>
</div>
<form id="composer" class="composer" autocomplete="off">
<textarea
id="text-input"
class="composer__input"
rows="1"
placeholder="Type a message, or use the mic…"
disabled
></textarea>
<button
id="send-btn"
class="btn btn--primary composer__send"
type="submit"
disabled
title="Send message (Enter)"
>
Send
</button>
</form>
<div class="controls__row">
<label class="device-picker">
<span class="device-picker__label">Microphone</span>
<select id="mic-select" class="device-picker__select" disabled>
<option value="">Default microphone</option>
</select>
</label>
<button
id="mic-btn"
class="mic-btn"
type="button"
disabled
aria-pressed="false"
title="Mic is off"
>
<svg
class="mic-btn__icon"
viewBox="0 0 24 24"
width="24"
height="24"
aria-hidden="true"
>
<path
d="M12 14a3 3 0 0 0 3-3V6a3 3 0 1 0-6 0v5a3 3 0 0 0 3 3Z"
fill="currentColor"
/>
<path
d="M19 11a1 1 0 1 0-2 0 5 5 0 0 1-10 0 1 1 0 1 0-2 0 7 7 0 0 0 6 6.92V21a1 1 0 1 0 2 0v-3.08A7 7 0 0 0 19 11Z"
fill="currentColor"
/>
</svg>
<span class="mic-btn__label">Enable mic</span>
</button>
<div class="indicators">
<span id="mic-indicator" class="indicator">
<span class="indicator__dot indicator__dot--mic"></span>
<span class="indicator__label">Mic</span>
</span>
<span id="bot-indicator" class="indicator">
<span class="indicator__dot indicator__dot--bot"></span>
<span class="indicator__label">Bot</span>
</span>
</div>
<button id="clear-btn" class="btn btn--ghost" type="button">
Clear
</button>
</div>
<p class="hint">
Press <kbd>Enter</kbd> to send, <kbd>Shift</kbd>+<kbd>Enter</kbd>
for newline. Sending text will interrupt the bot if it's speaking.
Browser echo cancellation is on; use headphones if echo persists.
</p> </p>
</footer>
</div>
<section class="ws-log" aria-label="WebSocket log">
<div class="ws-log__header">
<div class="ws-log__header-left">
<h2>WebSocket Log</h2>
<div class="ws-log__legend" aria-hidden="true">
<span class="ws-log__legend-item ws-log__legend-item--send">Send</span>
<span class="ws-log__legend-item ws-log__legend-item--recv">Recv</span>
</div>
</div>
<button id="clear-ws-log-btn" class="btn btn--ghost" type="button">
Clear log
</button>
</div> </div>
</div> <div id="ws-log" class="ws-log__body" role="log" aria-live="polite">
</section> <div class="ws-log__empty">No websocket events yet.</div>
<section class="ws-log" aria-label="WebSocket log">
<div class="ws-log__header">
<h2>WebSocket Log</h2>
<button id="clear-ws-log-btn" class="btn btn--ghost" type="button">
Clear log
</button>
</div>
<div id="ws-log" class="ws-log__body" role="log" aria-live="polite">
<div class="ws-log__empty">No websocket events yet.</div>
</div>
</section>
<footer class="controls" aria-label="Chat controls">
<div class="meter" aria-hidden="true">
<div id="meter-fill" class="meter__fill"></div>
</div>
<form id="composer" class="composer" autocomplete="off">
<textarea
id="text-input"
class="composer__input"
rows="1"
placeholder="Type a message, or use the mic…"
disabled
></textarea>
<button
id="send-btn"
class="btn btn--primary composer__send"
type="submit"
disabled
title="Send message (Enter)"
>
Send
</button>
</form>
<div class="controls__row">
<label class="device-picker">
<span class="device-picker__label">Microphone</span>
<select id="mic-select" class="device-picker__select" disabled>
<option value="">Default microphone</option>
</select>
</label>
<button
id="mic-btn"
class="mic-btn"
type="button"
disabled
aria-pressed="false"
title="Mic is off"
>
<svg
class="mic-btn__icon"
viewBox="0 0 24 24"
width="24"
height="24"
aria-hidden="true"
>
<path
d="M12 14a3 3 0 0 0 3-3V6a3 3 0 1 0-6 0v5a3 3 0 0 0 3 3Z"
fill="currentColor"
/>
<path
d="M19 11a1 1 0 1 0-2 0 5 5 0 0 1-10 0 1 1 0 1 0-2 0 7 7 0 0 0 6 6.92V21a1 1 0 1 0 2 0v-3.08A7 7 0 0 0 19 11Z"
fill="currentColor"
/>
</svg>
<span class="mic-btn__label">Enable mic</span>
</button>
<div class="indicators">
<span id="mic-indicator" class="indicator">
<span class="indicator__dot indicator__dot--mic"></span>
<span class="indicator__label">Mic</span>
</span>
<span id="bot-indicator" class="indicator">
<span class="indicator__dot indicator__dot--bot"></span>
<span class="indicator__label">Bot</span>
</span>
</div> </div>
</section>
<button id="clear-btn" class="btn btn--ghost" type="button"> </div>
Clear
</button>
</div>
<p class="hint">
Press <kbd>Enter</kbd> to send, <kbd>Shift</kbd>+<kbd>Enter</kbd>
for newline. Sending text will interrupt the bot if it's speaking.
Browser echo cancellation is on; use headphones if echo persists.
</p>
</footer>
</main> </main>
<script type="module" src="./app.js"></script> <script type="module" src="./app.js"></script>

211
examples/webpage/styles.css
View File

@@ -45,30 +45,49 @@ body {
body { body {
display: flex; display: flex;
justify-content: center; justify-content: center;
padding: 24px; padding: 12px;
} }
.app { .app {
display: grid; display: grid;
grid-template-rows: auto minmax(0, 1fr) auto auto; grid-template-rows: auto minmax(0, 1fr);
gap: 16px; gap: 12px;
width: min(880px, 100%); width: min(1440px, 100%);
height: calc(100vh - 48px); height: calc(100dvh - 24px);
max-height: calc(100vh - 24px);
background: var(--bg-elevated); background: var(--bg-elevated);
border: 1px solid var(--border); border: 1px solid var(--border);
border-radius: 20px; border-radius: 20px;
box-shadow: var(--shadow); box-shadow: var(--shadow);
padding: 18px; padding: 14px;
}
.app__body {
display: grid;
grid-template-columns: minmax(0, 1fr) clamp(300px, 32vw, 420px);
gap: 12px;
min-height: 0;
}
.app__main {
display: grid;
grid-template-rows: minmax(0, 1fr) auto;
gap: 0;
min-height: 0;
overflow: hidden;
background: var(--bg);
border: 1px solid var(--border);
border-radius: var(--radius);
} }
/* Header ---------------------------------------------------------------- */ /* Header ---------------------------------------------------------------- */
.app__header { .app__header {
display: grid; display: grid;
grid-template-columns: auto 1fr auto; grid-template-columns: auto minmax(0, 1fr) auto;
align-items: center; align-items: center;
gap: 16px; gap: 12px;
padding-bottom: 14px; padding-bottom: 10px;
border-bottom: 1px solid var(--border); border-bottom: 1px solid var(--border);
} }
@@ -119,11 +138,12 @@ body {
color: var(--text); color: var(--text);
border: 1px solid var(--border); border: 1px solid var(--border);
border-radius: 10px; border-radius: 10px;
padding: 8px 10px; padding: 7px 10px;
font: inherit; font: inherit;
font-size: 13px; font-size: 12px;
outline: none; outline: none;
min-width: 240px; width: 100%;
min-width: 0;
} }
.connection__field input:focus { .connection__field input:focus {
@@ -168,11 +188,9 @@ body {
.chat { .chat {
overflow: hidden; overflow: hidden;
background: var(--bg);
border: 1px solid var(--border);
border-radius: var(--radius);
display: flex; display: flex;
flex-direction: column; flex-direction: column;
min-height: 0;
} }
.chat__log { .chat__log {
@@ -255,6 +273,9 @@ body {
/* WebSocket log --------------------------------------------------------- */ /* WebSocket log --------------------------------------------------------- */
.ws-log { .ws-log {
display: flex;
flex-direction: column;
min-height: 0;
background: var(--bg); background: var(--bg);
border: 1px solid var(--border); border: 1px solid var(--border);
border-radius: var(--radius); border-radius: var(--radius);
@@ -265,9 +286,40 @@ body {
display: flex; display: flex;
align-items: center; align-items: center;
justify-content: space-between; justify-content: space-between;
gap: 12px; gap: 8px;
padding: 8px 10px; padding: 8px 12px;
border-bottom: 1px solid var(--border); border-bottom: 1px solid var(--border);
flex-shrink: 0;
background: var(--bg-soft);
}
.ws-log__legend {
display: flex;
align-items: center;
gap: 8px;
font-size: 10px;
color: var(--text-dim);
}
.ws-log__legend-item {
display: inline-flex;
align-items: center;
gap: 4px;
}
.ws-log__legend-item::before {
content: "";
width: 8px;
height: 8px;
border-radius: 2px;
}
.ws-log__legend-item--send::before {
background: var(--success);
}
.ws-log__legend-item--recv::before {
background: var(--accent-strong);
} }
.ws-log__header h2 { .ws-log__header h2 {
@@ -278,54 +330,89 @@ body {
letter-spacing: 0.8px; letter-spacing: 0.8px;
} }
.ws-log__body { .ws-log__header-left {
max-height: 130px; display: flex;
overflow-y: auto; align-items: center;
padding: 8px 10px; gap: 12px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace; min-width: 0;
font-size: 11px;
line-height: 1.45;
color: var(--text-dim);
} }
.ws-log__empty { .ws-log__body {
flex: 1;
min-height: 0;
overflow-y: auto;
overflow-x: hidden;
padding: 6px 8px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 10.5px;
line-height: 1.4;
color: var(--text-dim); color: var(--text-dim);
opacity: 0.7;
} }
.ws-log__entry { .ws-log__entry {
display: grid; display: grid;
grid-template-columns: 72px 44px 1fr; grid-template-columns: 58px 42px minmax(0, 1fr);
gap: 8px; gap: 6px;
align-items: start;
padding: 5px 4px;
border-radius: 6px;
white-space: pre-wrap; white-space: pre-wrap;
word-break: break-word; word-break: break-word;
} }
.ws-log__entry:hover {
background: rgba(255, 255, 255, 0.03);
}
.ws-log__time { .ws-log__time {
color: var(--text-dim); color: var(--text-dim);
opacity: 0.75; opacity: 0.7;
font-size: 10px;
} }
.ws-log__direction { .ws-log__direction {
color: var(--accent-strong); font-weight: 700;
font-size: 9px;
letter-spacing: 0.4px;
text-transform: uppercase; text-transform: uppercase;
padding: 2px 0;
}
.ws-log__detail {
min-width: 0;
overflow-wrap: anywhere;
} }
.ws-log__entry--send .ws-log__direction { .ws-log__entry--send .ws-log__direction {
color: var(--success); color: var(--success);
} }
.ws-log__entry--recv .ws-log__direction {
color: var(--accent-strong);
}
.ws-log__entry--system .ws-log__direction {
color: var(--text-dim);
}
.ws-log__entry--error .ws-log__direction { .ws-log__entry--error .ws-log__direction {
color: var(--danger); color: var(--danger);
} }
.ws-log__empty {
color: var(--text-dim);
opacity: 0.7;
padding: 8px 4px;
}
/* Controls -------------------------------------------------------------- */ /* Controls -------------------------------------------------------------- */
.controls { .controls {
display: grid; display: grid;
gap: 10px; gap: 8px;
padding: 14px 16px 6px; padding: 10px 12px 8px;
border-top: 1px solid var(--border); border-top: 1px solid var(--border);
background: var(--bg-elevated);
} }
.meter { .meter {
@@ -390,14 +477,17 @@ body {
.controls__row { .controls__row {
display: flex; display: flex;
align-items: center; align-items: center;
gap: 14px; gap: 10px;
flex-wrap: wrap;
} }
.device-picker { .device-picker {
display: flex; display: flex;
flex-direction: column; flex-direction: column;
gap: 4px; gap: 3px;
min-width: 220px; min-width: 0;
flex: 1 1 160px;
max-width: 240px;
} }
.device-picker__label { .device-picker__label {
@@ -413,11 +503,11 @@ body {
color: var(--text); color: var(--text);
border: 1px solid var(--border); border: 1px solid var(--border);
border-radius: 10px; border-radius: 10px;
padding: 8px 30px 8px 10px; padding: 7px 28px 7px 10px;
font: inherit; font: inherit;
font-size: 13px; font-size: 12px;
outline: none; outline: none;
max-width: 300px; width: 100%;
cursor: pointer; cursor: pointer;
} }
@@ -434,15 +524,17 @@ body {
.mic-btn { .mic-btn {
display: inline-flex; display: inline-flex;
align-items: center; align-items: center;
gap: 8px; gap: 6px;
padding: 9px 14px; padding: 8px 12px;
border-radius: 999px; border-radius: 999px;
background: var(--bg-soft); background: var(--bg-soft);
color: var(--text); color: var(--text);
border: 1px solid var(--border); border: 1px solid var(--border);
font: inherit; font: inherit;
font-size: 12px;
font-weight: 600; font-weight: 600;
cursor: pointer; cursor: pointer;
flex-shrink: 0;
transition: transform 0.08s ease, background 0.15s ease, color 0.15s ease, transition: transform 0.08s ease, background 0.15s ease, color 0.15s ease,
border-color 0.15s ease; border-color 0.15s ease;
} }
@@ -549,9 +641,10 @@ body {
} }
.hint { .hint {
margin: 4px 2px 0; margin: 0;
font-size: 12px; font-size: 11px;
color: var(--text-dim); color: var(--text-dim);
opacity: 0.85;
} }
.hint kbd { .hint kbd {
@@ -590,16 +683,32 @@ body {
/* Responsive ------------------------------------------------------------ */ /* Responsive ------------------------------------------------------------ */
@media (max-width: 820px) {
.app__body {
grid-template-columns: 1fr;
grid-template-rows: minmax(0, 1fr) min(240px, 32vh);
}
.ws-log__legend {
display: none;
}
.hint {
display: none;
}
}
@media (max-width: 720px) { @media (max-width: 720px) {
body { body {
padding: 0; padding: 0;
} }
.app { .app {
height: 100vh; height: 100dvh;
max-height: 100vh;
border-radius: 0; border-radius: 0;
border: none; border: none;
padding: 12px; padding: 10px;
} }
.app__header { .app__header {
@@ -612,24 +721,16 @@ body {
} }
.ws-log__entry { .ws-log__entry {
grid-template-columns: 68px 40px 1fr; grid-template-columns: 54px 38px minmax(0, 1fr);
} }
.status { .status {
justify-content: flex-end; justify-content: flex-end;
} }
.controls__row {
flex-wrap: wrap;
}
.device-picker { .device-picker {
min-width: 100%; flex: 1 1 100%;
}
.device-picker__select {
max-width: none; max-width: none;
width: 100%;
} }
.indicators { .indicators {