wx44wx/AI-VideoAssistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement WS v1 protocol and runtime-config powered debug drawer

Browse Source
This commit is contained in:
Xin Wang
2026-02-09 08:19:39 +08:00
parent 0fc56e2685
commit fb6d1eb1da
13 changed files with 986 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
engine/examples/web_client.html
View File

@@ -547,7 +547,7 @@
setStatus(true, "Session open");
logLine("sys", "WebSocket connected");
ensureAudioContext();
sendCommand({ command: "invite", option: { codec: "pcm", sampleRate: targetSampleRate } });
sendCommand({ type: "hello", version: "v1" });
};
ws.onclose = () => {
@@ -574,7 +574,10 @@
}
function disconnect() {
if (ws) ws.close();
if (ws && ws.readyState === WebSocket.OPEN) {
sendCommand({ type: "session.stop", reason: "client_disconnect" });
ws.close();
}
ws = null;
setStatus(false, "Disconnected");
}
@@ -585,40 +588,48 @@
return;
}
ws.send(JSON.stringify(cmd));
logLine("sys", `→ ${cmd.command}`, cmd);
logLine("sys", `→ ${cmd.type}`, cmd);
}
function handleEvent(event) {
const type = event.event || "unknown";
const type = event.type || "unknown";
logLine("event", type, event);
if (type === "transcript") {
if (event.isFinal && event.text) {
if (type === "hello.ack") {
sendCommand({
type: "session.start",
audio: { encoding: "pcm_s16le", sample_rate_hz: targetSampleRate, channels: 1 },
});
}
if (type === "transcript.final") {
if (event.text) {
setInterim("You", "");
addChat("You", event.text);
} else if (event.text) {
interimUserText += event.text;
setInterim("You", interimUserText);
}
}
if (type === "llmResponse") {
if (event.isFinal && event.text) {
if (type === "transcript.delta" && event.text) {
setInterim("You", event.text);
}
if (type === "assistant.response.final") {
if (event.text) {
setInterim("AI", "");
addChat("AI", event.text);
} else if (event.text) {
interimAiText += event.text;
setInterim("AI", interimAiText);
}
}
if (type === "trackStart") {
if (type === "assistant.response.delta" && event.text) {
interimAiText += event.text;
setInterim("AI", interimAiText);
}
if (type === "output.audio.start") {
// New bot audio: stop any previous playback to avoid overlap
stopPlayback();
discardAudio = false;
interimAiText = "";
}
if (type === "speaking") {
if (type === "input.speech_started") {
// User started speaking: clear any in-flight audio to avoid overlap
stopPlayback();
}
if (type === "interrupt") {
if (type === "response.interrupted") {
stopPlayback();
}
}
@@ -716,7 +727,7 @@
if (!text) return;
ensureAudioContext();
addChat("You", text);
sendCommand({ command: "chat", text });
sendCommand({ type: "input.text", text });
chatInput.value = "";
});
clearLogBtn.addEventListener("click", () => {