POC: Add MoQ transport
This commit is contained in:
vipyne
878
moq_prebuilt/client/app.js
Normal file
878
moq_prebuilt/client/app.js
Normal file
@@ -0,0 +1,878 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026, Daily
|
||||
*
|
||||
* SPDX-License-Identifier: BSD 2-Clause License
|
||||
*/
|
||||
|
||||
/**
|
||||
* MOQ browser client — moq-lite-02.
|
||||
*
|
||||
* Connects to a moq-lite relay via WebTransport, captures microphone audio,
|
||||
* and plays back received audio using stream-per-request model.
|
||||
*
|
||||
* Flow:
|
||||
* 1. WebTransport connect
|
||||
* 2. Open bidi stream for SETUP: write CLIENT_SETUP, read SERVER_SETUP
|
||||
* 3. Listen for incoming bidi streams (relay sends SUBSCRIBE / ANNOUNCE)
|
||||
* 4. Listen for incoming uni streams (receive bot audio as GROUP + FRAME)
|
||||
* 5. Open bidi stream for SUBSCRIBE (to bot's audio track)
|
||||
* 6. Send mic PCM as uni streams (GROUP + FRAME per chunk)
|
||||
*/
|
||||
|
||||
const {
|
||||
MOQL_VERSION,
|
||||
Role,
|
||||
StreamType,
|
||||
CLIENT_SETUP_TYPE,
|
||||
SERVER_SETUP_TYPE,
|
||||
concat,
|
||||
encodeClientSetup,
|
||||
parseServerSetup,
|
||||
encodeSubscribe,
|
||||
encodeSubscribeOk,
|
||||
decodeSubscribe,
|
||||
encodeGroupAndFrame,
|
||||
parseGroupStream,
|
||||
encodeVarint,
|
||||
decodeVarint,
|
||||
encodeString,
|
||||
decodeString,
|
||||
} = window.MOQ;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// State
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let transport = null;
|
||||
let micStream = null;
|
||||
let audioContext = null;
|
||||
let micWorklet = null;
|
||||
let connected = false;
|
||||
|
||||
// Subscription IDs
|
||||
let nextSubscribeId = 1;
|
||||
|
||||
// Publishing state
|
||||
let pubSubscribeId = null; // set when relay subscribes to our audio
|
||||
let pubGroupSeq = 0;
|
||||
|
||||
// Subscription state (for receiving bot audio and transcript)
|
||||
let subAudioSubscribeId = null;
|
||||
let subTranscriptSubscribeId = null;
|
||||
|
||||
// Playback
|
||||
let playbackTime = 0;
|
||||
|
||||
// Config (populated from /api/config)
|
||||
let config = {};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// UI helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function setStatus(text, className) {
|
||||
const el = document.getElementById("status");
|
||||
el.textContent = text;
|
||||
el.className = "status " + (className || "");
|
||||
}
|
||||
|
||||
function log(msg, level = "info") {
|
||||
if (level === "error") {
|
||||
console.error("[moq]", msg);
|
||||
} else if (level === "warn") {
|
||||
console.warn("[moq]", msg);
|
||||
} else {
|
||||
console.log("[moq]", msg);
|
||||
}
|
||||
const el = document.getElementById("log");
|
||||
const line = document.createElement("div");
|
||||
line.textContent = `[${new Date().toLocaleTimeString()}] ${msg}`;
|
||||
if (level === "error") line.style.color = "#e74c3c";
|
||||
else if (level === "warn") line.style.color = "#f39c12";
|
||||
el.appendChild(line);
|
||||
el.scrollTop = el.scrollHeight;
|
||||
}
|
||||
|
||||
function hexdump(data, maxBytes = 40) {
|
||||
const bytes = data instanceof Uint8Array ? data : new Uint8Array(data);
|
||||
const hex = Array.from(bytes.slice(0, maxBytes)).map(b => b.toString(16).padStart(2, '0')).join(' ');
|
||||
return bytes.length > maxBytes ? `${hex}... (${bytes.length} bytes total)` : `${hex} (${bytes.length} bytes)`;
|
||||
}
|
||||
|
||||
// Turn-based transcript: each in-progress LLM/user turn is one row whose
|
||||
// text accumulates as per-token RTVI messages arrive. Mirrors the
|
||||
// ConversationProvider model in @pipecat-ai/voice-ui-kit.
|
||||
//
|
||||
// A user "turn" runs from the first user-transcription until the bot
|
||||
// starts speaking — Deepgram can emit several final transcripts inside
|
||||
// one spoken utterance, and we want all of them in the same bubble.
|
||||
const turnState = {
|
||||
assistantRow: null, // DOM <div> for the in-progress assistant turn
|
||||
assistantText: "", // accumulated text for the in-progress turn
|
||||
userRow: null, // DOM <div> for the in-progress user turn
|
||||
userCommitted: "", // accumulated final user text within the turn
|
||||
userInterim: "", // latest non-final user text (tentative tail)
|
||||
};
|
||||
|
||||
function newTranscriptRow(role) {
|
||||
const el = document.getElementById("transcript");
|
||||
if (!el) return null;
|
||||
const row = document.createElement("div");
|
||||
const isUser = role === "user";
|
||||
row.className = `transcript-row transcript-${isUser ? "user" : "assistant"}`;
|
||||
const label = document.createElement("span");
|
||||
label.className = "transcript-role";
|
||||
label.textContent = isUser ? "user" : "assistant";
|
||||
const body = document.createElement("span");
|
||||
body.className = "transcript-text";
|
||||
row.appendChild(label);
|
||||
row.appendChild(body);
|
||||
el.appendChild(row);
|
||||
el.scrollTop = el.scrollHeight;
|
||||
return row;
|
||||
}
|
||||
|
||||
function setRowText(row, text) {
|
||||
if (!row) return;
|
||||
const body = row.querySelector(".transcript-text");
|
||||
if (body) body.textContent = text;
|
||||
const el = document.getElementById("transcript");
|
||||
if (el) el.scrollTop = el.scrollHeight;
|
||||
}
|
||||
|
||||
function clearTranscript() {
|
||||
const el = document.getElementById("transcript");
|
||||
if (el) el.innerHTML = "";
|
||||
turnState.assistantRow = null;
|
||||
turnState.assistantText = "";
|
||||
turnState.userRow = null;
|
||||
turnState.userCommitted = "";
|
||||
turnState.userInterim = "";
|
||||
}
|
||||
|
||||
// Concatenate streaming tokens with a sensible separator. Some LLM tokens
|
||||
// already include leading whitespace (e.g. " world"); avoid double spaces.
|
||||
function appendStreamingText(prev, chunk) {
|
||||
if (!prev) return chunk;
|
||||
if (!chunk) return prev;
|
||||
const needsSpace = !prev.endsWith(" ") && !/^[\s.,!?;:'")\]]/.test(chunk);
|
||||
return prev + (needsSpace ? " " : "") + chunk;
|
||||
}
|
||||
|
||||
// RTVI message dispatcher. Matches the ConversationProvider semantics from
|
||||
// @pipecat-ai/voice-ui-kit: per-LLM-turn aggregation for the bot, upsert
|
||||
// for user transcripts.
|
||||
function handleRtviMessage(msg) {
|
||||
if (!msg || typeof msg !== "object") return;
|
||||
const type = msg.type;
|
||||
const data = msg.data || {};
|
||||
|
||||
switch (type) {
|
||||
case "bot-llm-started":
|
||||
// Don't create the row yet — wait for the first bot-llm-text so we
|
||||
// don't show an empty "assistant" label if no text follows.
|
||||
turnState.assistantRow = null;
|
||||
turnState.assistantText = "";
|
||||
// Bot is responding: the user's turn just ended. The next
|
||||
// user-transcription should start a fresh row.
|
||||
turnState.userRow = null;
|
||||
turnState.userCommitted = "";
|
||||
turnState.userInterim = "";
|
||||
break;
|
||||
|
||||
case "bot-llm-text":
|
||||
if (!data.text) break;
|
||||
if (!turnState.assistantRow) {
|
||||
// Lazy: covers both the post-started case and pipelines that emit
|
||||
// bot-llm-text without a preceding bot-llm-started.
|
||||
turnState.assistantRow = newTranscriptRow("assistant");
|
||||
}
|
||||
turnState.assistantText = appendStreamingText(turnState.assistantText, data.text);
|
||||
setRowText(turnState.assistantRow, turnState.assistantText);
|
||||
break;
|
||||
|
||||
case "bot-llm-stopped":
|
||||
turnState.assistantRow = null;
|
||||
turnState.assistantText = "";
|
||||
break;
|
||||
|
||||
case "user-transcription": {
|
||||
const text = data.text || "";
|
||||
const final = data.final !== false;
|
||||
if (!turnState.userRow) {
|
||||
turnState.userRow = newTranscriptRow("user");
|
||||
turnState.userCommitted = "";
|
||||
turnState.userInterim = "";
|
||||
}
|
||||
if (final) {
|
||||
turnState.userCommitted = turnState.userCommitted
|
||||
? appendStreamingText(turnState.userCommitted, text)
|
||||
: text;
|
||||
turnState.userInterim = "";
|
||||
setRowText(turnState.userRow, turnState.userCommitted);
|
||||
} else {
|
||||
turnState.userInterim = text;
|
||||
const combined = turnState.userCommitted
|
||||
? appendStreamingText(turnState.userCommitted, text)
|
||||
: text;
|
||||
setRowText(turnState.userRow, combined);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
// Ignore other RTVI messages (metrics, speech start/stop, bot-output,
|
||||
// bot-tts-*, function calls, etc.). Uncomment to discover what's
|
||||
// flowing on the transcript track.
|
||||
// log(`RTVI: ${type}`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Audio playback — queue received 16-bit PCM into Web Audio
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function initAudioPlayback() {
|
||||
if (audioContext) return;
|
||||
audioContext = new AudioContext({ sampleRate: 24000 });
|
||||
playbackTime = audioContext.currentTime;
|
||||
}
|
||||
|
||||
function playPcmChunk(pcmBytes) {
|
||||
if (!audioContext) return;
|
||||
|
||||
// pcmBytes is 16-bit signed LE PCM — copy to aligned buffer to avoid offset issues
|
||||
const aligned = new ArrayBuffer(pcmBytes.byteLength);
|
||||
new Uint8Array(aligned).set(pcmBytes);
|
||||
const samples = new Int16Array(aligned);
|
||||
const floats = new Float32Array(samples.length);
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
floats[i] = samples[i] / 32768;
|
||||
}
|
||||
|
||||
const buffer = audioContext.createBuffer(1, floats.length, 24000);
|
||||
buffer.getChannelData(0).set(floats);
|
||||
|
||||
const source = audioContext.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(audioContext.destination);
|
||||
|
||||
const now = audioContext.currentTime;
|
||||
if (playbackTime < now) playbackTime = now;
|
||||
source.start(playbackTime);
|
||||
playbackTime += buffer.duration;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Microphone capture — 16kHz 16-bit PCM via AudioWorklet
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const WORKLET_CODE = `
|
||||
class PcmCapture extends AudioWorkletProcessor {
|
||||
process(inputs) {
|
||||
const input = inputs[0];
|
||||
if (input.length > 0) {
|
||||
const floats = input[0];
|
||||
const pcm = new Int16Array(floats.length);
|
||||
for (let i = 0; i < floats.length; i++) {
|
||||
pcm[i] = Math.max(-32768, Math.min(32767, Math.round(floats[i] * 32768)));
|
||||
}
|
||||
this.port.postMessage(pcm.buffer, [pcm.buffer]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('pcm-capture', PcmCapture);
|
||||
`;
|
||||
|
||||
async function startMic() {
|
||||
micStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: { sampleRate: 16000, channelCount: 1, echoCancellation: true },
|
||||
});
|
||||
|
||||
const ctx = new AudioContext({ sampleRate: 16000 });
|
||||
const blob = new Blob([WORKLET_CODE], { type: "application/javascript" });
|
||||
const url = URL.createObjectURL(blob);
|
||||
await ctx.audioWorklet.addModule(url);
|
||||
URL.revokeObjectURL(url);
|
||||
|
||||
const source = ctx.createMediaStreamSource(micStream);
|
||||
micWorklet = new AudioWorkletNode(ctx, "pcm-capture");
|
||||
micWorklet.port.onmessage = (e) => {
|
||||
if (connected && transport && pubSubscribeId !== null) {
|
||||
sendAudioUniStream(new Uint8Array(e.data));
|
||||
}
|
||||
};
|
||||
source.connect(micWorklet);
|
||||
micWorklet.connect(ctx.destination);
|
||||
|
||||
log("Microphone started (16kHz PCM)");
|
||||
}
|
||||
|
||||
function stopMic() {
|
||||
if (micStream) {
|
||||
micStream.getTracks().forEach((t) => t.stop());
|
||||
micStream = null;
|
||||
}
|
||||
if (micWorklet) {
|
||||
micWorklet.disconnect();
|
||||
micWorklet = null;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Uni stream send (publish audio)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let audioSendCount = 0;
|
||||
let audioSendErrors = 0;
|
||||
|
||||
async function sendAudioUniStream(pcmBytes) {
|
||||
if (!transport || pubSubscribeId === null) return;
|
||||
|
||||
try {
|
||||
const groupSeq = pubGroupSeq++;
|
||||
const data = encodeGroupAndFrame(pubSubscribeId, groupSeq, pcmBytes);
|
||||
|
||||
const uni = await transport.createUnidirectionalStream();
|
||||
const writer = uni.getWriter();
|
||||
await writer.write(data);
|
||||
await writer.close();
|
||||
|
||||
audioSendCount++;
|
||||
if (audioSendCount % 100 === 0) {
|
||||
log(`Audio TX: ${audioSendCount} chunks sent (${pcmBytes.byteLength} bytes/chunk, sub=${pubSubscribeId}, seq=${groupSeq})`);
|
||||
}
|
||||
} catch (e) {
|
||||
audioSendErrors++;
|
||||
if (audioSendErrors <= 5) {
|
||||
log(`Audio send error #${audioSendErrors}: ${e.message} (sub=${pubSubscribeId}, seq=${pubGroupSeq})`, "warn");
|
||||
} else if (audioSendErrors === 6) {
|
||||
log("Suppressing further audio send errors...", "warn");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Uni stream receive (receive bot audio)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let audioRecvCount = 0;
|
||||
|
||||
async function receiveUniStreams() {
|
||||
log("Uni stream listener started (waiting for bot audio)");
|
||||
const reader = transport.incomingUnidirectionalStreams.getReader();
|
||||
try {
|
||||
while (true) {
|
||||
const { value: stream, done } = await reader.read();
|
||||
if (done) {
|
||||
log("Uni stream reader finished (relay closed incoming streams)");
|
||||
break;
|
||||
}
|
||||
handleIncomingUniStream(stream);
|
||||
}
|
||||
} catch (e) {
|
||||
if (connected) log(`Uni stream reader stopped: ${e.message}`, "warn");
|
||||
}
|
||||
}
|
||||
|
||||
async function handleIncomingUniStream(stream) {
|
||||
try {
|
||||
// Read all data from the stream
|
||||
const reader = stream.getReader();
|
||||
let buf = new Uint8Array(0);
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
buf = concat(buf, new Uint8Array(value));
|
||||
}
|
||||
|
||||
if (buf.length === 0) return;
|
||||
|
||||
// Parse GROUP + FRAMEs
|
||||
const { subscribeId, groupSeq, frames } = parseGroupStream(buf);
|
||||
|
||||
if (subscribeId === subTranscriptSubscribeId) {
|
||||
for (const frame of frames) {
|
||||
if (!frame.byteLength) continue;
|
||||
try {
|
||||
const text = new TextDecoder().decode(frame);
|
||||
handleRtviMessage(JSON.parse(text));
|
||||
} catch (err) {
|
||||
log(`Transcript frame decode error: ${err.message}`, "warn");
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
audioRecvCount++;
|
||||
|
||||
if (audioRecvCount <= 3 || audioRecvCount % 100 === 0) {
|
||||
const totalBytes = frames.reduce((sum, f) => sum + f.byteLength, 0);
|
||||
log(`Audio RX #${audioRecvCount}: sub=${subscribeId} seq=${groupSeq} frames=${frames.length} bytes=${totalBytes}`);
|
||||
}
|
||||
|
||||
for (const frame of frames) {
|
||||
if (frame.byteLength > 0) {
|
||||
playPcmChunk(frame);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
log(`Uni stream parse error: ${e.message} (buffer may be malformed)`, "warn");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Incoming bidi stream handler (relay sends SUBSCRIBE / ANNOUNCE)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let incomingBidiCount = 0;
|
||||
|
||||
async function handleIncomingBidiStreams() {
|
||||
log("Bidi stream listener started (waiting for relay SUBSCRIBE/ANNOUNCE)");
|
||||
const reader = transport.incomingBidirectionalStreams.getReader();
|
||||
try {
|
||||
while (true) {
|
||||
const { value: stream, done } = await reader.read();
|
||||
if (done) {
|
||||
log("Incoming bidi stream reader finished (relay closed)");
|
||||
break;
|
||||
}
|
||||
incomingBidiCount++;
|
||||
log(`Incoming bidi stream #${incomingBidiCount}`);
|
||||
handleIncomingBidiStream(stream);
|
||||
}
|
||||
} catch (e) {
|
||||
if (connected) log(`Bidi stream reader stopped: ${e.message}`, "warn");
|
||||
}
|
||||
}
|
||||
|
||||
async function handleIncomingBidiStream(stream) {
|
||||
try {
|
||||
const streamReader = stream.readable.getReader();
|
||||
let buf = new Uint8Array(0);
|
||||
|
||||
// Read all available data (relay sends stream type + message body)
|
||||
while (true) {
|
||||
const { value, done } = await streamReader.read();
|
||||
if (value) buf = concat(buf, new Uint8Array(value));
|
||||
// Break once we have data or stream ends
|
||||
if (done || buf.length > 0) break;
|
||||
}
|
||||
|
||||
if (buf.length === 0) {
|
||||
log("Incoming bidi stream was empty (0 bytes)", "warn");
|
||||
return;
|
||||
}
|
||||
|
||||
log(`Incoming bidi data: [${hexdump(buf, 30)}]`);
|
||||
|
||||
// Decode stream type
|
||||
const [streamType, offset] = decodeVarint(buf, 0);
|
||||
const streamTypeName = streamType === StreamType.SUBSCRIBE ? "SUBSCRIBE"
|
||||
: streamType === StreamType.ANNOUNCE ? "ANNOUNCE"
|
||||
: streamType === StreamType.SESSION ? "SESSION"
|
||||
: `UNKNOWN(${streamType})`;
|
||||
log(`Incoming bidi stream type: ${streamTypeName} (${streamType})`);
|
||||
|
||||
if (streamType === StreamType.SUBSCRIBE) {
|
||||
// We may need more data for the full subscribe message
|
||||
while (buf.length < offset + 3) {
|
||||
const { value, done } = await streamReader.read();
|
||||
if (value) buf = concat(buf, new Uint8Array(value));
|
||||
if (done) break;
|
||||
}
|
||||
|
||||
// Relay is subscribing to our track
|
||||
const sub = decodeSubscribe(buf, offset);
|
||||
log(`SUBSCRIBE from relay: broadcast="${sub.broadcastPath}" track="${sub.trackName}" sub_id=${sub.subscribeId} priority=${sub.priority}`);
|
||||
|
||||
// Accept SUBSCRIBE only for our publish track inside our own
|
||||
// broadcast. With per-participant broadcast paths, the relay only
|
||||
// routes SUBSCRIBEs targeting <namespace>/<client_id> to us, so a
|
||||
// mismatch here means something is misconfigured.
|
||||
const ourBroadcast = `${config.namespace}/${config.client_id}`;
|
||||
if (sub.broadcastPath !== ourBroadcast || sub.trackName !== config.publish_track) {
|
||||
log(`Rejecting SUBSCRIBE for "${sub.broadcastPath}/${sub.trackName}" (we only publish "${ourBroadcast}/${config.publish_track}")`, "warn");
|
||||
stream.writable.abort();
|
||||
return;
|
||||
}
|
||||
|
||||
// Store subscribe_id for publishing
|
||||
pubSubscribeId = sub.subscribeId;
|
||||
|
||||
// Send SUBSCRIBE_OK on the writable side
|
||||
const writer = stream.writable.getWriter();
|
||||
await writer.write(encodeSubscribeOk());
|
||||
writer.releaseLock();
|
||||
|
||||
log(`Sent SUBSCRIBE_OK for sub_id=${sub.subscribeId} — ready to publish ${config.publish_track}`);
|
||||
|
||||
} else if (streamType === StreamType.ANNOUNCE) {
|
||||
// Relay sending ANNOUNCE_PLEASE — read the full message
|
||||
// Minimum is 2 bytes after stream type: varint(body_len) + varint(string_len)
|
||||
while (buf.length < offset + 2) {
|
||||
const { value, done } = await streamReader.read();
|
||||
if (value) buf = concat(buf, new Uint8Array(value));
|
||||
if (done) break;
|
||||
}
|
||||
|
||||
// Parse ANNOUNCE_PLEASE: varint(body_len) + string(path_prefix)
|
||||
let pos = offset;
|
||||
let bodyLen;
|
||||
[bodyLen, pos] = decodeVarint(buf, pos);
|
||||
let pathPrefix = "";
|
||||
if (bodyLen > 0) {
|
||||
[pathPrefix, pos] = decodeString(buf, pos);
|
||||
}
|
||||
log(`ANNOUNCE_PLEASE from relay: prefix="${pathPrefix}" bodyLen=${bodyLen}`);
|
||||
|
||||
// Respond with ANNOUNCE_INIT for our per-participant broadcast path.
|
||||
// Each participant uses a distinct suffix (e.g. "pipecat/client0")
|
||||
// so the relay can route SUBSCRIBEs to the right side.
|
||||
const broadcast = `${config.namespace}/${config.client_id}`;
|
||||
let suffix = broadcast;
|
||||
if (pathPrefix) {
|
||||
if (broadcast.startsWith(pathPrefix)) {
|
||||
suffix = broadcast.slice(pathPrefix.length).replace(/^\//, "");
|
||||
} else {
|
||||
suffix = null;
|
||||
}
|
||||
}
|
||||
|
||||
const writer = stream.writable.getWriter();
|
||||
if (suffix !== null) {
|
||||
let body = encodeVarint(1); // 1 suffix
|
||||
body = concat(body, encodeString(suffix));
|
||||
const initMsg = concat(encodeVarint(body.length), body);
|
||||
await writer.write(initMsg);
|
||||
log(`Sent ANNOUNCE_INIT: 1 suffix="${suffix}" [${hexdump(initMsg)}]`);
|
||||
} else {
|
||||
const initMsg = concat(encodeVarint(1), encodeVarint(0));
|
||||
await writer.write(initMsg);
|
||||
log(`Sent ANNOUNCE_INIT: 0 suffixes (our broadcast doesn't match prefix "${pathPrefix}")`);
|
||||
}
|
||||
writer.releaseLock();
|
||||
|
||||
} else {
|
||||
log(`Unknown bidi stream type ${streamType}, raw: [${hexdump(buf)}]`, "warn");
|
||||
}
|
||||
} catch (e) {
|
||||
log(`Incoming bidi stream error: ${e.message}`, "error");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Connect flow
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function doConnect() {
|
||||
const t0 = performance.now();
|
||||
clearTranscript();
|
||||
try {
|
||||
// 1. Fetch config from the FastAPI server
|
||||
log("Fetching /api/config...");
|
||||
const resp = await fetch("/api/config");
|
||||
config = await resp.json();
|
||||
log(`Config: relay=${config.relay_host}:${config.relay_port}, path="${config.path}", ns="${config.namespace}", client_id="${config.client_id}", bot_id="${config.bot_id}", publish="${config.publish_track}", subscribe="${config.subscribe_track}", insecure=${config.insecure}, cert_hash=${config.cert_hash ? config.cert_hash.slice(0, 12) + "..." : "none"}`);
|
||||
|
||||
// 1b. Ask the server to start the bot. It blocks until the bot has
|
||||
// finished its MOQ handshake with the relay, so our SUBSCRIBE is
|
||||
// guaranteed to land at a publisher the relay already knows.
|
||||
log("Starting bot via /start...");
|
||||
const startResp = await fetch("/start", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ namespace: config.namespace }),
|
||||
});
|
||||
if (!startResp.ok) {
|
||||
throw new Error(`/start returned HTTP ${startResp.status}`);
|
||||
}
|
||||
const startInfo = await startResp.json();
|
||||
log(`Bot started: session=${startInfo.sessionId}, relay=${startInfo.relay}`);
|
||||
|
||||
// 2. Build relay URL
|
||||
const scheme = "https";
|
||||
const host = config.relay_host;
|
||||
const relayUrl = `${scheme}://${host}:${config.relay_port}${config.path || "/moq"}`;
|
||||
log(`Connecting to ${relayUrl}`);
|
||||
setStatus("Connecting...", "connecting");
|
||||
|
||||
// 3. Open WebTransport (connects via HTTP/3 over QUIC to the relay)
|
||||
const options = {};
|
||||
if (config.cert_hash) {
|
||||
const hashBytes = Uint8Array.from(atob(config.cert_hash), c => c.charCodeAt(0));
|
||||
options.serverCertificateHashes = [
|
||||
{ algorithm: "sha-256", value: hashBytes.buffer },
|
||||
];
|
||||
log(`Using certificate pinning: sha256=${config.cert_hash.slice(0, 16)}...`);
|
||||
} else {
|
||||
log("No certificate pinning (no cert_hash in config)", "warn");
|
||||
}
|
||||
log(`WebTransport options: ${JSON.stringify({...options, serverCertificateHashes: options.serverCertificateHashes ? "[present]" : undefined})}`);
|
||||
|
||||
transport = new WebTransport(relayUrl, options);
|
||||
|
||||
// Monitor connection closure — extract QUIC error details
|
||||
transport.closed.then(info => {
|
||||
const elapsed = (performance.now() - t0).toFixed(0);
|
||||
log(`Transport closed cleanly after ${elapsed}ms: closeCode=${info?.closeCode} reason="${info?.reason || ""}"`);
|
||||
}).catch(err => {
|
||||
const elapsed = (performance.now() - t0).toFixed(0);
|
||||
// WebTransportCloseInfo may have closeCode and reason
|
||||
const code = err?.closeCode ?? err?.code ?? "?";
|
||||
const reason = err?.reason ?? err?.message ?? String(err);
|
||||
log(`Transport closed with error after ${elapsed}ms: code=${code} reason="${reason}"`, "error");
|
||||
log(` Error type: ${err?.constructor?.name}, keys: ${Object.keys(err || {}).join(",")}`, "error");
|
||||
});
|
||||
|
||||
// Wait for the QUIC + HTTP/3 handshake
|
||||
const readyStart = performance.now();
|
||||
try {
|
||||
await transport.ready;
|
||||
} catch (readyErr) {
|
||||
const readyMs = (performance.now() - readyStart).toFixed(0);
|
||||
// Get the real error from transport.closed
|
||||
let closedErr = null;
|
||||
try { await transport.closed; } catch (e) { closedErr = e; }
|
||||
const detail = closedErr?.message || closedErr?.reason || String(closedErr || "");
|
||||
const code = closedErr?.closeCode ?? closedErr?.code ?? "?";
|
||||
log(`WebTransport handshake FAILED after ${readyMs}ms`, "error");
|
||||
log(` ready error: ${readyErr.message}`, "error");
|
||||
log(` closed error: code=${code} detail="${detail}"`, "error");
|
||||
log(` URL: ${relayUrl}`, "error");
|
||||
log(` Cert pinning: ${config.cert_hash ? "yes" : "no"}`, "error");
|
||||
log(`Troubleshooting:`, "error");
|
||||
log(` - Is the relay running at ${host}:${config.relay_port}?`, "error");
|
||||
log(` - Does relay config have [web.http] listen = "[::]:${config.relay_port}"?`, "error");
|
||||
log(` - If using tls.generate in relay config, cert hash will change every restart`, "error");
|
||||
log(` - Try 127.0.0.1 instead of localhost (QUIC/UDP has no IPv6->IPv4 fallback)`, "error");
|
||||
throw readyErr;
|
||||
}
|
||||
const readyMs = (performance.now() - readyStart).toFixed(0);
|
||||
log(`WebTransport connected (handshake took ${readyMs}ms)`);
|
||||
|
||||
// 4. Open bidi stream for SETUP handshake
|
||||
log("Opening bidi stream for CLIENT_SETUP...");
|
||||
let setupBidi;
|
||||
try {
|
||||
setupBidi = await transport.createBidirectionalStream();
|
||||
log("Bidi stream created for SETUP");
|
||||
} catch (e) {
|
||||
// This is the "Connection lost" case — get more details
|
||||
let closedErr = null;
|
||||
try { await transport.closed; } catch (ce) { closedErr = ce; }
|
||||
const code = closedErr?.closeCode ?? closedErr?.code ?? "?";
|
||||
const reason = closedErr?.reason ?? closedErr?.message ?? "";
|
||||
log(`Failed to create bidi stream: ${e.message}`, "error");
|
||||
log(` Underlying close: code=${code} reason="${reason}"`, "error");
|
||||
log(` This usually means the relay rejected the WebTransport session.`, "error");
|
||||
log(` Check the relay terminal for error logs (auth failure, path mismatch, etc.)`, "error");
|
||||
log(` Relay URL path was: "${config.path || "/moq"}" — does the relay expect this path?`, "error");
|
||||
throw e;
|
||||
}
|
||||
const setupWriter = setupBidi.writable.getWriter();
|
||||
const setupStreamReader = setupBidi.readable.getReader();
|
||||
|
||||
// Write stream type varint(0x20) + CLIENT_SETUP (u16 body size for WebTransport)
|
||||
const setupMsg = encodeClientSetup(Role.PUBSUB, [MOQL_VERSION], config.path || "/moq");
|
||||
const fullSetup = concat(encodeVarint(CLIENT_SETUP_TYPE), setupMsg);
|
||||
log(`Sending CLIENT_SETUP: role=PUBSUB version=0x${MOQL_VERSION.toString(16)} path="${config.path || "/moq"}" [${hexdump(fullSetup)}]`);
|
||||
try {
|
||||
await setupWriter.write(fullSetup);
|
||||
log("CLIENT_SETUP sent, waiting for SERVER_SETUP...");
|
||||
} catch (e) {
|
||||
log(`Failed to write CLIENT_SETUP: ${e.message}`, "error");
|
||||
throw e;
|
||||
}
|
||||
|
||||
// 5. Read SERVER_SETUP response
|
||||
const setupTimeout = setTimeout(() => {
|
||||
log("SERVER_SETUP not received after 5s — relay may not understand our protocol", "warn");
|
||||
}, 5000);
|
||||
const { value: setupResp, done: setupDone } = await setupStreamReader.read();
|
||||
clearTimeout(setupTimeout);
|
||||
|
||||
if (setupDone && !setupResp) {
|
||||
log("Setup stream closed by relay without sending SERVER_SETUP", "error");
|
||||
log(" The relay may not support moq-lite-02 (0xff0dad02)", "error");
|
||||
throw new Error("No SERVER_SETUP received");
|
||||
}
|
||||
|
||||
if (setupResp) {
|
||||
const respData = new Uint8Array(setupResp);
|
||||
log(`Received SERVER_SETUP raw: [${hexdump(respData)}]`);
|
||||
// Skip stream type varint (0x21) if present
|
||||
let offset = 0;
|
||||
if (respData[0] === SERVER_SETUP_TYPE) {
|
||||
offset = 1;
|
||||
} else {
|
||||
const [st, newOff] = decodeVarint(respData, 0);
|
||||
if (st === SERVER_SETUP_TYPE) offset = newOff;
|
||||
}
|
||||
const { version } = parseServerSetup(respData.subarray(offset));
|
||||
const versionName = version === 0xff0dad02 ? "moq-lite-02" : `unknown (0x${version.toString(16)})`;
|
||||
log(`SERVER_SETUP: relay version=${versionName} (0x${version.toString(16)}), client version=moq-lite-02 (0x${MOQL_VERSION.toString(16)})`);
|
||||
if (version !== MOQL_VERSION) {
|
||||
log(`PROTOCOL MISMATCH! Browser speaks moq-lite-02 but relay speaks ${versionName}. Things will likely break.`, "error");
|
||||
}
|
||||
}
|
||||
// Keep setup stream open — don't close it
|
||||
setupWriter.releaseLock();
|
||||
|
||||
connected = true;
|
||||
const totalMs = (performance.now() - t0).toFixed(0);
|
||||
setStatus("Connected", "connected");
|
||||
log(`Connected to relay (total setup took ${totalMs}ms)`);
|
||||
|
||||
// 6. Start listening for incoming streams FIRST
|
||||
// (relay may send ANNOUNCE_PLEASE and SUBSCRIBE before we subscribe)
|
||||
log("Listening for incoming bidi + uni streams...");
|
||||
handleIncomingBidiStreams();
|
||||
receiveUniStreams();
|
||||
|
||||
// 7. Init audio playback (needs user gesture — we're inside a click handler)
|
||||
initAudioPlayback();
|
||||
log("Audio playback initialized (24kHz)");
|
||||
|
||||
// 8. Start mic (so we're ready to publish when relay subscribes)
|
||||
await startMic();
|
||||
|
||||
// 9. Subscribe to bot's audio + transcript tracks (non-fatal if bot
|
||||
// isn't connected yet)
|
||||
await subscribeToAudio();
|
||||
subscribeToTranscript();
|
||||
|
||||
} catch (e) {
|
||||
const elapsed = (performance.now() - t0).toFixed(0);
|
||||
log(`Connection failed after ${elapsed}ms: ${e.message}`, "error");
|
||||
if (e.stack) log(` Stack: ${e.stack.split("\n").slice(1, 3).join(" | ")}`, "error");
|
||||
if (e.stack) log(` Stack: ${e.stack}`);
|
||||
setStatus("Error: " + e.message, "error");
|
||||
}
|
||||
}
|
||||
|
||||
async function subscribeToAudio(retryCount = 0) {
|
||||
const botBroadcast = `${config.namespace}/${config.bot_id}`;
|
||||
const botTrack = config.subscribe_track;
|
||||
const fullPath = `${botBroadcast}/${botTrack}`;
|
||||
const maxRetries = 10;
|
||||
const retryDelay = 2000; // 2 seconds between retries
|
||||
|
||||
subAudioSubscribeId = nextSubscribeId++;
|
||||
|
||||
try {
|
||||
// Open a new bidi stream for SUBSCRIBE
|
||||
log(`Subscribing to ${fullPath} (sub_id=${subAudioSubscribeId}, attempt ${retryCount + 1}/${maxRetries + 1})`);
|
||||
const subBidi = await transport.createBidirectionalStream();
|
||||
const subWriter = subBidi.writable.getWriter();
|
||||
const subReader = subBidi.readable.getReader();
|
||||
|
||||
const subMsg = encodeSubscribe(subAudioSubscribeId, botBroadcast, botTrack);
|
||||
log(`Sending SUBSCRIBE: [${hexdump(subMsg)}]`);
|
||||
await subWriter.write(subMsg);
|
||||
|
||||
// Read SUBSCRIBE_OK (or handle RESET_STREAM gracefully)
|
||||
log("Waiting for SUBSCRIBE_OK from relay...");
|
||||
const { value: okResp, done } = await subReader.read();
|
||||
if (okResp) {
|
||||
const okData = new Uint8Array(okResp);
|
||||
log(`Received SUBSCRIBE_OK for ${fullPath}: [${hexdump(okData)}]`);
|
||||
return; // Success
|
||||
} else if (done) {
|
||||
log(`SUBSCRIBE stream closed by relay without response (track "${fullPath}" may not exist — bot not connected yet?)`, "warn");
|
||||
}
|
||||
|
||||
subWriter.releaseLock();
|
||||
} catch (e) {
|
||||
// RESET_STREAM means the track doesn't exist yet — not fatal
|
||||
log(`Subscribe to ${fullPath} failed: ${e.message}`, "warn");
|
||||
log(` (This is normal if the bot hasn't connected to the relay yet)`, "warn");
|
||||
}
|
||||
|
||||
// Retry if bot isn't connected yet
|
||||
if (connected && retryCount < maxRetries) {
|
||||
log(`Will retry subscribe in ${retryDelay / 1000}s (attempt ${retryCount + 1}/${maxRetries})...`);
|
||||
setTimeout(() => subscribeToAudio(retryCount + 1), retryDelay);
|
||||
} else if (retryCount >= maxRetries) {
|
||||
log(`Gave up subscribing to ${fullPath} after ${maxRetries} attempts`, "error");
|
||||
}
|
||||
}
|
||||
|
||||
async function subscribeToTranscript() {
|
||||
const trackName = config.transcript_track || "transcript";
|
||||
const botBroadcast = `${config.namespace}/${config.bot_id}`;
|
||||
const fullPath = `${botBroadcast}/${trackName}`;
|
||||
subTranscriptSubscribeId = nextSubscribeId++;
|
||||
try {
|
||||
log(`Subscribing to ${fullPath} (sub_id=${subTranscriptSubscribeId})`);
|
||||
const subBidi = await transport.createBidirectionalStream();
|
||||
const subWriter = subBidi.writable.getWriter();
|
||||
const subReader = subBidi.readable.getReader();
|
||||
const subMsg = encodeSubscribe(subTranscriptSubscribeId, botBroadcast, trackName);
|
||||
await subWriter.write(subMsg);
|
||||
subWriter.releaseLock();
|
||||
// Keep the reader open so the relay can stream SUBSCRIBE_OK / errors.
|
||||
subReader.read().then(({ value, done }) => {
|
||||
if (value && value.byteLength) {
|
||||
log(`Transcript SUBSCRIBE_OK: [${hexdump(new Uint8Array(value))}]`);
|
||||
} else if (done) {
|
||||
log(`Transcript subscribe stream closed by relay (track may not exist)`, "warn");
|
||||
}
|
||||
}).catch(() => {});
|
||||
} catch (e) {
|
||||
log(`Subscribe to ${fullPath} failed: ${e.message}`, "warn");
|
||||
}
|
||||
}
|
||||
|
||||
async function doDisconnect() {
|
||||
log(`Disconnecting... (sent=${audioSendCount} chunks, received=${audioRecvCount} chunks, errors=${audioSendErrors})`);
|
||||
connected = false;
|
||||
stopMic();
|
||||
|
||||
if (transport) {
|
||||
try {
|
||||
transport.close();
|
||||
log("WebTransport closed");
|
||||
} catch (e) {
|
||||
log(`Error closing transport: ${e.message}`, "warn");
|
||||
}
|
||||
transport = null;
|
||||
}
|
||||
|
||||
pubSubscribeId = null;
|
||||
pubGroupSeq = 0;
|
||||
subAudioSubscribeId = null;
|
||||
subTranscriptSubscribeId = null;
|
||||
nextSubscribeId = 1;
|
||||
audioSendCount = 0;
|
||||
audioSendErrors = 0;
|
||||
audioRecvCount = 0;
|
||||
incomingBidiCount = 0;
|
||||
|
||||
if (audioContext) {
|
||||
audioContext.close();
|
||||
audioContext = null;
|
||||
}
|
||||
|
||||
setStatus("Disconnected", "disconnected");
|
||||
log("Disconnected");
|
||||
|
||||
document.getElementById("connectBtn").disabled = false;
|
||||
document.getElementById("disconnectBtn").disabled = true;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Button handlers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
document.getElementById("connectBtn").addEventListener("click", async () => {
|
||||
document.getElementById("connectBtn").disabled = true;
|
||||
document.getElementById("disconnectBtn").disabled = false;
|
||||
await doConnect();
|
||||
});
|
||||
|
||||
document.getElementById("disconnectBtn").addEventListener("click", async () => {
|
||||
await doDisconnect();
|
||||
});
|
||||
36
moq_prebuilt/client/index.html
Normal file
36
moq_prebuilt/client/index.html
Normal file
@@ -0,0 +1,36 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Pipecat MOQ Client</title>
|
||||
<link rel="stylesheet" href="style.css" />
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>Pipecat MOQ Client</h1>
|
||||
|
||||
<div class="card">
|
||||
<h3>Connection</h3>
|
||||
<div id="status" class="status disconnected">Disconnected</div>
|
||||
<div class="controls">
|
||||
<button id="connectBtn">Connect</button>
|
||||
<button id="disconnectBtn" disabled>Disconnect</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card transcript-card">
|
||||
<h3>Transcript</h3>
|
||||
<div id="transcript" class="transcript"></div>
|
||||
</div>
|
||||
|
||||
<div class="card log-card">
|
||||
<h3>Log</h3>
|
||||
<div id="log" class="log"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="moq-client.js"></script>
|
||||
<script src="app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
324
moq_prebuilt/client/moq-client.js
Normal file
324
moq_prebuilt/client/moq-client.js
Normal file
@@ -0,0 +1,324 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026, Daily
|
||||
*
|
||||
* SPDX-License-Identifier: BSD 2-Clause License
|
||||
*/
|
||||
|
||||
/**
|
||||
* MOQ (Media over QUIC) protocol client — moq-lite-02.
|
||||
*
|
||||
* Implements varint codec, message encode/decode for moq-lite-02 protocol
|
||||
* so the browser can talk to a moq-lite relay over WebTransport.
|
||||
*
|
||||
* Key differences from draft-07:
|
||||
* - Stream-per-request model (no shared control stream)
|
||||
* - Setup uses u8(0x20/0x21) framing; via WebTransport uses u16 body size
|
||||
* - SUBSCRIBE/ANNOUNCE each get their own bidi stream
|
||||
* - Media data flows on uni streams as GROUP + FRAME
|
||||
* - No QUIC datagrams
|
||||
*/
|
||||
(function () {
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Constants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const MOQL_VERSION = 0xff0dad02; // moq-lite-02
|
||||
|
||||
const Role = Object.freeze({
|
||||
PUBLISHER: 0x01,
|
||||
SUBSCRIBER: 0x02,
|
||||
PUBSUB: 0x03,
|
||||
});
|
||||
|
||||
// Stream type varints (first thing written on a new bidi stream)
|
||||
const StreamType = Object.freeze({
|
||||
SESSION: 0,
|
||||
ANNOUNCE: 1,
|
||||
SUBSCRIBE: 2,
|
||||
});
|
||||
|
||||
// Setup message type bytes
|
||||
const CLIENT_SETUP_TYPE = 0x20;
|
||||
const SERVER_SETUP_TYPE = 0x21;
|
||||
|
||||
// Uni stream type
|
||||
const UNI_STREAM_TYPE_GROUP = 0;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// QUIC variable-length integer codec
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function encodeVarint(value) {
|
||||
if (value < 0x40) {
|
||||
return new Uint8Array([value]);
|
||||
} else if (value < 0x4000) {
|
||||
const buf = new Uint8Array(2);
|
||||
new DataView(buf.buffer).setUint16(0, value | 0x4000);
|
||||
return buf;
|
||||
} else if (value < 0x40000000) {
|
||||
const buf = new Uint8Array(4);
|
||||
new DataView(buf.buffer).setUint32(0, (value | 0x80000000) >>> 0);
|
||||
return buf;
|
||||
} else {
|
||||
const buf = new Uint8Array(8);
|
||||
const dv = new DataView(buf.buffer);
|
||||
const hi = Math.floor(value / 0x100000000) | 0xc0000000;
|
||||
const lo = value >>> 0;
|
||||
dv.setUint32(0, hi >>> 0);
|
||||
dv.setUint32(4, lo);
|
||||
return buf;
|
||||
}
|
||||
}
|
||||
|
||||
function decodeVarint(data, offset) {
|
||||
const dv = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
||||
const first = data[offset];
|
||||
const lengthBits = first >> 6;
|
||||
|
||||
if (lengthBits === 0) {
|
||||
return [first, offset + 1];
|
||||
} else if (lengthBits === 1) {
|
||||
return [dv.getUint16(offset) & 0x3fff, offset + 2];
|
||||
} else if (lengthBits === 2) {
|
||||
return [dv.getUint32(offset) & 0x3fffffff, offset + 4];
|
||||
} else {
|
||||
const hi = dv.getUint32(offset) & 0x3fffffff;
|
||||
const lo = dv.getUint32(offset + 4);
|
||||
return [hi * 0x100000000 + lo, offset + 8];
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// String helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const decoder = new TextDecoder();
|
||||
|
||||
function encodeString(str) {
|
||||
const bytes = encoder.encode(str);
|
||||
return concat(encodeVarint(bytes.length), bytes);
|
||||
}
|
||||
|
||||
function decodeString(data, offset) {
|
||||
const [len, off] = decodeVarint(data, offset);
|
||||
const str = decoder.decode(data.subarray(off, off + len));
|
||||
return [str, off + len];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Buffer helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function concat(...arrays) {
|
||||
let total = 0;
|
||||
for (const a of arrays) total += a.byteLength;
|
||||
const result = new Uint8Array(total);
|
||||
let pos = 0;
|
||||
for (const a of arrays) {
|
||||
result.set(a instanceof Uint8Array ? a : new Uint8Array(a), pos);
|
||||
pos += a.byteLength;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Setup messages (WebTransport uses u16 body size prefix)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Encode CLIENT_SETUP for WebTransport (browser).
|
||||
*
|
||||
* WebTransport framing: varint(0x20) as stream type, then u16(body_len) + body
|
||||
* on a dedicated bidi stream.
|
||||
*/
|
||||
function encodeClientSetup(role, versions, path) {
|
||||
let body = encodeVarint(versions.length);
|
||||
for (const v of versions) {
|
||||
body = concat(body, encodeVarint(v));
|
||||
}
|
||||
|
||||
// Parameters
|
||||
let numParams = 1; // role
|
||||
if (path) numParams += 1;
|
||||
body = concat(body, encodeVarint(numParams));
|
||||
|
||||
// Role param (key=0)
|
||||
body = concat(body, encodeVarint(0), encodeVarint(1), encodeVarint(role));
|
||||
|
||||
// Path param (key=1)
|
||||
if (path) {
|
||||
const pathBytes = encoder.encode(path);
|
||||
body = concat(body, encodeVarint(1), encodeVarint(pathBytes.length), pathBytes);
|
||||
}
|
||||
|
||||
// WebTransport framing: u16(body_len) + body
|
||||
const frame = new Uint8Array(2 + body.length);
|
||||
new DataView(frame.buffer).setUint16(0, body.length);
|
||||
frame.set(body, 2);
|
||||
|
||||
return frame;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse SERVER_SETUP from WebTransport.
|
||||
*
|
||||
* WebTransport framing: u16(body_len) + body
|
||||
* Body: varint(selected_version) + params...
|
||||
*/
|
||||
function parseServerSetup(data) {
|
||||
const dv = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
||||
const bodyLen = dv.getUint16(0);
|
||||
let offset = 2;
|
||||
const [version, newOff] = decodeVarint(data, offset);
|
||||
return { version, bodyEnd: 2 + bodyLen };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Subscribe messages (on dedicated bidi stream with stream_type=2)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Encode SUBSCRIBE message body.
|
||||
*
|
||||
* Stream format: varint(2) + varint(body_len) + body
|
||||
* Body: varint(sub_id) + string(broadcast_path) + string(track_name) + u8(priority)
|
||||
*/
|
||||
function encodeSubscribe(subscribeId, broadcastPath, trackName, priority = 128) {
|
||||
let body = concat(
|
||||
encodeVarint(subscribeId),
|
||||
encodeString(broadcastPath),
|
||||
encodeString(trackName),
|
||||
new Uint8Array([priority]),
|
||||
);
|
||||
|
||||
// Stream type + length-prefixed body
|
||||
return concat(
|
||||
encodeVarint(StreamType.SUBSCRIBE),
|
||||
encodeVarint(body.length),
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode SUBSCRIBE_OK response.
|
||||
* Format: varint(0) — empty body.
|
||||
*/
|
||||
function encodeSubscribeOk() {
|
||||
return encodeVarint(0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a SUBSCRIBE message from an incoming bidi stream.
|
||||
* Input starts after the stream type varint has been consumed.
|
||||
*/
|
||||
function decodeSubscribe(data, offset = 0) {
|
||||
let bodyLen;
|
||||
[bodyLen, offset] = decodeVarint(data, offset);
|
||||
const bodyEnd = offset + bodyLen;
|
||||
|
||||
let subscribeId, broadcastPath, trackName;
|
||||
[subscribeId, offset] = decodeVarint(data, offset);
|
||||
[broadcastPath, offset] = decodeString(data, offset);
|
||||
[trackName, offset] = decodeString(data, offset);
|
||||
const priority = data[offset];
|
||||
offset += 1;
|
||||
|
||||
return { subscribeId, broadcastPath, trackName, priority, end: bodyEnd };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// GROUP + FRAME messages (on unidirectional streams)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Encode GROUP header + single FRAME for a uni stream.
|
||||
*
|
||||
* Format: u8(0) + varint(header_body_len) + varint(subscribe_id) + varint(group_seq)
|
||||
* + varint(payload_len) + payload
|
||||
*/
|
||||
function encodeGroupAndFrame(subscribeId, groupSeq, payload) {
|
||||
const headerBody = concat(
|
||||
encodeVarint(subscribeId),
|
||||
encodeVarint(groupSeq),
|
||||
);
|
||||
|
||||
const frame = concat(
|
||||
encodeVarint(payload.byteLength),
|
||||
payload instanceof Uint8Array ? payload : new Uint8Array(payload),
|
||||
);
|
||||
|
||||
return concat(
|
||||
new Uint8Array([UNI_STREAM_TYPE_GROUP]),
|
||||
encodeVarint(headerBody.length),
|
||||
headerBody,
|
||||
frame,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse GROUP header + FRAMEs from a complete uni stream buffer.
|
||||
*
|
||||
* Returns { subscribeId, groupSeq, frames: [Uint8Array, ...] }
|
||||
*/
|
||||
function parseGroupStream(data) {
|
||||
let offset = 0;
|
||||
|
||||
// u8(0) stream type
|
||||
const streamType = data[offset];
|
||||
offset += 1;
|
||||
|
||||
// varint(body_len)
|
||||
let bodyLen;
|
||||
[bodyLen, offset] = decodeVarint(data, offset);
|
||||
const bodyStart = offset;
|
||||
|
||||
// varint(subscribe_id)
|
||||
let subscribeId;
|
||||
[subscribeId, offset] = decodeVarint(data, offset);
|
||||
|
||||
// varint(group_seq)
|
||||
let groupSeq;
|
||||
[groupSeq, offset] = decodeVarint(data, offset);
|
||||
|
||||
// Advance past header body
|
||||
offset = bodyStart + bodyLen;
|
||||
|
||||
// Parse frames: varint(payload_len) + payload, repeated
|
||||
const frames = [];
|
||||
while (offset < data.length) {
|
||||
let payloadLen;
|
||||
[payloadLen, offset] = decodeVarint(data, offset);
|
||||
frames.push(data.subarray(offset, offset + payloadLen));
|
||||
offset += payloadLen;
|
||||
}
|
||||
|
||||
return { subscribeId, groupSeq, frames };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Exports (global for vanilla JS)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
window.MOQ = {
|
||||
MOQL_VERSION,
|
||||
Role,
|
||||
StreamType,
|
||||
CLIENT_SETUP_TYPE,
|
||||
SERVER_SETUP_TYPE,
|
||||
encodeVarint,
|
||||
decodeVarint,
|
||||
encodeString,
|
||||
decodeString,
|
||||
concat,
|
||||
encodeClientSetup,
|
||||
parseServerSetup,
|
||||
encodeSubscribe,
|
||||
encodeSubscribeOk,
|
||||
decodeSubscribe,
|
||||
encodeGroupAndFrame,
|
||||
parseGroupStream,
|
||||
};
|
||||
|
||||
})();
|
||||
242
moq_prebuilt/client/style.css
Normal file
242
moq_prebuilt/client/style.css
Normal file
@@ -0,0 +1,242 @@
|
||||
/* Palette borrowed from @pipecat-ai/voice-ui-kit (dark theme) so this
|
||||
custom MOQ client visually matches the smallwebrtc playground. */
|
||||
:root {
|
||||
--background: oklch(0.141 0.005 285.823);
|
||||
--foreground: oklch(0.985 0 0);
|
||||
--card: oklch(0.21 0.006 285.885);
|
||||
--muted: oklch(0.274 0.006 286.033);
|
||||
--muted-foreground: oklch(0.705 0.015 286.067);
|
||||
--border: oklch(1 0 0 / 10%);
|
||||
--border-strong: oklch(1 0 0 / 16%);
|
||||
--primary: oklch(0.85 0.13 162); /* green Connect */
|
||||
--primary-hover: oklch(0.78 0.14 162);
|
||||
--destructive: oklch(0.577 0.245 27.325); /* red Disconnect */
|
||||
--destructive-hover: oklch(0.52 0.225 27.325);
|
||||
--warning: oklch(0.795 0.184 86.047);
|
||||
--user-bubble: oklch(0.305 0.045 265);
|
||||
--assistant-bubble: oklch(0.255 0.012 286);
|
||||
--radius-md: 6px;
|
||||
--radius-lg: 8px;
|
||||
--radius-xl: 12px;
|
||||
--font-sans: -apple-system, BlinkMacSystemFont, "Inter", "Segoe UI", Roboto, sans-serif;
|
||||
--font-mono: "SF Mono", "JetBrains Mono", "Fira Code", "Cascadia Code", Consolas, monospace;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
html, body {
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: var(--font-sans);
|
||||
background: var(--background);
|
||||
color: var(--foreground);
|
||||
font-size: 14px;
|
||||
line-height: 1.5;
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
padding: 48px 24px 80px;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
}
|
||||
|
||||
.container {
|
||||
max-width: 720px;
|
||||
width: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: 1.05rem;
|
||||
font-weight: 600;
|
||||
letter-spacing: -0.005em;
|
||||
color: var(--foreground);
|
||||
margin: 0 0 8px 4px;
|
||||
}
|
||||
|
||||
/* ---------- Card ---------- */
|
||||
|
||||
.card {
|
||||
background: var(--card);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-xl);
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
.card h3 {
|
||||
font-family: var(--font-mono);
|
||||
font-size: 0.68rem;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.12em;
|
||||
text-transform: uppercase;
|
||||
color: var(--muted-foreground);
|
||||
margin-bottom: 14px;
|
||||
}
|
||||
|
||||
/* ---------- Connection card (status + buttons) ---------- */
|
||||
|
||||
.status {
|
||||
font-size: 1rem;
|
||||
font-weight: 500;
|
||||
padding: 4px 0 14px;
|
||||
color: var(--muted-foreground);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.status::before {
|
||||
content: "";
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
background: var(--muted-foreground);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.status.disconnected { color: var(--muted-foreground); }
|
||||
.status.disconnected::before { background: var(--muted-foreground); }
|
||||
.status.connecting { color: var(--warning); }
|
||||
.status.connecting::before { background: var(--warning); }
|
||||
.status.connected { color: var(--primary); }
|
||||
.status.connected::before { background: var(--primary); box-shadow: 0 0 0 4px oklch(0.85 0.13 162 / 0.15); }
|
||||
.status.error { color: var(--destructive); }
|
||||
.status.error::before { background: var(--destructive); }
|
||||
|
||||
.controls {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
button {
|
||||
font-family: inherit;
|
||||
font-size: 0.85rem;
|
||||
font-weight: 500;
|
||||
letter-spacing: 0.01em;
|
||||
padding: 8px 18px;
|
||||
border: 1px solid transparent;
|
||||
border-radius: var(--radius-md);
|
||||
cursor: pointer;
|
||||
transition: background 0.15s, border-color 0.15s, opacity 0.15s;
|
||||
}
|
||||
|
||||
button:disabled {
|
||||
opacity: 0.35;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
#connectBtn {
|
||||
background: var(--primary);
|
||||
color: oklch(0.16 0.02 162);
|
||||
}
|
||||
|
||||
#connectBtn:hover:not(:disabled) {
|
||||
background: var(--primary-hover);
|
||||
}
|
||||
|
||||
#disconnectBtn {
|
||||
background: transparent;
|
||||
border-color: var(--destructive);
|
||||
color: var(--destructive);
|
||||
}
|
||||
|
||||
#disconnectBtn:hover:not(:disabled) {
|
||||
background: oklch(0.577 0.245 27.325 / 0.15);
|
||||
}
|
||||
|
||||
/* ---------- Transcript ---------- */
|
||||
|
||||
.transcript {
|
||||
font-size: 0.95rem;
|
||||
line-height: 1.55;
|
||||
max-height: 360px;
|
||||
min-height: 48px;
|
||||
overflow-y: auto;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
padding-right: 4px;
|
||||
}
|
||||
|
||||
.transcript:empty::before {
|
||||
content: "Waiting for conversation…";
|
||||
color: var(--muted-foreground);
|
||||
font-style: italic;
|
||||
opacity: 0.6;
|
||||
}
|
||||
|
||||
.transcript-row {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 3px;
|
||||
padding: 10px 14px;
|
||||
border-radius: var(--radius-lg);
|
||||
border: 1px solid var(--border);
|
||||
max-width: 88%;
|
||||
align-self: flex-start;
|
||||
background: var(--assistant-bubble);
|
||||
}
|
||||
|
||||
.transcript-user {
|
||||
background: var(--user-bubble);
|
||||
}
|
||||
|
||||
.transcript-role {
|
||||
font-family: var(--font-mono);
|
||||
font-size: 0.62rem;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.12em;
|
||||
text-transform: uppercase;
|
||||
color: var(--muted-foreground);
|
||||
}
|
||||
|
||||
.transcript-text {
|
||||
color: var(--foreground);
|
||||
white-space: pre-wrap;
|
||||
word-wrap: break-word;
|
||||
}
|
||||
|
||||
/* ---------- Log ---------- */
|
||||
|
||||
.log {
|
||||
font-family: var(--font-mono);
|
||||
font-size: 0.72rem;
|
||||
line-height: 1.7;
|
||||
max-height: 360px;
|
||||
overflow-y: auto;
|
||||
color: var(--muted-foreground);
|
||||
}
|
||||
|
||||
.log div {
|
||||
padding: 2px 0;
|
||||
border-bottom: 1px solid oklch(1 0 0 / 0.04);
|
||||
}
|
||||
|
||||
.log div:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
/* ---------- Scrollbar polish ---------- */
|
||||
|
||||
.transcript::-webkit-scrollbar,
|
||||
.log::-webkit-scrollbar {
|
||||
width: 6px;
|
||||
}
|
||||
|
||||
.transcript::-webkit-scrollbar-thumb,
|
||||
.log::-webkit-scrollbar-thumb {
|
||||
background: var(--border-strong);
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
.transcript::-webkit-scrollbar-track,
|
||||
.log::-webkit-scrollbar-track {
|
||||
background: transparent;
|
||||
}
|
||||
Reference in New Issue
Block a user