From 7ecba34c4085259e27150f8c58d14dc753ffc55b Mon Sep 17 00:00:00 2001 From: vipyne Date: Fri, 13 Feb 2026 00:22:06 -0600 Subject: [PATCH] POC: Add MoQ transport --- .gitignore | 5 +- examples/transports/transports-moq.py | 209 ++++ moq_prebuilt/PLAN-moq-transport-package.md | 115 +++ moq_prebuilt/__init__.py | 0 moq_prebuilt/client/app.js | 878 ++++++++++++++++ moq_prebuilt/client/index.html | 36 + moq_prebuilt/client/moq-client.js | 324 ++++++ moq_prebuilt/client/style.css | 242 +++++ moq_prebuilt/frontend.py | 31 + pyproject.toml | 1 + scripts/moq-dev-setup.sh | 98 ++ src/pipecat/runner/run.py | 199 +++- src/pipecat/runner/types.py | 29 +- src/pipecat/runner/utils.py | 21 + src/pipecat/transports/moq/__init__.py | 26 + src/pipecat/transports/moq/protocol.py | 546 ++++++++++ src/pipecat/transports/moq/transport.py | 1072 ++++++++++++++++++++ uv.lock | 80 +- 18 files changed, 3906 insertions(+), 6 deletions(-) create mode 100644 examples/transports/transports-moq.py create mode 100644 moq_prebuilt/PLAN-moq-transport-package.md create mode 100644 moq_prebuilt/__init__.py create mode 100644 moq_prebuilt/client/app.js create mode 100644 moq_prebuilt/client/index.html create mode 100644 moq_prebuilt/client/moq-client.js create mode 100644 moq_prebuilt/client/style.css create mode 100644 moq_prebuilt/frontend.py create mode 100755 scripts/moq-dev-setup.sh create mode 100644 src/pipecat/transports/moq/__init__.py create mode 100644 src/pipecat/transports/moq/protocol.py create mode 100644 src/pipecat/transports/moq/transport.py diff --git a/.gitignore b/.gitignore index 512ec7316..fc3fcf6e7 100644 --- a/.gitignore +++ b/.gitignore @@ -61,4 +61,7 @@ docs/api/api .python-version # Pipecat -whisker_setup.py \ No newline at end of file +whisker_setup.py + +# MoQ transport +*.pem \ No newline at end of file diff --git a/examples/transports/transports-moq.py b/examples/transports/transports-moq.py new file mode 100644 index 000000000..91cbac9b6 --- /dev/null +++ b/examples/transports/transports-moq.py @@ -0,0 +1,209 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""MOQ (Media over QUIC) transport example. + +This example demonstrates using the MOQ transport for real-time voice +conversations over QUIC, connecting to a MOQ relay server. It uses the +unified runner pattern that works with Daily, WebRTC, and MOQ transports. + +MOQ provides WebRTC-like latency without WebRTC constraints, using QUIC +for prioritization and partial reliability. + +Requirements: + uv sync --extra moq --extra silero --extra deepgram --extra cartesia \ + --extra openai --extra runner + + # You also need a MOQ relay running locally. Clone moq-relay from + # https://github.com/kixelated/moq and then run scripts/moq-dev-setup.sh + # from this repo, pointing at the relay checkout. The script generates + # a self-signed cert, symlinks it into both repos, and prints the + # exact relay + bot run commands to copy. + # + # git clone https://github.com/kixelated/moq.git ../moq + # ./scripts/moq-dev-setup.sh ../moq + # + # Then in two terminals run the commands the script printed (relay + # binds QUIC on UDP [::]:4080 with --auth-public ''). + +Usage: + # Run with MOQ transport (connects to local relay set up by the script): + uv run python examples/transports/transports-moq.py \\ + -t moq --moq-cert moq-cert.pem --moq-insecure --moq-path / + + # Connect to a remote relay (CA-signed cert, no pinning needed): + uv run python examples/transports/transports-moq.py \\ + -t moq --moq-host moq.example.com + + # With a custom namespace (different "room"): + uv run python examples/transports/transports-moq.py \\ + -t moq --moq-cert moq-cert.pem --moq-insecure --moq-namespace my-room + + # Then open the browser client at http://localhost:7860 and click Connect. + + # Can also run with other transports (no relay needed): + uv run python examples/transports/transports-moq.py -t webrtc + uv run python examples/transports/transports-moq.py -t daily +""" + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import MOQRunnerArguments, RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.moq import MOQParams +from pipecat.transports.moq.protocol import MOQRole + +load_dotenv(override=True) + +# Transport-specific parameters using lambdas for deferred creation +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "moq": lambda: MOQParams( + audio_in_enabled=True, + audio_out_enabled=True, + role=MOQRole.PUBSUB, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + """Run the bot with the given transport.""" + logger.info("Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful assistant in a real-time voice call. " + "Your goal is to demonstrate your capabilities in a succinct way. " + "Your output will be spoken aloud, so avoid special characters that can't easily be " + "spoken, such as emojis or bullet points. Respond to what the user said in a creative " + "and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + user_aggregator, # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + assistant_aggregator, # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + # For MOQ, we need to handle connection and events differently + if isinstance(runner_args, MOQRunnerArguments): + @transport.event_handler("on_connected") + async def on_connected(transport): + logger.info("Connected to MOQ relay (waiting for client to join)") + if runner_args.ready_event is not None: + runner_args.ready_event.set() + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport): + logger.info("Client subscribed — starting conversation") + messages.append( + {"role": "system", "content": "Please introduce yourself to the user."} + ) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_disconnected") + async def on_disconnected(transport): + logger.info("Disconnected from MOQ relay") + await task.cancel() + + @transport.event_handler("on_error") + async def on_error(transport, message, exception): + logger.error(f"MOQ error: {message}") + + # MOQInputTransport.start() auto-connects to the relay when the + # pipeline starts, so we don't dial transport.connect() here. + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + try: + await runner.run(task) + finally: + await transport.disconnect() + else: + # Daily and WebRTC use on_client_connected/on_client_disconnected + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info("Client connected") + messages.append( + {"role": "system", "content": "Please introduce yourself to the user."} + ) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info("Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat runner.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/moq_prebuilt/PLAN-moq-transport-package.md b/moq_prebuilt/PLAN-moq-transport-package.md new file mode 100644 index 000000000..815718ed5 --- /dev/null +++ b/moq_prebuilt/PLAN-moq-transport-package.md @@ -0,0 +1,115 @@ +# Plan: `@pipecat-ai/moq-transport` + +A future Pipecat Client SDK transport plugin for Media-over-QUIC. Lets any +Pipecat JS UI — including the smallwebrtc playground — talk to a +MOQ-backed bot with zero UI changes, just by swapping the transport. + +## Why this exists + +The Pipecat Client SDK (`@pipecat-ai/client-js`) is transport-agnostic. UIs +talk to a `PipecatClient`, which delegates to a pluggable `Transport`. The +existing transports are: + +- `@pipecat-ai/small-webrtc-transport` (WebRTC) +- `@pipecat-ai/daily-transport` (Daily) +- `@pipecat-ai/websocket-transport` (raw WebSocket / telephony) +- `@pipecat-ai/gemini-live-websocket-transport` + +There is **no** `@pipecat-ai/moq-transport`. So MOQ bots today need a hand- +rolled client (which is what `moq_prebuilt/client/` is). Publishing a +proper transport plugin would: + +1. Replace `moq_prebuilt/client/` with the prebuilt smallwebrtc UI — same + conversation panel, metrics, device pickers, transcript overlay, etc. +2. Let downstream Pipecat apps drop MOQ in as easily as WebRTC. +3. Centralize the MOQ wire-protocol work in one package instead of every + demo reinventing it. + +## What we already have + +About 80% of the wire-protocol work is done in +`moq_prebuilt/client/app.js` and `moq_prebuilt/client/moq-client.js`: + +- moq-lite-02 codec (CLIENT_SETUP, SERVER_SETUP, ANNOUNCE_PLEASE / + ANNOUNCE_INIT, SUBSCRIBE / SUBSCRIBE_OK, GROUP / FRAME). +- WebTransport bring-up with certificate pinning. +- Per-participant broadcast paths (`/`) and + multi-track routing (`bot-audio`, `user-audio`, `transcript`). +- Mic capture via AudioWorklet → 16 kHz PCM publish. +- Bot-audio 24 kHz PCM playback via Web Audio. +- RTVI message parsing on the transcript track (`bot-llm-*`, + `user-transcription`, etc.). + +The package's job is to **wrap that in the `Transport` interface** and +emit the events `PipecatClient` expects. + +## Effort estimate + +### Usable v1 — 3–5 days + +For a single bot / single client dev setup that runs against the +playground UI. + +| Day | Work | +| --- | ---- | +| 1 | TypeScript package skeleton (build, tsconfig, ESM/CJS exports, npm scripts). Port existing protocol logic into a class. | +| 2 | Implement `Transport` interface: `connect()`, `disconnect()`, `sendMessage()`, `getDevices()`. Wire the RTVI message parsing on the transcript track into the event names `PipecatClient` expects (`connected`, `disconnected`, `trackStarted`, `userTranscript`, `botStartedSpeaking`, …). | +| 3 | Device handling — mic picker, mute, sample-rate negotiation, AudioWorklet inlined as a string blob. | +| 4 | Run against `ConsoleTemplate` from `@pipecat-ai/voice-ui-kit`. Fix lifecycle bugs (mid-call mute, reconnect, etc.). | +| 5 | Docs, example app, README. | + +### Production-grade — ~2 weeks on top + +- Multi-participant discovery via outbound `ANNOUNCE_PLEASE` + reacting to + `ANNOUNCE_UPDATE` so multiple bots / multiple clients can share a namespace. +- CA-signed cert path (not just self-signed `serverCertificateHashes` + pinning). +- Reconnection / network-blip recovery. +- Test suite (unit + at least one e2e against a real relay). +- Cross-browser sanity (Chrome reference, Safari WebTransport just + shipped, Firefox still behind a flag). +- npm publish workflow / CI. + +## Known unknowns (could blow the estimate) + +- **PipecatClient `Transport` surface area** — haven't done a full read of + `@pipecat-ai/client-js`. If the interface expects something MOQ can't + cleanly model (peer-connection introspection, ICE candidates, anything + WebRTC-specific that leaks into the abstraction), we need + workarounds/polyfills. +- **Audio interop with the voice-ui-kit's audio visualizers** — + ``, level meters, etc. read from a `MediaStreamTrack`. MOQ + delivers raw PCM uni-streams, not a media-stream-track. We'd need to + synthesize a `MediaStreamTrack` from the decoded PCM so the rest of the + kit just works. Could be a half-day rabbit hole or could be quick — there + are off-the-shelf libraries (`AudioWorklet` + `MediaStreamDestination`) but + the integration details aren't trivial. + +## Suggested next step (cheap reconnaissance) + +Spend ~2 hours reading: +- `@pipecat-ai/client-js/src/transport.ts` — the `Transport` abstract + interface. +- `@pipecat-ai/small-webrtc-transport` — closest analogue, since both + open a session and stream media bidirectionally. +- `@pipecat-ai/websocket-transport` — simpler reference; might be a closer + fit for MOQ since neither is WebRTC. + +Output: a concrete TypeScript API surface for `MoqTransport` (constructor +options, method signatures, events emitted) + a sharper estimate. Two +hours of investment to either commit to the project or pass on it +informedly. + +## Open questions for whoever picks this up + +- One participant identity per `Transport` instance, or should the + transport manage multiple peers internally (matching how WebRTC SFUs do + it)? +- How should `getDevices()` map to MOQ — pure browser device enumeration + (since there's no SDK-side device negotiation)? +- Where does the per-call ID come from? Server-allocated via + `/start`, or client-generated UUID? (Today we hardcode `bot0` / + `client0`.) +- Cert pinning UX — for prod, can we get away with assuming the relay has + a real CA cert, or do we need an explicit `serverCertificateHashes` + config knob? diff --git a/moq_prebuilt/__init__.py b/moq_prebuilt/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/moq_prebuilt/client/app.js b/moq_prebuilt/client/app.js new file mode 100644 index 000000000..5788bf691 --- /dev/null +++ b/moq_prebuilt/client/app.js @@ -0,0 +1,878 @@ +/* + * Copyright (c) 2024-2026, Daily + * + * SPDX-License-Identifier: BSD 2-Clause License + */ + +/** + * MOQ browser client — moq-lite-02. + * + * Connects to a moq-lite relay via WebTransport, captures microphone audio, + * and plays back received audio using stream-per-request model. + * + * Flow: + * 1. WebTransport connect + * 2. Open bidi stream for SETUP: write CLIENT_SETUP, read SERVER_SETUP + * 3. Listen for incoming bidi streams (relay sends SUBSCRIBE / ANNOUNCE) + * 4. Listen for incoming uni streams (receive bot audio as GROUP + FRAME) + * 5. Open bidi stream for SUBSCRIBE (to bot's audio track) + * 6. Send mic PCM as uni streams (GROUP + FRAME per chunk) + */ + +const { + MOQL_VERSION, + Role, + StreamType, + CLIENT_SETUP_TYPE, + SERVER_SETUP_TYPE, + concat, + encodeClientSetup, + parseServerSetup, + encodeSubscribe, + encodeSubscribeOk, + decodeSubscribe, + encodeGroupAndFrame, + parseGroupStream, + encodeVarint, + decodeVarint, + encodeString, + decodeString, +} = window.MOQ; + +// --------------------------------------------------------------------------- +// State +// --------------------------------------------------------------------------- + +let transport = null; +let micStream = null; +let audioContext = null; +let micWorklet = null; +let connected = false; + +// Subscription IDs +let nextSubscribeId = 1; + +// Publishing state +let pubSubscribeId = null; // set when relay subscribes to our audio +let pubGroupSeq = 0; + +// Subscription state (for receiving bot audio and transcript) +let subAudioSubscribeId = null; +let subTranscriptSubscribeId = null; + +// Playback +let playbackTime = 0; + +// Config (populated from /api/config) +let config = {}; + +// --------------------------------------------------------------------------- +// UI helpers +// --------------------------------------------------------------------------- + +function setStatus(text, className) { + const el = document.getElementById("status"); + el.textContent = text; + el.className = "status " + (className || ""); +} + +function log(msg, level = "info") { + if (level === "error") { + console.error("[moq]", msg); + } else if (level === "warn") { + console.warn("[moq]", msg); + } else { + console.log("[moq]", msg); + } + const el = document.getElementById("log"); + const line = document.createElement("div"); + line.textContent = `[${new Date().toLocaleTimeString()}] ${msg}`; + if (level === "error") line.style.color = "#e74c3c"; + else if (level === "warn") line.style.color = "#f39c12"; + el.appendChild(line); + el.scrollTop = el.scrollHeight; +} + +function hexdump(data, maxBytes = 40) { + const bytes = data instanceof Uint8Array ? data : new Uint8Array(data); + const hex = Array.from(bytes.slice(0, maxBytes)).map(b => b.toString(16).padStart(2, '0')).join(' '); + return bytes.length > maxBytes ? `${hex}... (${bytes.length} bytes total)` : `${hex} (${bytes.length} bytes)`; +} + +// Turn-based transcript: each in-progress LLM/user turn is one row whose +// text accumulates as per-token RTVI messages arrive. Mirrors the +// ConversationProvider model in @pipecat-ai/voice-ui-kit. +// +// A user "turn" runs from the first user-transcription until the bot +// starts speaking — Deepgram can emit several final transcripts inside +// one spoken utterance, and we want all of them in the same bubble. +const turnState = { + assistantRow: null, // DOM
for the in-progress assistant turn + assistantText: "", // accumulated text for the in-progress turn + userRow: null, // DOM
for the in-progress user turn + userCommitted: "", // accumulated final user text within the turn + userInterim: "", // latest non-final user text (tentative tail) +}; + +function newTranscriptRow(role) { + const el = document.getElementById("transcript"); + if (!el) return null; + const row = document.createElement("div"); + const isUser = role === "user"; + row.className = `transcript-row transcript-${isUser ? "user" : "assistant"}`; + const label = document.createElement("span"); + label.className = "transcript-role"; + label.textContent = isUser ? "user" : "assistant"; + const body = document.createElement("span"); + body.className = "transcript-text"; + row.appendChild(label); + row.appendChild(body); + el.appendChild(row); + el.scrollTop = el.scrollHeight; + return row; +} + +function setRowText(row, text) { + if (!row) return; + const body = row.querySelector(".transcript-text"); + if (body) body.textContent = text; + const el = document.getElementById("transcript"); + if (el) el.scrollTop = el.scrollHeight; +} + +function clearTranscript() { + const el = document.getElementById("transcript"); + if (el) el.innerHTML = ""; + turnState.assistantRow = null; + turnState.assistantText = ""; + turnState.userRow = null; + turnState.userCommitted = ""; + turnState.userInterim = ""; +} + +// Concatenate streaming tokens with a sensible separator. Some LLM tokens +// already include leading whitespace (e.g. " world"); avoid double spaces. +function appendStreamingText(prev, chunk) { + if (!prev) return chunk; + if (!chunk) return prev; + const needsSpace = !prev.endsWith(" ") && !/^[\s.,!?;:'")\]]/.test(chunk); + return prev + (needsSpace ? " " : "") + chunk; +} + +// RTVI message dispatcher. Matches the ConversationProvider semantics from +// @pipecat-ai/voice-ui-kit: per-LLM-turn aggregation for the bot, upsert +// for user transcripts. +function handleRtviMessage(msg) { + if (!msg || typeof msg !== "object") return; + const type = msg.type; + const data = msg.data || {}; + + switch (type) { + case "bot-llm-started": + // Don't create the row yet — wait for the first bot-llm-text so we + // don't show an empty "assistant" label if no text follows. + turnState.assistantRow = null; + turnState.assistantText = ""; + // Bot is responding: the user's turn just ended. The next + // user-transcription should start a fresh row. + turnState.userRow = null; + turnState.userCommitted = ""; + turnState.userInterim = ""; + break; + + case "bot-llm-text": + if (!data.text) break; + if (!turnState.assistantRow) { + // Lazy: covers both the post-started case and pipelines that emit + // bot-llm-text without a preceding bot-llm-started. + turnState.assistantRow = newTranscriptRow("assistant"); + } + turnState.assistantText = appendStreamingText(turnState.assistantText, data.text); + setRowText(turnState.assistantRow, turnState.assistantText); + break; + + case "bot-llm-stopped": + turnState.assistantRow = null; + turnState.assistantText = ""; + break; + + case "user-transcription": { + const text = data.text || ""; + const final = data.final !== false; + if (!turnState.userRow) { + turnState.userRow = newTranscriptRow("user"); + turnState.userCommitted = ""; + turnState.userInterim = ""; + } + if (final) { + turnState.userCommitted = turnState.userCommitted + ? appendStreamingText(turnState.userCommitted, text) + : text; + turnState.userInterim = ""; + setRowText(turnState.userRow, turnState.userCommitted); + } else { + turnState.userInterim = text; + const combined = turnState.userCommitted + ? appendStreamingText(turnState.userCommitted, text) + : text; + setRowText(turnState.userRow, combined); + } + break; + } + + default: + // Ignore other RTVI messages (metrics, speech start/stop, bot-output, + // bot-tts-*, function calls, etc.). Uncomment to discover what's + // flowing on the transcript track. + // log(`RTVI: ${type}`); + break; + } +} + +// --------------------------------------------------------------------------- +// Audio playback — queue received 16-bit PCM into Web Audio +// --------------------------------------------------------------------------- + +function initAudioPlayback() { + if (audioContext) return; + audioContext = new AudioContext({ sampleRate: 24000 }); + playbackTime = audioContext.currentTime; +} + +function playPcmChunk(pcmBytes) { + if (!audioContext) return; + + // pcmBytes is 16-bit signed LE PCM — copy to aligned buffer to avoid offset issues + const aligned = new ArrayBuffer(pcmBytes.byteLength); + new Uint8Array(aligned).set(pcmBytes); + const samples = new Int16Array(aligned); + const floats = new Float32Array(samples.length); + for (let i = 0; i < samples.length; i++) { + floats[i] = samples[i] / 32768; + } + + const buffer = audioContext.createBuffer(1, floats.length, 24000); + buffer.getChannelData(0).set(floats); + + const source = audioContext.createBufferSource(); + source.buffer = buffer; + source.connect(audioContext.destination); + + const now = audioContext.currentTime; + if (playbackTime < now) playbackTime = now; + source.start(playbackTime); + playbackTime += buffer.duration; +} + +// --------------------------------------------------------------------------- +// Microphone capture — 16kHz 16-bit PCM via AudioWorklet +// --------------------------------------------------------------------------- + +const WORKLET_CODE = ` +class PcmCapture extends AudioWorkletProcessor { + process(inputs) { + const input = inputs[0]; + if (input.length > 0) { + const floats = input[0]; + const pcm = new Int16Array(floats.length); + for (let i = 0; i < floats.length; i++) { + pcm[i] = Math.max(-32768, Math.min(32767, Math.round(floats[i] * 32768))); + } + this.port.postMessage(pcm.buffer, [pcm.buffer]); + } + return true; + } +} +registerProcessor('pcm-capture', PcmCapture); +`; + +async function startMic() { + micStream = await navigator.mediaDevices.getUserMedia({ + audio: { sampleRate: 16000, channelCount: 1, echoCancellation: true }, + }); + + const ctx = new AudioContext({ sampleRate: 16000 }); + const blob = new Blob([WORKLET_CODE], { type: "application/javascript" }); + const url = URL.createObjectURL(blob); + await ctx.audioWorklet.addModule(url); + URL.revokeObjectURL(url); + + const source = ctx.createMediaStreamSource(micStream); + micWorklet = new AudioWorkletNode(ctx, "pcm-capture"); + micWorklet.port.onmessage = (e) => { + if (connected && transport && pubSubscribeId !== null) { + sendAudioUniStream(new Uint8Array(e.data)); + } + }; + source.connect(micWorklet); + micWorklet.connect(ctx.destination); + + log("Microphone started (16kHz PCM)"); +} + +function stopMic() { + if (micStream) { + micStream.getTracks().forEach((t) => t.stop()); + micStream = null; + } + if (micWorklet) { + micWorklet.disconnect(); + micWorklet = null; + } +} + +// --------------------------------------------------------------------------- +// Uni stream send (publish audio) +// --------------------------------------------------------------------------- + +let audioSendCount = 0; +let audioSendErrors = 0; + +async function sendAudioUniStream(pcmBytes) { + if (!transport || pubSubscribeId === null) return; + + try { + const groupSeq = pubGroupSeq++; + const data = encodeGroupAndFrame(pubSubscribeId, groupSeq, pcmBytes); + + const uni = await transport.createUnidirectionalStream(); + const writer = uni.getWriter(); + await writer.write(data); + await writer.close(); + + audioSendCount++; + if (audioSendCount % 100 === 0) { + log(`Audio TX: ${audioSendCount} chunks sent (${pcmBytes.byteLength} bytes/chunk, sub=${pubSubscribeId}, seq=${groupSeq})`); + } + } catch (e) { + audioSendErrors++; + if (audioSendErrors <= 5) { + log(`Audio send error #${audioSendErrors}: ${e.message} (sub=${pubSubscribeId}, seq=${pubGroupSeq})`, "warn"); + } else if (audioSendErrors === 6) { + log("Suppressing further audio send errors...", "warn"); + } + } +} + +// --------------------------------------------------------------------------- +// Uni stream receive (receive bot audio) +// --------------------------------------------------------------------------- + +let audioRecvCount = 0; + +async function receiveUniStreams() { + log("Uni stream listener started (waiting for bot audio)"); + const reader = transport.incomingUnidirectionalStreams.getReader(); + try { + while (true) { + const { value: stream, done } = await reader.read(); + if (done) { + log("Uni stream reader finished (relay closed incoming streams)"); + break; + } + handleIncomingUniStream(stream); + } + } catch (e) { + if (connected) log(`Uni stream reader stopped: ${e.message}`, "warn"); + } +} + +async function handleIncomingUniStream(stream) { + try { + // Read all data from the stream + const reader = stream.getReader(); + let buf = new Uint8Array(0); + while (true) { + const { value, done } = await reader.read(); + if (done) break; + buf = concat(buf, new Uint8Array(value)); + } + + if (buf.length === 0) return; + + // Parse GROUP + FRAMEs + const { subscribeId, groupSeq, frames } = parseGroupStream(buf); + + if (subscribeId === subTranscriptSubscribeId) { + for (const frame of frames) { + if (!frame.byteLength) continue; + try { + const text = new TextDecoder().decode(frame); + handleRtviMessage(JSON.parse(text)); + } catch (err) { + log(`Transcript frame decode error: ${err.message}`, "warn"); + } + } + return; + } + + audioRecvCount++; + + if (audioRecvCount <= 3 || audioRecvCount % 100 === 0) { + const totalBytes = frames.reduce((sum, f) => sum + f.byteLength, 0); + log(`Audio RX #${audioRecvCount}: sub=${subscribeId} seq=${groupSeq} frames=${frames.length} bytes=${totalBytes}`); + } + + for (const frame of frames) { + if (frame.byteLength > 0) { + playPcmChunk(frame); + } + } + } catch (e) { + log(`Uni stream parse error: ${e.message} (buffer may be malformed)`, "warn"); + } +} + +// --------------------------------------------------------------------------- +// Incoming bidi stream handler (relay sends SUBSCRIBE / ANNOUNCE) +// --------------------------------------------------------------------------- + +let incomingBidiCount = 0; + +async function handleIncomingBidiStreams() { + log("Bidi stream listener started (waiting for relay SUBSCRIBE/ANNOUNCE)"); + const reader = transport.incomingBidirectionalStreams.getReader(); + try { + while (true) { + const { value: stream, done } = await reader.read(); + if (done) { + log("Incoming bidi stream reader finished (relay closed)"); + break; + } + incomingBidiCount++; + log(`Incoming bidi stream #${incomingBidiCount}`); + handleIncomingBidiStream(stream); + } + } catch (e) { + if (connected) log(`Bidi stream reader stopped: ${e.message}`, "warn"); + } +} + +async function handleIncomingBidiStream(stream) { + try { + const streamReader = stream.readable.getReader(); + let buf = new Uint8Array(0); + + // Read all available data (relay sends stream type + message body) + while (true) { + const { value, done } = await streamReader.read(); + if (value) buf = concat(buf, new Uint8Array(value)); + // Break once we have data or stream ends + if (done || buf.length > 0) break; + } + + if (buf.length === 0) { + log("Incoming bidi stream was empty (0 bytes)", "warn"); + return; + } + + log(`Incoming bidi data: [${hexdump(buf, 30)}]`); + + // Decode stream type + const [streamType, offset] = decodeVarint(buf, 0); + const streamTypeName = streamType === StreamType.SUBSCRIBE ? "SUBSCRIBE" + : streamType === StreamType.ANNOUNCE ? "ANNOUNCE" + : streamType === StreamType.SESSION ? "SESSION" + : `UNKNOWN(${streamType})`; + log(`Incoming bidi stream type: ${streamTypeName} (${streamType})`); + + if (streamType === StreamType.SUBSCRIBE) { + // We may need more data for the full subscribe message + while (buf.length < offset + 3) { + const { value, done } = await streamReader.read(); + if (value) buf = concat(buf, new Uint8Array(value)); + if (done) break; + } + + // Relay is subscribing to our track + const sub = decodeSubscribe(buf, offset); + log(`SUBSCRIBE from relay: broadcast="${sub.broadcastPath}" track="${sub.trackName}" sub_id=${sub.subscribeId} priority=${sub.priority}`); + + // Accept SUBSCRIBE only for our publish track inside our own + // broadcast. With per-participant broadcast paths, the relay only + // routes SUBSCRIBEs targeting / to us, so a + // mismatch here means something is misconfigured. + const ourBroadcast = `${config.namespace}/${config.client_id}`; + if (sub.broadcastPath !== ourBroadcast || sub.trackName !== config.publish_track) { + log(`Rejecting SUBSCRIBE for "${sub.broadcastPath}/${sub.trackName}" (we only publish "${ourBroadcast}/${config.publish_track}")`, "warn"); + stream.writable.abort(); + return; + } + + // Store subscribe_id for publishing + pubSubscribeId = sub.subscribeId; + + // Send SUBSCRIBE_OK on the writable side + const writer = stream.writable.getWriter(); + await writer.write(encodeSubscribeOk()); + writer.releaseLock(); + + log(`Sent SUBSCRIBE_OK for sub_id=${sub.subscribeId} — ready to publish ${config.publish_track}`); + + } else if (streamType === StreamType.ANNOUNCE) { + // Relay sending ANNOUNCE_PLEASE — read the full message + // Minimum is 2 bytes after stream type: varint(body_len) + varint(string_len) + while (buf.length < offset + 2) { + const { value, done } = await streamReader.read(); + if (value) buf = concat(buf, new Uint8Array(value)); + if (done) break; + } + + // Parse ANNOUNCE_PLEASE: varint(body_len) + string(path_prefix) + let pos = offset; + let bodyLen; + [bodyLen, pos] = decodeVarint(buf, pos); + let pathPrefix = ""; + if (bodyLen > 0) { + [pathPrefix, pos] = decodeString(buf, pos); + } + log(`ANNOUNCE_PLEASE from relay: prefix="${pathPrefix}" bodyLen=${bodyLen}`); + + // Respond with ANNOUNCE_INIT for our per-participant broadcast path. + // Each participant uses a distinct suffix (e.g. "pipecat/client0") + // so the relay can route SUBSCRIBEs to the right side. + const broadcast = `${config.namespace}/${config.client_id}`; + let suffix = broadcast; + if (pathPrefix) { + if (broadcast.startsWith(pathPrefix)) { + suffix = broadcast.slice(pathPrefix.length).replace(/^\//, ""); + } else { + suffix = null; + } + } + + const writer = stream.writable.getWriter(); + if (suffix !== null) { + let body = encodeVarint(1); // 1 suffix + body = concat(body, encodeString(suffix)); + const initMsg = concat(encodeVarint(body.length), body); + await writer.write(initMsg); + log(`Sent ANNOUNCE_INIT: 1 suffix="${suffix}" [${hexdump(initMsg)}]`); + } else { + const initMsg = concat(encodeVarint(1), encodeVarint(0)); + await writer.write(initMsg); + log(`Sent ANNOUNCE_INIT: 0 suffixes (our broadcast doesn't match prefix "${pathPrefix}")`); + } + writer.releaseLock(); + + } else { + log(`Unknown bidi stream type ${streamType}, raw: [${hexdump(buf)}]`, "warn"); + } + } catch (e) { + log(`Incoming bidi stream error: ${e.message}`, "error"); + } +} + +// --------------------------------------------------------------------------- +// Connect flow +// --------------------------------------------------------------------------- + +async function doConnect() { + const t0 = performance.now(); + clearTranscript(); + try { + // 1. Fetch config from the FastAPI server + log("Fetching /api/config..."); + const resp = await fetch("/api/config"); + config = await resp.json(); + log(`Config: relay=${config.relay_host}:${config.relay_port}, path="${config.path}", ns="${config.namespace}", client_id="${config.client_id}", bot_id="${config.bot_id}", publish="${config.publish_track}", subscribe="${config.subscribe_track}", insecure=${config.insecure}, cert_hash=${config.cert_hash ? config.cert_hash.slice(0, 12) + "..." : "none"}`); + + // 1b. Ask the server to start the bot. It blocks until the bot has + // finished its MOQ handshake with the relay, so our SUBSCRIBE is + // guaranteed to land at a publisher the relay already knows. + log("Starting bot via /start..."); + const startResp = await fetch("/start", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ namespace: config.namespace }), + }); + if (!startResp.ok) { + throw new Error(`/start returned HTTP ${startResp.status}`); + } + const startInfo = await startResp.json(); + log(`Bot started: session=${startInfo.sessionId}, relay=${startInfo.relay}`); + + // 2. Build relay URL + const scheme = "https"; + const host = config.relay_host; + const relayUrl = `${scheme}://${host}:${config.relay_port}${config.path || "/moq"}`; + log(`Connecting to ${relayUrl}`); + setStatus("Connecting...", "connecting"); + + // 3. Open WebTransport (connects via HTTP/3 over QUIC to the relay) + const options = {}; + if (config.cert_hash) { + const hashBytes = Uint8Array.from(atob(config.cert_hash), c => c.charCodeAt(0)); + options.serverCertificateHashes = [ + { algorithm: "sha-256", value: hashBytes.buffer }, + ]; + log(`Using certificate pinning: sha256=${config.cert_hash.slice(0, 16)}...`); + } else { + log("No certificate pinning (no cert_hash in config)", "warn"); + } + log(`WebTransport options: ${JSON.stringify({...options, serverCertificateHashes: options.serverCertificateHashes ? "[present]" : undefined})}`); + + transport = new WebTransport(relayUrl, options); + + // Monitor connection closure — extract QUIC error details + transport.closed.then(info => { + const elapsed = (performance.now() - t0).toFixed(0); + log(`Transport closed cleanly after ${elapsed}ms: closeCode=${info?.closeCode} reason="${info?.reason || ""}"`); + }).catch(err => { + const elapsed = (performance.now() - t0).toFixed(0); + // WebTransportCloseInfo may have closeCode and reason + const code = err?.closeCode ?? err?.code ?? "?"; + const reason = err?.reason ?? err?.message ?? String(err); + log(`Transport closed with error after ${elapsed}ms: code=${code} reason="${reason}"`, "error"); + log(` Error type: ${err?.constructor?.name}, keys: ${Object.keys(err || {}).join(",")}`, "error"); + }); + + // Wait for the QUIC + HTTP/3 handshake + const readyStart = performance.now(); + try { + await transport.ready; + } catch (readyErr) { + const readyMs = (performance.now() - readyStart).toFixed(0); + // Get the real error from transport.closed + let closedErr = null; + try { await transport.closed; } catch (e) { closedErr = e; } + const detail = closedErr?.message || closedErr?.reason || String(closedErr || ""); + const code = closedErr?.closeCode ?? closedErr?.code ?? "?"; + log(`WebTransport handshake FAILED after ${readyMs}ms`, "error"); + log(` ready error: ${readyErr.message}`, "error"); + log(` closed error: code=${code} detail="${detail}"`, "error"); + log(` URL: ${relayUrl}`, "error"); + log(` Cert pinning: ${config.cert_hash ? "yes" : "no"}`, "error"); + log(`Troubleshooting:`, "error"); + log(` - Is the relay running at ${host}:${config.relay_port}?`, "error"); + log(` - Does relay config have [web.http] listen = "[::]:${config.relay_port}"?`, "error"); + log(` - If using tls.generate in relay config, cert hash will change every restart`, "error"); + log(` - Try 127.0.0.1 instead of localhost (QUIC/UDP has no IPv6->IPv4 fallback)`, "error"); + throw readyErr; + } + const readyMs = (performance.now() - readyStart).toFixed(0); + log(`WebTransport connected (handshake took ${readyMs}ms)`); + + // 4. Open bidi stream for SETUP handshake + log("Opening bidi stream for CLIENT_SETUP..."); + let setupBidi; + try { + setupBidi = await transport.createBidirectionalStream(); + log("Bidi stream created for SETUP"); + } catch (e) { + // This is the "Connection lost" case — get more details + let closedErr = null; + try { await transport.closed; } catch (ce) { closedErr = ce; } + const code = closedErr?.closeCode ?? closedErr?.code ?? "?"; + const reason = closedErr?.reason ?? closedErr?.message ?? ""; + log(`Failed to create bidi stream: ${e.message}`, "error"); + log(` Underlying close: code=${code} reason="${reason}"`, "error"); + log(` This usually means the relay rejected the WebTransport session.`, "error"); + log(` Check the relay terminal for error logs (auth failure, path mismatch, etc.)`, "error"); + log(` Relay URL path was: "${config.path || "/moq"}" — does the relay expect this path?`, "error"); + throw e; + } + const setupWriter = setupBidi.writable.getWriter(); + const setupStreamReader = setupBidi.readable.getReader(); + + // Write stream type varint(0x20) + CLIENT_SETUP (u16 body size for WebTransport) + const setupMsg = encodeClientSetup(Role.PUBSUB, [MOQL_VERSION], config.path || "/moq"); + const fullSetup = concat(encodeVarint(CLIENT_SETUP_TYPE), setupMsg); + log(`Sending CLIENT_SETUP: role=PUBSUB version=0x${MOQL_VERSION.toString(16)} path="${config.path || "/moq"}" [${hexdump(fullSetup)}]`); + try { + await setupWriter.write(fullSetup); + log("CLIENT_SETUP sent, waiting for SERVER_SETUP..."); + } catch (e) { + log(`Failed to write CLIENT_SETUP: ${e.message}`, "error"); + throw e; + } + + // 5. Read SERVER_SETUP response + const setupTimeout = setTimeout(() => { + log("SERVER_SETUP not received after 5s — relay may not understand our protocol", "warn"); + }, 5000); + const { value: setupResp, done: setupDone } = await setupStreamReader.read(); + clearTimeout(setupTimeout); + + if (setupDone && !setupResp) { + log("Setup stream closed by relay without sending SERVER_SETUP", "error"); + log(" The relay may not support moq-lite-02 (0xff0dad02)", "error"); + throw new Error("No SERVER_SETUP received"); + } + + if (setupResp) { + const respData = new Uint8Array(setupResp); + log(`Received SERVER_SETUP raw: [${hexdump(respData)}]`); + // Skip stream type varint (0x21) if present + let offset = 0; + if (respData[0] === SERVER_SETUP_TYPE) { + offset = 1; + } else { + const [st, newOff] = decodeVarint(respData, 0); + if (st === SERVER_SETUP_TYPE) offset = newOff; + } + const { version } = parseServerSetup(respData.subarray(offset)); + const versionName = version === 0xff0dad02 ? "moq-lite-02" : `unknown (0x${version.toString(16)})`; + log(`SERVER_SETUP: relay version=${versionName} (0x${version.toString(16)}), client version=moq-lite-02 (0x${MOQL_VERSION.toString(16)})`); + if (version !== MOQL_VERSION) { + log(`PROTOCOL MISMATCH! Browser speaks moq-lite-02 but relay speaks ${versionName}. Things will likely break.`, "error"); + } + } + // Keep setup stream open — don't close it + setupWriter.releaseLock(); + + connected = true; + const totalMs = (performance.now() - t0).toFixed(0); + setStatus("Connected", "connected"); + log(`Connected to relay (total setup took ${totalMs}ms)`); + + // 6. Start listening for incoming streams FIRST + // (relay may send ANNOUNCE_PLEASE and SUBSCRIBE before we subscribe) + log("Listening for incoming bidi + uni streams..."); + handleIncomingBidiStreams(); + receiveUniStreams(); + + // 7. Init audio playback (needs user gesture — we're inside a click handler) + initAudioPlayback(); + log("Audio playback initialized (24kHz)"); + + // 8. Start mic (so we're ready to publish when relay subscribes) + await startMic(); + + // 9. Subscribe to bot's audio + transcript tracks (non-fatal if bot + // isn't connected yet) + await subscribeToAudio(); + subscribeToTranscript(); + + } catch (e) { + const elapsed = (performance.now() - t0).toFixed(0); + log(`Connection failed after ${elapsed}ms: ${e.message}`, "error"); + if (e.stack) log(` Stack: ${e.stack.split("\n").slice(1, 3).join(" | ")}`, "error"); + if (e.stack) log(` Stack: ${e.stack}`); + setStatus("Error: " + e.message, "error"); + } +} + +async function subscribeToAudio(retryCount = 0) { + const botBroadcast = `${config.namespace}/${config.bot_id}`; + const botTrack = config.subscribe_track; + const fullPath = `${botBroadcast}/${botTrack}`; + const maxRetries = 10; + const retryDelay = 2000; // 2 seconds between retries + + subAudioSubscribeId = nextSubscribeId++; + + try { + // Open a new bidi stream for SUBSCRIBE + log(`Subscribing to ${fullPath} (sub_id=${subAudioSubscribeId}, attempt ${retryCount + 1}/${maxRetries + 1})`); + const subBidi = await transport.createBidirectionalStream(); + const subWriter = subBidi.writable.getWriter(); + const subReader = subBidi.readable.getReader(); + + const subMsg = encodeSubscribe(subAudioSubscribeId, botBroadcast, botTrack); + log(`Sending SUBSCRIBE: [${hexdump(subMsg)}]`); + await subWriter.write(subMsg); + + // Read SUBSCRIBE_OK (or handle RESET_STREAM gracefully) + log("Waiting for SUBSCRIBE_OK from relay..."); + const { value: okResp, done } = await subReader.read(); + if (okResp) { + const okData = new Uint8Array(okResp); + log(`Received SUBSCRIBE_OK for ${fullPath}: [${hexdump(okData)}]`); + return; // Success + } else if (done) { + log(`SUBSCRIBE stream closed by relay without response (track "${fullPath}" may not exist — bot not connected yet?)`, "warn"); + } + + subWriter.releaseLock(); + } catch (e) { + // RESET_STREAM means the track doesn't exist yet — not fatal + log(`Subscribe to ${fullPath} failed: ${e.message}`, "warn"); + log(` (This is normal if the bot hasn't connected to the relay yet)`, "warn"); + } + + // Retry if bot isn't connected yet + if (connected && retryCount < maxRetries) { + log(`Will retry subscribe in ${retryDelay / 1000}s (attempt ${retryCount + 1}/${maxRetries})...`); + setTimeout(() => subscribeToAudio(retryCount + 1), retryDelay); + } else if (retryCount >= maxRetries) { + log(`Gave up subscribing to ${fullPath} after ${maxRetries} attempts`, "error"); + } +} + +async function subscribeToTranscript() { + const trackName = config.transcript_track || "transcript"; + const botBroadcast = `${config.namespace}/${config.bot_id}`; + const fullPath = `${botBroadcast}/${trackName}`; + subTranscriptSubscribeId = nextSubscribeId++; + try { + log(`Subscribing to ${fullPath} (sub_id=${subTranscriptSubscribeId})`); + const subBidi = await transport.createBidirectionalStream(); + const subWriter = subBidi.writable.getWriter(); + const subReader = subBidi.readable.getReader(); + const subMsg = encodeSubscribe(subTranscriptSubscribeId, botBroadcast, trackName); + await subWriter.write(subMsg); + subWriter.releaseLock(); + // Keep the reader open so the relay can stream SUBSCRIBE_OK / errors. + subReader.read().then(({ value, done }) => { + if (value && value.byteLength) { + log(`Transcript SUBSCRIBE_OK: [${hexdump(new Uint8Array(value))}]`); + } else if (done) { + log(`Transcript subscribe stream closed by relay (track may not exist)`, "warn"); + } + }).catch(() => {}); + } catch (e) { + log(`Subscribe to ${fullPath} failed: ${e.message}`, "warn"); + } +} + +async function doDisconnect() { + log(`Disconnecting... (sent=${audioSendCount} chunks, received=${audioRecvCount} chunks, errors=${audioSendErrors})`); + connected = false; + stopMic(); + + if (transport) { + try { + transport.close(); + log("WebTransport closed"); + } catch (e) { + log(`Error closing transport: ${e.message}`, "warn"); + } + transport = null; + } + + pubSubscribeId = null; + pubGroupSeq = 0; + subAudioSubscribeId = null; + subTranscriptSubscribeId = null; + nextSubscribeId = 1; + audioSendCount = 0; + audioSendErrors = 0; + audioRecvCount = 0; + incomingBidiCount = 0; + + if (audioContext) { + audioContext.close(); + audioContext = null; + } + + setStatus("Disconnected", "disconnected"); + log("Disconnected"); + + document.getElementById("connectBtn").disabled = false; + document.getElementById("disconnectBtn").disabled = true; +} + +// --------------------------------------------------------------------------- +// Button handlers +// --------------------------------------------------------------------------- + +document.getElementById("connectBtn").addEventListener("click", async () => { + document.getElementById("connectBtn").disabled = true; + document.getElementById("disconnectBtn").disabled = false; + await doConnect(); +}); + +document.getElementById("disconnectBtn").addEventListener("click", async () => { + await doDisconnect(); +}); diff --git a/moq_prebuilt/client/index.html b/moq_prebuilt/client/index.html new file mode 100644 index 000000000..2f6947bb3 --- /dev/null +++ b/moq_prebuilt/client/index.html @@ -0,0 +1,36 @@ + + + + + + Pipecat MOQ Client + + + +
+

Pipecat MOQ Client

+ +
+

Connection

+
Disconnected
+
+ + +
+
+ +
+

Transcript

+
+
+ +
+

Log

+
+
+
+ + + + + diff --git a/moq_prebuilt/client/moq-client.js b/moq_prebuilt/client/moq-client.js new file mode 100644 index 000000000..527a34786 --- /dev/null +++ b/moq_prebuilt/client/moq-client.js @@ -0,0 +1,324 @@ +/* + * Copyright (c) 2024-2026, Daily + * + * SPDX-License-Identifier: BSD 2-Clause License + */ + +/** + * MOQ (Media over QUIC) protocol client — moq-lite-02. + * + * Implements varint codec, message encode/decode for moq-lite-02 protocol + * so the browser can talk to a moq-lite relay over WebTransport. + * + * Key differences from draft-07: + * - Stream-per-request model (no shared control stream) + * - Setup uses u8(0x20/0x21) framing; via WebTransport uses u16 body size + * - SUBSCRIBE/ANNOUNCE each get their own bidi stream + * - Media data flows on uni streams as GROUP + FRAME + * - No QUIC datagrams + */ +(function () { + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const MOQL_VERSION = 0xff0dad02; // moq-lite-02 + +const Role = Object.freeze({ + PUBLISHER: 0x01, + SUBSCRIBER: 0x02, + PUBSUB: 0x03, +}); + +// Stream type varints (first thing written on a new bidi stream) +const StreamType = Object.freeze({ + SESSION: 0, + ANNOUNCE: 1, + SUBSCRIBE: 2, +}); + +// Setup message type bytes +const CLIENT_SETUP_TYPE = 0x20; +const SERVER_SETUP_TYPE = 0x21; + +// Uni stream type +const UNI_STREAM_TYPE_GROUP = 0; + +// --------------------------------------------------------------------------- +// QUIC variable-length integer codec +// --------------------------------------------------------------------------- + +function encodeVarint(value) { + if (value < 0x40) { + return new Uint8Array([value]); + } else if (value < 0x4000) { + const buf = new Uint8Array(2); + new DataView(buf.buffer).setUint16(0, value | 0x4000); + return buf; + } else if (value < 0x40000000) { + const buf = new Uint8Array(4); + new DataView(buf.buffer).setUint32(0, (value | 0x80000000) >>> 0); + return buf; + } else { + const buf = new Uint8Array(8); + const dv = new DataView(buf.buffer); + const hi = Math.floor(value / 0x100000000) | 0xc0000000; + const lo = value >>> 0; + dv.setUint32(0, hi >>> 0); + dv.setUint32(4, lo); + return buf; + } +} + +function decodeVarint(data, offset) { + const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); + const first = data[offset]; + const lengthBits = first >> 6; + + if (lengthBits === 0) { + return [first, offset + 1]; + } else if (lengthBits === 1) { + return [dv.getUint16(offset) & 0x3fff, offset + 2]; + } else if (lengthBits === 2) { + return [dv.getUint32(offset) & 0x3fffffff, offset + 4]; + } else { + const hi = dv.getUint32(offset) & 0x3fffffff; + const lo = dv.getUint32(offset + 4); + return [hi * 0x100000000 + lo, offset + 8]; + } +} + +// --------------------------------------------------------------------------- +// String helpers +// --------------------------------------------------------------------------- + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +function encodeString(str) { + const bytes = encoder.encode(str); + return concat(encodeVarint(bytes.length), bytes); +} + +function decodeString(data, offset) { + const [len, off] = decodeVarint(data, offset); + const str = decoder.decode(data.subarray(off, off + len)); + return [str, off + len]; +} + +// --------------------------------------------------------------------------- +// Buffer helpers +// --------------------------------------------------------------------------- + +function concat(...arrays) { + let total = 0; + for (const a of arrays) total += a.byteLength; + const result = new Uint8Array(total); + let pos = 0; + for (const a of arrays) { + result.set(a instanceof Uint8Array ? a : new Uint8Array(a), pos); + pos += a.byteLength; + } + return result; +} + +// --------------------------------------------------------------------------- +// Setup messages (WebTransport uses u16 body size prefix) +// --------------------------------------------------------------------------- + +/** + * Encode CLIENT_SETUP for WebTransport (browser). + * + * WebTransport framing: varint(0x20) as stream type, then u16(body_len) + body + * on a dedicated bidi stream. + */ +function encodeClientSetup(role, versions, path) { + let body = encodeVarint(versions.length); + for (const v of versions) { + body = concat(body, encodeVarint(v)); + } + + // Parameters + let numParams = 1; // role + if (path) numParams += 1; + body = concat(body, encodeVarint(numParams)); + + // Role param (key=0) + body = concat(body, encodeVarint(0), encodeVarint(1), encodeVarint(role)); + + // Path param (key=1) + if (path) { + const pathBytes = encoder.encode(path); + body = concat(body, encodeVarint(1), encodeVarint(pathBytes.length), pathBytes); + } + + // WebTransport framing: u16(body_len) + body + const frame = new Uint8Array(2 + body.length); + new DataView(frame.buffer).setUint16(0, body.length); + frame.set(body, 2); + + return frame; +} + +/** + * Parse SERVER_SETUP from WebTransport. + * + * WebTransport framing: u16(body_len) + body + * Body: varint(selected_version) + params... + */ +function parseServerSetup(data) { + const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); + const bodyLen = dv.getUint16(0); + let offset = 2; + const [version, newOff] = decodeVarint(data, offset); + return { version, bodyEnd: 2 + bodyLen }; +} + +// --------------------------------------------------------------------------- +// Subscribe messages (on dedicated bidi stream with stream_type=2) +// --------------------------------------------------------------------------- + +/** + * Encode SUBSCRIBE message body. + * + * Stream format: varint(2) + varint(body_len) + body + * Body: varint(sub_id) + string(broadcast_path) + string(track_name) + u8(priority) + */ +function encodeSubscribe(subscribeId, broadcastPath, trackName, priority = 128) { + let body = concat( + encodeVarint(subscribeId), + encodeString(broadcastPath), + encodeString(trackName), + new Uint8Array([priority]), + ); + + // Stream type + length-prefixed body + return concat( + encodeVarint(StreamType.SUBSCRIBE), + encodeVarint(body.length), + body, + ); +} + +/** + * Encode SUBSCRIBE_OK response. + * Format: varint(0) — empty body. + */ +function encodeSubscribeOk() { + return encodeVarint(0); +} + +/** + * Decode a SUBSCRIBE message from an incoming bidi stream. + * Input starts after the stream type varint has been consumed. + */ +function decodeSubscribe(data, offset = 0) { + let bodyLen; + [bodyLen, offset] = decodeVarint(data, offset); + const bodyEnd = offset + bodyLen; + + let subscribeId, broadcastPath, trackName; + [subscribeId, offset] = decodeVarint(data, offset); + [broadcastPath, offset] = decodeString(data, offset); + [trackName, offset] = decodeString(data, offset); + const priority = data[offset]; + offset += 1; + + return { subscribeId, broadcastPath, trackName, priority, end: bodyEnd }; +} + +// --------------------------------------------------------------------------- +// GROUP + FRAME messages (on unidirectional streams) +// --------------------------------------------------------------------------- + +/** + * Encode GROUP header + single FRAME for a uni stream. + * + * Format: u8(0) + varint(header_body_len) + varint(subscribe_id) + varint(group_seq) + * + varint(payload_len) + payload + */ +function encodeGroupAndFrame(subscribeId, groupSeq, payload) { + const headerBody = concat( + encodeVarint(subscribeId), + encodeVarint(groupSeq), + ); + + const frame = concat( + encodeVarint(payload.byteLength), + payload instanceof Uint8Array ? payload : new Uint8Array(payload), + ); + + return concat( + new Uint8Array([UNI_STREAM_TYPE_GROUP]), + encodeVarint(headerBody.length), + headerBody, + frame, + ); +} + +/** + * Parse GROUP header + FRAMEs from a complete uni stream buffer. + * + * Returns { subscribeId, groupSeq, frames: [Uint8Array, ...] } + */ +function parseGroupStream(data) { + let offset = 0; + + // u8(0) stream type + const streamType = data[offset]; + offset += 1; + + // varint(body_len) + let bodyLen; + [bodyLen, offset] = decodeVarint(data, offset); + const bodyStart = offset; + + // varint(subscribe_id) + let subscribeId; + [subscribeId, offset] = decodeVarint(data, offset); + + // varint(group_seq) + let groupSeq; + [groupSeq, offset] = decodeVarint(data, offset); + + // Advance past header body + offset = bodyStart + bodyLen; + + // Parse frames: varint(payload_len) + payload, repeated + const frames = []; + while (offset < data.length) { + let payloadLen; + [payloadLen, offset] = decodeVarint(data, offset); + frames.push(data.subarray(offset, offset + payloadLen)); + offset += payloadLen; + } + + return { subscribeId, groupSeq, frames }; +} + +// --------------------------------------------------------------------------- +// Exports (global for vanilla JS) +// --------------------------------------------------------------------------- + +window.MOQ = { + MOQL_VERSION, + Role, + StreamType, + CLIENT_SETUP_TYPE, + SERVER_SETUP_TYPE, + encodeVarint, + decodeVarint, + encodeString, + decodeString, + concat, + encodeClientSetup, + parseServerSetup, + encodeSubscribe, + encodeSubscribeOk, + decodeSubscribe, + encodeGroupAndFrame, + parseGroupStream, +}; + +})(); diff --git a/moq_prebuilt/client/style.css b/moq_prebuilt/client/style.css new file mode 100644 index 000000000..801adae14 --- /dev/null +++ b/moq_prebuilt/client/style.css @@ -0,0 +1,242 @@ +/* Palette borrowed from @pipecat-ai/voice-ui-kit (dark theme) so this + custom MOQ client visually matches the smallwebrtc playground. */ +:root { + --background: oklch(0.141 0.005 285.823); + --foreground: oklch(0.985 0 0); + --card: oklch(0.21 0.006 285.885); + --muted: oklch(0.274 0.006 286.033); + --muted-foreground: oklch(0.705 0.015 286.067); + --border: oklch(1 0 0 / 10%); + --border-strong: oklch(1 0 0 / 16%); + --primary: oklch(0.85 0.13 162); /* green Connect */ + --primary-hover: oklch(0.78 0.14 162); + --destructive: oklch(0.577 0.245 27.325); /* red Disconnect */ + --destructive-hover: oklch(0.52 0.225 27.325); + --warning: oklch(0.795 0.184 86.047); + --user-bubble: oklch(0.305 0.045 265); + --assistant-bubble: oklch(0.255 0.012 286); + --radius-md: 6px; + --radius-lg: 8px; + --radius-xl: 12px; + --font-sans: -apple-system, BlinkMacSystemFont, "Inter", "Segoe UI", Roboto, sans-serif; + --font-mono: "SF Mono", "JetBrains Mono", "Fira Code", "Cascadia Code", Consolas, monospace; +} + +* { + box-sizing: border-box; + margin: 0; + padding: 0; +} + +html, body { + height: 100%; +} + +body { + font-family: var(--font-sans); + background: var(--background); + color: var(--foreground); + font-size: 14px; + line-height: 1.5; + min-height: 100vh; + display: flex; + justify-content: center; + padding: 48px 24px 80px; + -webkit-font-smoothing: antialiased; +} + +.container { + max-width: 720px; + width: 100%; + display: flex; + flex-direction: column; + gap: 16px; +} + +h1 { + font-size: 1.05rem; + font-weight: 600; + letter-spacing: -0.005em; + color: var(--foreground); + margin: 0 0 8px 4px; +} + +/* ---------- Card ---------- */ + +.card { + background: var(--card); + border: 1px solid var(--border); + border-radius: var(--radius-xl); + padding: 20px; +} + +.card h3 { + font-family: var(--font-mono); + font-size: 0.68rem; + font-weight: 600; + letter-spacing: 0.12em; + text-transform: uppercase; + color: var(--muted-foreground); + margin-bottom: 14px; +} + +/* ---------- Connection card (status + buttons) ---------- */ + +.status { + font-size: 1rem; + font-weight: 500; + padding: 4px 0 14px; + color: var(--muted-foreground); + display: flex; + align-items: center; + gap: 10px; +} + +.status::before { + content: ""; + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--muted-foreground); + flex-shrink: 0; +} + +.status.disconnected { color: var(--muted-foreground); } +.status.disconnected::before { background: var(--muted-foreground); } +.status.connecting { color: var(--warning); } +.status.connecting::before { background: var(--warning); } +.status.connected { color: var(--primary); } +.status.connected::before { background: var(--primary); box-shadow: 0 0 0 4px oklch(0.85 0.13 162 / 0.15); } +.status.error { color: var(--destructive); } +.status.error::before { background: var(--destructive); } + +.controls { + display: flex; + gap: 8px; +} + +button { + font-family: inherit; + font-size: 0.85rem; + font-weight: 500; + letter-spacing: 0.01em; + padding: 8px 18px; + border: 1px solid transparent; + border-radius: var(--radius-md); + cursor: pointer; + transition: background 0.15s, border-color 0.15s, opacity 0.15s; +} + +button:disabled { + opacity: 0.35; + cursor: not-allowed; +} + +#connectBtn { + background: var(--primary); + color: oklch(0.16 0.02 162); +} + +#connectBtn:hover:not(:disabled) { + background: var(--primary-hover); +} + +#disconnectBtn { + background: transparent; + border-color: var(--destructive); + color: var(--destructive); +} + +#disconnectBtn:hover:not(:disabled) { + background: oklch(0.577 0.245 27.325 / 0.15); +} + +/* ---------- Transcript ---------- */ + +.transcript { + font-size: 0.95rem; + line-height: 1.55; + max-height: 360px; + min-height: 48px; + overflow-y: auto; + display: flex; + flex-direction: column; + gap: 8px; + padding-right: 4px; +} + +.transcript:empty::before { + content: "Waiting for conversation…"; + color: var(--muted-foreground); + font-style: italic; + opacity: 0.6; +} + +.transcript-row { + display: flex; + flex-direction: column; + gap: 3px; + padding: 10px 14px; + border-radius: var(--radius-lg); + border: 1px solid var(--border); + max-width: 88%; + align-self: flex-start; + background: var(--assistant-bubble); +} + +.transcript-user { + background: var(--user-bubble); +} + +.transcript-role { + font-family: var(--font-mono); + font-size: 0.62rem; + font-weight: 600; + letter-spacing: 0.12em; + text-transform: uppercase; + color: var(--muted-foreground); +} + +.transcript-text { + color: var(--foreground); + white-space: pre-wrap; + word-wrap: break-word; +} + +/* ---------- Log ---------- */ + +.log { + font-family: var(--font-mono); + font-size: 0.72rem; + line-height: 1.7; + max-height: 360px; + overflow-y: auto; + color: var(--muted-foreground); +} + +.log div { + padding: 2px 0; + border-bottom: 1px solid oklch(1 0 0 / 0.04); +} + +.log div:last-child { + border-bottom: none; +} + +/* ---------- Scrollbar polish ---------- */ + +.transcript::-webkit-scrollbar, +.log::-webkit-scrollbar { + width: 6px; +} + +.transcript::-webkit-scrollbar-thumb, +.log::-webkit-scrollbar-thumb { + background: var(--border-strong); + border-radius: 3px; +} + +.transcript::-webkit-scrollbar-track, +.log::-webkit-scrollbar-track { + background: transparent; +} diff --git a/moq_prebuilt/frontend.py b/moq_prebuilt/frontend.py new file mode 100644 index 000000000..57d07f5af --- /dev/null +++ b/moq_prebuilt/frontend.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import logging +import os + +from fastapi.staticfiles import StaticFiles + +# Define possible paths to the client directory +base_dir = os.path.dirname(__file__) +possible_client_paths = [ + os.path.abspath(os.path.join(base_dir, "client")), # in package + os.path.abspath(os.path.join(base_dir, "..", "client")), # in dev +] + +client_dir = None + +for path in possible_client_paths: + logging.info(f"Checking MOQ client directory: {path}") + if os.path.isdir(path): + client_dir = path + break + +if not client_dir: + logging.error("MOQ prebuilt client not found in any of the expected locations.") + raise RuntimeError("MOQ prebuilt client not found.") + +MOQPrebuiltUI = StaticFiles(directory=client_dir, html=True) diff --git a/pyproject.toml b/pyproject.toml index 2f3de7ea9..4ee942cf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,7 @@ mem0 = [ "mem0ai>=1.0.8,<2" ] mistral = ["mistralai>=2.0.0,<3"] mlx-whisper = [ "mlx-whisper~=0.4.2" ] moondream = [ "accelerate~=1.10.0", "einops~=0.8.0", "pyvips[binary]~=3.0.0", "timm~=1.0.13", "transformers>=4.48.0,<6" ] +moq = [ "aioquic>=1.2.0,<2", "cryptography>=43.0.0" ] nebius = [] neuphonic = [ "pipecat-ai[websockets-base]" ] novita = [] diff --git a/scripts/moq-dev-setup.sh b/scripts/moq-dev-setup.sh new file mode 100755 index 000000000..313491f0a --- /dev/null +++ b/scripts/moq-dev-setup.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# +# Generate a self-signed cert for local MOQ dev, symlink it into both the +# relay repo and this pipecat repo, and print the commands to run everything. +# +# Usage: +# ./scripts/moq-dev-setup.sh /path/to/moq-relay +# + +set -euo pipefail + +RELAY_DIR="${1:?Usage: $0 /path/to/moq-relay}" +PIPECAT_DIR="$(cd "$(dirname "$0")/.." && pwd)" + +# Resolve relay dir to absolute path +RELAY_DIR="$(cd "$RELAY_DIR" && pwd)" + +CERT_DIR="$PIPECAT_DIR/.moq-certs" +CERT_FILE="$CERT_DIR/moq-cert.pem" +KEY_FILE="$CERT_DIR/moq-key.pem" + +echo "==> Relay dir: $RELAY_DIR" +echo "==> Pipecat dir: $PIPECAT_DIR" +echo "==> Cert dir: $CERT_DIR" +echo + +# --------------------------------------------------------------------------- +# 1. Generate cert + key +# --------------------------------------------------------------------------- +mkdir -p "$CERT_DIR" + +echo "==> Generating self-signed cert (valid 14 days — MOQ max)..." +openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 \ + -keyout "$KEY_FILE" \ + -out "$CERT_FILE" \ + -days 14 \ + -nodes \ + -subj "/CN=localhost" \ + -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" \ + 2>/dev/null + +echo " $CERT_FILE" +echo " $KEY_FILE" +echo + +# --------------------------------------------------------------------------- +# 2. Compute SHA-256 fingerprint (used by WebTransport cert pinning) +# --------------------------------------------------------------------------- +FINGERPRINT=$(openssl x509 -in "$CERT_FILE" -outform der \ + | openssl dgst -sha256 -binary \ + | base64) + +echo "==> Certificate SHA-256 fingerprint:" +echo " $FINGERPRINT" +echo + +# --------------------------------------------------------------------------- +# 3. Symlink into relay dir and pipecat dir +# --------------------------------------------------------------------------- +echo "==> Symlinking certs..." + +for DIR in "$RELAY_DIR" "$PIPECAT_DIR"; do + for FILE in "$CERT_FILE" "$KEY_FILE"; do + BASENAME="$(basename "$FILE")" + TARGET="$DIR/$BASENAME" + rm -f "$TARGET" + ln -s "$FILE" "$TARGET" + echo " $TARGET -> $FILE" + done +done +echo + +# --------------------------------------------------------------------------- +# 4. Print run commands +# --------------------------------------------------------------------------- +echo "===========================================" +echo " Run these in separate terminals:" +echo "===========================================" +echo +echo "# 1. Start the relay (binds QUIC/WebTransport on UDP [::]:4080)" +echo "cd $RELAY_DIR" +echo "cargo run --bin moq-relay -- \\" +echo " --server-bind '[::]:4080' \\" +echo " --tls-cert moq-cert.pem \\" +echo " --tls-key moq-key.pem \\" +echo " --auth-public ''" +echo +echo "# 2. Start the bot" +echo "cd $PIPECAT_DIR" +echo "uv run python examples/transports/transports-moq.py \\" +echo " -t moq \\" +echo " --moq-cert moq-cert.pem \\" +echo " --moq-insecure \\" +echo " --moq-path /" +echo +echo "# 3. Open browser" +echo "open http://localhost:7860" +echo diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py index f3965b79c..896256f43 100644 --- a/src/pipecat/runner/run.py +++ b/src/pipecat/runner/run.py @@ -53,6 +53,7 @@ Supported transports: - Daily - Creates rooms and tokens, runs bot as participant - WebRTC - Provides local WebRTC interface with prebuilt UI +- MOQ - Media over QUIC, connects to a MOQ relay for pub/sub streaming - Telephony - Handles webhook and WebSocket connections for Twilio, Telnyx, Plivo, Exotel To run locally: @@ -61,6 +62,7 @@ To run locally: - ESP32: `python bot.py -t webrtc --esp32 --host 192.168.1.100` - Daily (server): `python bot.py -t daily` - Daily (direct, testing only): `python bot.py -d` +- MOQ: `python bot.py -t moq --moq-host relay.example.com --moq-insecure` - Telephony: `python bot.py -t twilio -x your_username.ngrok.io` - Exotel: `python bot.py -t exotel` (no proxy needed, but ngrok connection to HTTP 7860 is required) """ @@ -83,6 +85,7 @@ from loguru import logger from pipecat.runner.types import ( DailyRunnerArguments, + MOQRunnerArguments, RunnerArguments, SmallWebRTCRunnerArguments, WebSocketRunnerArguments, @@ -93,7 +96,7 @@ try: from dotenv import load_dotenv from fastapi import BackgroundTasks, FastAPI, Header, HTTPException, Request, WebSocket from fastapi.middleware.cors import CORSMiddleware - from fastapi.responses import HTMLResponse, RedirectResponse + from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse except ImportError as e: logger.error(f"Runner dependencies not available: {e}") logger.error("To use Pipecat runners, install with: pip install pipecat-ai[runner]") @@ -203,6 +206,8 @@ def _configure_server_app(args: argparse.Namespace): _setup_whatsapp_routes(app, args) elif args.transport == "daily": _setup_daily_routes(app, args) + elif args.transport == "moq": + _setup_moq_routes(app, args) elif args.transport in TELEPHONY_TRANSPORTS: _setup_telephony_routes(app, args) else: @@ -800,6 +805,131 @@ def _setup_daily_routes(app: FastAPI, args: argparse.Namespace): } + +def _setup_moq_routes(app: FastAPI, args: argparse.Namespace): + """Set up MOQ (Media over QUIC) specific routes.""" + MOQPrebuiltUI = None + try: + from moq_prebuilt.frontend import MOQPrebuiltUI + except ImportError: + # Fallback: look for moq_prebuilt relative to the pipecat repo root + try: + import pipecat + + repo_root = os.path.dirname(os.path.dirname(os.path.dirname(pipecat.__file__))) + client_dir = os.path.join(repo_root, "moq_prebuilt", "client") + if os.path.isdir(client_dir): + from starlette.staticfiles import StaticFiles + + MOQPrebuiltUI = StaticFiles(directory=client_dir, html=True) + except Exception: + pass + if not MOQPrebuiltUI: + logger.warning("moq_prebuilt client not found, MOQ client UI will not be available") + + # Track active MOQ sessions + moq_sessions: Dict[str, Any] = {} + + # Mount the frontend + if MOQPrebuiltUI: + app.mount("/client", MOQPrebuiltUI) + + @app.get("/", include_in_schema=False) + async def root_redirect(): + """Redirect root requests to client interface.""" + return RedirectResponse(url="/client/") + + @app.get("/api/config") + async def moq_config(): + """Return MOQ relay connection config for the browser client.""" + cert_hash = None + if getattr(args, "moq_cert", None): + try: + import base64 + import hashlib + + from cryptography import x509 + from cryptography.hazmat.primitives import serialization + + with open(args.moq_cert, "rb") as f: + cert = x509.load_pem_x509_certificate(f.read()) + der_bytes = cert.public_bytes(serialization.Encoding.DER) + digest = hashlib.sha256(der_bytes).digest() + cert_hash = base64.b64encode(digest).decode() + except Exception as e: + logger.warning(f"Could not compute cert fingerprint: {e}") + + return { + "relay_host": args.moq_host, + "relay_port": args.moq_port, + "path": args.moq_path, + "namespace": args.moq_namespace, + "insecure": args.moq_insecure, + "cert_hash": cert_hash, + # Per-participant broadcast paths. Browser publishes its mic + # under // and subscribes + # to the bot under //. + "client_id": args.moq_client_id, + "bot_id": args.moq_bot_id, + "publish_track": "user-audio", + "subscribe_track": "bot-audio", + "transcript_track": "transcript", + } + + @app.post("/start") + async def start_moq_bot(request: Request): + """Start a MOQ bot session.""" + try: + request_data = await request.json() + logger.debug(f"Received MOQ start request: {request_data}") + except Exception: + request_data = {} + + body = request_data.get("body", {}) + namespace = request_data.get("namespace", args.moq_namespace) + + bot_module = _get_bot_module() + + ready_event = asyncio.Event() + runner_args = MOQRunnerArguments( + host=args.moq_host, + port=args.moq_port, + path=args.moq_path, + namespace=namespace, + participant_id=args.moq_bot_id, + peer_id=args.moq_client_id, + verify_ssl=not args.moq_insecure, + body=body, + ready_event=ready_event, + ) + runner_args.cli_args = args + + session_id = str(uuid.uuid4()) + moq_sessions[session_id] = { + "namespace": namespace, + "host": args.moq_host, + "port": args.moq_port, + } + + # Spawn the bot and wait until it signals it has finished the MOQ + # handshake, so the browser's SUBSCRIBE arrives at a publisher the + # relay already knows about. + asyncio.create_task(bot_module.bot(runner_args)) + try: + await asyncio.wait_for(ready_event.wait(), timeout=15.0) + except asyncio.TimeoutError: + return JSONResponse( + status_code=504, + content={"error": "Bot did not connect to MOQ relay within 15s"}, + ) + + return { + "sessionId": session_id, + "namespace": namespace, + "relay": f"{args.moq_host}:{args.moq_port}", + } + + def _setup_telephony_routes(app: FastAPI, args: argparse.Namespace): """Set up telephony-specific routes.""" # XML response templates (Exotel doesn't use XML webhooks) @@ -957,7 +1087,7 @@ def main(parser: argparse.ArgumentParser | None = None): "-t", "--transport", type=str, - choices=["daily", "webrtc", *TELEPHONY_TRANSPORTS], + choices=["daily", "webrtc", "moq", *TELEPHONY_TRANSPORTS], default="webrtc", help="Transport type", ) @@ -992,6 +1122,62 @@ def main(parser: argparse.ArgumentParser | None = None): help="Ensure requried WhatsApp environment variables are present", ) + # MOQ-specific arguments + parser.add_argument( + "--moq-host", + type=str, + default="localhost", + help="MOQ relay host address (default: localhost)", + ) + parser.add_argument( + "--moq-port", + type=int, + default=4080, + help="MOQ relay port (default: 4080)", + ) + parser.add_argument( + "--moq-path", + type=str, + default="/moq", + help="MOQ endpoint path (default: /moq)", + ) + parser.add_argument( + "--moq-namespace", + type=str, + default="pipecat", + help="MOQ namespace/room (default: pipecat)", + ) + parser.add_argument( + "--moq-bot-id", + type=str, + default="bot0", + help="This bot's participant id; broadcasts under / (default: bot0)", + ) + parser.add_argument( + "--moq-client-id", + type=str, + default="client0", + help="Peer client's participant id the bot subscribes to (default: client0)", + ) + parser.add_argument( + "--moq-insecure", + action="store_true", + default=False, + help="Disable SSL certificate verification for MOQ relay", + ) + parser.add_argument( + "--moq-cert", + type=str, + default=None, + help="Path to relay TLS certificate (PEM) for WebTransport cert pinning", + ) + parser.add_argument( + "--moq-web-port", + type=int, + default=None, + help="MOQ relay WebTransport port for browser clients (defaults to --moq-port)", + ) + args = parser.parse_args() # Validate and clean proxy hostname @@ -1051,6 +1237,15 @@ def main(parser: argparse.ArgumentParser | None = None): else: print(f" → Open http://{args.host}:{args.port} in your browser to start a session") print() + elif args.transport == "moq": + print() + print(f"🚀 Bot ready! (MOQ)") + print(f" → Connecting to MOQ relay at {args.moq_host}:{args.moq_port}") + print(f" → Namespace: {args.moq_namespace}") + print(f" → Status page: http://{args.host}:{args.port}") + print() + print(f" Connect a MOQ client to the same relay and namespace to start talking.") + print() RUNNER_DOWNLOADS_FOLDER = args.folder RUNNER_HOST = args.host diff --git a/src/pipecat/runner/types.py b/src/pipecat/runner/types.py index 70ce03f29..5bb38b59f 100644 --- a/src/pipecat/runner/types.py +++ b/src/pipecat/runner/types.py @@ -11,8 +11,9 @@ information to bot functions. """ import argparse +import asyncio from dataclasses import dataclass, field -from typing import Any +from typing import Any, Optional from fastapi import WebSocket from pydantic import BaseModel @@ -135,3 +136,29 @@ class LiveKitRunnerArguments(RunnerArguments): room_name: str url: str token: str + + +@dataclass +class MOQRunnerArguments(RunnerArguments): + """MOQ (Media over QUIC) transport session arguments for the runner. + + Parameters: + host: MOQ relay server hostname. + port: MOQ relay server port. + path: MOQ endpoint path on the relay. + namespace: MOQ namespace (like a room identifier). + verify_ssl: Whether to verify SSL certificates. + ready_event: Optional event the bot sets once it has connected to the + relay and finished the MOQ handshake. Lets the HTTP `/start` + endpoint block until the bot is reachable before telling the + browser to open its WebTransport. + """ + + host: str + port: int + path: str = "/moq" + namespace: str = "pipecat" + participant_id: str = "bot0" + peer_id: str = "client0" + verify_ssl: bool = True + ready_event: Optional[asyncio.Event] = field(default=None, kw_only=True) diff --git a/src/pipecat/runner/utils.py b/src/pipecat/runner/utils.py index 4715be6f3..fed53e8de 100644 --- a/src/pipecat/runner/utils.py +++ b/src/pipecat/runner/utils.py @@ -41,6 +41,7 @@ from loguru import logger from pipecat.runner.types import ( DailyRunnerArguments, LiveKitRunnerArguments, + MOQRunnerArguments, SmallWebRTCRunnerArguments, WebSocketRunnerArguments, ) @@ -582,5 +583,25 @@ async def create_transport( params=params, ) + elif isinstance(runner_args, MOQRunnerArguments): + params = _get_transport_params("moq", transport_params) + + from pipecat.transports.moq.transport import MOQParams, MOQTransport + + # Convert TransportParams to MOQParams if needed, applying runner args + if not isinstance(params, MOQParams): + params = MOQParams(**params.model_dump()) + params.verify_ssl = runner_args.verify_ssl + params.namespace = runner_args.namespace + params.participant_id = runner_args.participant_id + params.peer_id = runner_args.peer_id + + return MOQTransport( + params=params, + host=runner_args.host, + port=runner_args.port, + path=runner_args.path, + ) + else: raise ValueError(f"Unsupported runner arguments type: {type(runner_args)}") diff --git a/src/pipecat/transports/moq/__init__.py b/src/pipecat/transports/moq/__init__.py new file mode 100644 index 000000000..f2d330c26 --- /dev/null +++ b/src/pipecat/transports/moq/__init__.py @@ -0,0 +1,26 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""MOQ (Media over QUIC) transport implementation for Pipecat. + +This module provides MOQ transport functionality for real-time media streaming +using the QUIC protocol, supporting low-latency audio and video transmission +with pub/sub semantics. +""" + +from pipecat.transports.moq.transport import ( + MOQInputTransport, + MOQOutputTransport, + MOQParams, + MOQTransport, +) + +__all__ = [ + "MOQInputTransport", + "MOQOutputTransport", + "MOQParams", + "MOQTransport", +] diff --git a/src/pipecat/transports/moq/protocol.py b/src/pipecat/transports/moq/protocol.py new file mode 100644 index 000000000..2c7495731 --- /dev/null +++ b/src/pipecat/transports/moq/protocol.py @@ -0,0 +1,546 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""MOQ (Media over QUIC) protocol implementation — moq-lite-02. + +This module implements the moq-lite-02 protocol layer for pub/sub media +streaming over QUIC. It provides message types, encoding/decoding utilities, +and session management for real-time media transmission. + +moq-lite-02 uses a stream-per-request model: each operation (setup, +subscribe, announce) opens its own bidirectional QUIC stream, and media +data flows on unidirectional streams as GROUP + FRAME sequences. + +Based on moq-lite-02 (version code 0xff0dad02). +""" + +import struct +from dataclasses import dataclass, field +from enum import IntEnum +from typing import Dict, List, Optional, Tuple + +# --------------------------------------------------------------------------- +# Protocol constants +# --------------------------------------------------------------------------- + +MOQL_VERSION = 0xFF0DAD02 # moq-lite-02 +MOQL_ALPN = "moql" + +# Stream type bytes (written as first varint on bidi streams) +STREAM_TYPE_SESSION = 0 +STREAM_TYPE_ANNOUNCE = 1 +STREAM_TYPE_SUBSCRIBE = 2 + +# Setup message type bytes (u8) +CLIENT_SETUP_TYPE = 0x20 +SERVER_SETUP_TYPE = 0x21 + +# Unidirectional stream type (u8) +UNI_STREAM_TYPE_GROUP = 0 + +# Announce update status +ANNOUNCE_STATUS_ACTIVE = 0 +ANNOUNCE_STATUS_ENDED = 1 + + +class MOQRole(IntEnum): + """MOQ endpoint roles.""" + + PUBLISHER = 0x01 + SUBSCRIBER = 0x02 + PUBSUB = 0x03 + + +class MOQTrackType(IntEnum): + """Types of media tracks in MOQ.""" + + AUDIO = 0x01 + VIDEO = 0x02 + DATA = 0x03 + + +@dataclass +class MOQTrack: + """Represents a MOQ media track. + + Parameters: + broadcast_path: The broadcast path (e.g., "pipecat"). + name: The track name (e.g., "audio" or "video"). + track_type: The type of media track. + priority: Track priority (lower is higher priority). + """ + + broadcast_path: str + name: str + track_type: MOQTrackType = MOQTrackType.DATA + priority: int = 128 + + @property + def full_name(self) -> str: + """Get the full track identifier.""" + return f"{self.broadcast_path}/{self.name}" + + +class MOQCodec: + """Encoder/decoder for MOQ wire protocol messages (moq-lite-02).""" + + @staticmethod + def encode_varint(value: int) -> bytes: + """Encode an integer as a QUIC variable-length integer. + + Args: + value: The integer to encode. + + Returns: + The encoded bytes. + """ + if value < 0x40: + return struct.pack("!B", value) + elif value < 0x4000: + return struct.pack("!H", value | 0x4000) + elif value < 0x40000000: + return struct.pack("!I", value | 0x80000000) + else: + return struct.pack("!Q", value | 0xC000000000000000) + + @staticmethod + def decode_varint(data: bytes, offset: int = 0) -> Tuple[int, int]: + """Decode a QUIC variable-length integer. + + Args: + data: The bytes to decode from. + offset: Starting offset in the data. + + Returns: + Tuple of (decoded value, new offset). + """ + first_byte = data[offset] + length_bits = first_byte >> 6 + + if length_bits == 0: + return first_byte, offset + 1 + elif length_bits == 1: + value = struct.unpack("!H", data[offset : offset + 2])[0] & 0x3FFF + return value, offset + 2 + elif length_bits == 2: + value = struct.unpack("!I", data[offset : offset + 4])[0] & 0x3FFFFFFF + return value, offset + 4 + else: + value = struct.unpack("!Q", data[offset : offset + 8])[0] & 0x3FFFFFFFFFFFFFFF + return value, offset + 8 + + @staticmethod + def encode_string(value: str) -> bytes: + """Encode a string with varint length prefix. + + Args: + value: The string to encode. + + Returns: + The encoded bytes. + """ + encoded = value.encode("utf-8") + return MOQCodec.encode_varint(len(encoded)) + encoded + + @staticmethod + def decode_string(data: bytes, offset: int = 0) -> Tuple[str, int]: + """Decode a length-prefixed string. + + Args: + data: The bytes to decode from. + offset: Starting offset in the data. + + Returns: + Tuple of (decoded string, new offset). + """ + length, offset = MOQCodec.decode_varint(data, offset) + value = data[offset : offset + length].decode("utf-8") + return value, offset + length + + # ------------------------------------------------------------------ + # Setup messages (on a dedicated bidi stream with stream_type=0) + # ------------------------------------------------------------------ + + @staticmethod + def encode_client_setup( + role: MOQRole, + supported_versions: List[int], + path: Optional[str] = None, + ) -> bytes: + """Encode a CLIENT_SETUP message for raw QUIC (ALPN "moql"). + + Format: u8(0x20) + varint(body_len) + body + Body: varint(num_versions) + versions... + varint(num_params) + params... + + Args: + role: The role this client is taking. + supported_versions: List of supported MOQ versions. + path: Optional connection path. + + Returns: + The encoded message (including stream type). + """ + body = MOQCodec.encode_varint(len(supported_versions)) + for version in supported_versions: + body += MOQCodec.encode_varint(version) + + # Parameters + num_params = 1 # role + if path: + num_params += 1 + + body += MOQCodec.encode_varint(num_params) + + # Role parameter (key=0) + body += MOQCodec.encode_varint(0) # Parameter key + body += MOQCodec.encode_varint(1) # Parameter length + body += MOQCodec.encode_varint(role) + + # Path parameter (key=1) if provided + if path: + body += MOQCodec.encode_varint(1) # Parameter key + path_bytes = path.encode("utf-8") + body += MOQCodec.encode_varint(len(path_bytes)) + body += path_bytes + + # Frame: u8(0x20) + u16(body_len) + body + # The relay uses Draft14 wire encoding for ALPN "moql", + # which expects a big-endian u16 for the body length. + msg = struct.pack("!B", CLIENT_SETUP_TYPE) + msg += struct.pack("!H", len(body)) + msg += body + + return msg + + @staticmethod + def decode_server_setup(data: bytes, offset: int = 0) -> Tuple[int, int]: + """Decode a SERVER_SETUP message. + + Format: u8(0x21) + u16(body_len) + body + Body: varint(selected_version) + params... + + The relay uses Draft14 wire encoding for ALPN "moql", + which uses big-endian u16 for the body length. + + Args: + data: The bytes to decode from. + offset: Starting offset in the data. + + Returns: + Tuple of (selected_version, new offset). + """ + # Skip type byte (0x21) + msg_type = data[offset] + offset += 1 + assert msg_type == SERVER_SETUP_TYPE, f"Expected SERVER_SETUP (0x21), got {msg_type:#x}" + + # Body length (u16 big-endian, Draft14 wire encoding) + body_len = struct.unpack("!H", data[offset : offset + 2])[0] + offset += 2 + body_start = offset + + # Selected version + version, offset = MOQCodec.decode_varint(data, offset) + + # Skip remaining params + offset = body_start + body_len + + return version, offset + + # ------------------------------------------------------------------ + # Subscribe messages (on a dedicated bidi stream with stream_type=2) + # ------------------------------------------------------------------ + + @staticmethod + def encode_subscribe( + subscribe_id: int, + broadcast_path: str, + track_name: str, + priority: int = 128, + ) -> bytes: + """Encode a SUBSCRIBE message body. + + Body format: varint(sub_id) + string(broadcast_path) + string(track_name) + u8(priority) + + The stream type varint(2) is written separately by the caller. + + Args: + subscribe_id: Unique subscription identifier. + broadcast_path: Broadcast path (namespace). + track_name: Name of the track. + priority: Subscriber priority (u8). + + Returns: + The encoded subscribe body with varint length prefix. + """ + body = MOQCodec.encode_varint(subscribe_id) + body += MOQCodec.encode_string(broadcast_path) + body += MOQCodec.encode_string(track_name) + body += struct.pack("!B", priority) + + # Wrap: varint(body_len) + body + return MOQCodec.encode_varint(len(body)) + body + + @staticmethod + def encode_subscribe_ok() -> bytes: + """Encode a SUBSCRIBE_OK response. + + Format: varint(0) — empty body. + + Returns: + The encoded SUBSCRIBE_OK. + """ + return MOQCodec.encode_varint(0) + + @staticmethod + def decode_subscribe(data: bytes, offset: int = 0) -> Tuple[int, str, str, int, int]: + """Decode a SUBSCRIBE message body. + + Args: + data: The bytes to decode from. + offset: Starting offset. + + Returns: + Tuple of (subscribe_id, broadcast_path, track_name, priority, new offset). + """ + # Body length + body_len, offset = MOQCodec.decode_varint(data, offset) + body_end = offset + body_len + + subscribe_id, offset = MOQCodec.decode_varint(data, offset) + broadcast_path, offset = MOQCodec.decode_string(data, offset) + track_name, offset = MOQCodec.decode_string(data, offset) + priority = data[offset] + offset += 1 + + return subscribe_id, broadcast_path, track_name, priority, body_end + + @staticmethod + def decode_subscribe_ok(data: bytes, offset: int = 0) -> int: + """Decode a SUBSCRIBE_OK response. + + Format: varint(body_len) where body_len should be 0. + + Args: + data: The bytes to decode from. + offset: Starting offset. + + Returns: + New offset after decoding. + """ + body_len, offset = MOQCodec.decode_varint(data, offset) + return offset + body_len + + # ------------------------------------------------------------------ + # Announce messages (on a dedicated bidi stream with stream_type=1) + # ------------------------------------------------------------------ + + @staticmethod + def encode_announce_please(path_prefix: str) -> bytes: + """Encode an ANNOUNCE_PLEASE message body. + + Body format: string(path_prefix) + Framed as: varint(1) + varint(body_len) + body + + The stream type varint(1) is written separately by the caller. + + Args: + path_prefix: The path prefix to request announcements for. + + Returns: + The encoded announce_please body with varint length prefix. + """ + body = MOQCodec.encode_string(path_prefix) + return MOQCodec.encode_varint(len(body)) + body + + @staticmethod + def encode_announce_init(suffixes: List[str]) -> bytes: + """Encode an ANNOUNCE_INIT response. + + Body format: varint(count) + string(suffix)... + Framed as: varint(body_len) + body + + Args: + suffixes: List of path suffixes to announce. + + Returns: + The encoded announce_init. + """ + body = MOQCodec.encode_varint(len(suffixes)) + for suffix in suffixes: + body += MOQCodec.encode_string(suffix) + return MOQCodec.encode_varint(len(body)) + body + + @staticmethod + def encode_announce_update(status: int, path_suffix: str) -> bytes: + """Encode an ANNOUNCE_UPDATE message. + + Body format: u8(status) + string(path_suffix) + Framed as: varint(body_len) + body + + Args: + status: Announce status (0=active, 1=ended). + path_suffix: The path suffix being updated. + + Returns: + The encoded announce_update. + """ + body = struct.pack("!B", status) + body += MOQCodec.encode_string(path_suffix) + return MOQCodec.encode_varint(len(body)) + body + + @staticmethod + def decode_announce_please(data: bytes, offset: int = 0) -> Tuple[str, int]: + """Decode an ANNOUNCE_PLEASE message body. + + Args: + data: The bytes to decode from. + offset: Starting offset. + + Returns: + Tuple of (path_prefix, new offset). + """ + body_len, offset = MOQCodec.decode_varint(data, offset) + body_end = offset + body_len + path_prefix, offset = MOQCodec.decode_string(data, offset) + return path_prefix, body_end + + # ------------------------------------------------------------------ + # Data messages (on unidirectional streams) + # ------------------------------------------------------------------ + + @staticmethod + def encode_group_header(subscribe_id: int, group_sequence: int) -> bytes: + """Encode a GROUP header for a unidirectional data stream. + + Format: u8(0) + varint(body_len) + varint(subscribe_id) + varint(group_sequence) + + Args: + subscribe_id: The subscription ID this data is for. + group_sequence: The group sequence number. + + Returns: + The encoded GROUP header. + """ + body = MOQCodec.encode_varint(subscribe_id) + body += MOQCodec.encode_varint(group_sequence) + + msg = struct.pack("!B", UNI_STREAM_TYPE_GROUP) + msg += MOQCodec.encode_varint(len(body)) + msg += body + + return msg + + @staticmethod + def encode_frame(payload: bytes) -> bytes: + """Encode a FRAME within a group. + + Format: varint(payload_len) + payload + + Args: + payload: The media payload data. + + Returns: + The encoded frame. + """ + return MOQCodec.encode_varint(len(payload)) + payload + + @staticmethod + def decode_group_header(data: bytes, offset: int = 0) -> Tuple[int, int, int]: + """Decode a GROUP header from a unidirectional stream. + + Format: u8(0) + varint(body_len) + varint(subscribe_id) + varint(group_sequence) + + Args: + data: The bytes to decode from. + offset: Starting offset. + + Returns: + Tuple of (subscribe_id, group_sequence, new offset after header). + """ + # Stream type byte (should be 0) + stream_type = data[offset] + offset += 1 + + # Body length + body_len, offset = MOQCodec.decode_varint(data, offset) + body_start = offset + + subscribe_id, offset = MOQCodec.decode_varint(data, offset) + group_sequence, offset = MOQCodec.decode_varint(data, offset) + + # Ensure offset aligns with body_start + body_len + offset = body_start + body_len + + return subscribe_id, group_sequence, offset + + @staticmethod + def decode_frames(data: bytes, offset: int = 0) -> List[bytes]: + """Decode all FRAMEs from remaining data after GROUP header. + + Format: varint(payload_len) + payload, repeated. + + Args: + data: The bytes to decode from. + offset: Starting offset (after GROUP header). + + Returns: + List of payload byte arrays. + """ + frames = [] + while offset < len(data): + payload_len, offset = MOQCodec.decode_varint(data, offset) + payload = data[offset : offset + payload_len] + offset += payload_len + frames.append(bytes(payload)) + return frames + + +class MOQSession: + """Manages MOQ session state for moq-lite-02. + + Tracks subscriptions and group sequences for a single MOQ connection. + In moq-lite-02, track aliases are gone — subscribe_id is used directly. + """ + + def __init__(self, role: MOQRole = MOQRole.PUBSUB): + """Initialize the MOQ session. + + Args: + role: The role for this session (publisher, subscriber, or both). + """ + self.role = role + self.version: int = MOQL_VERSION + self.setup_complete: bool = False + + # Subscription tracking + self._next_subscribe_id: int = 1 + + # Group sequencing per subscribe_id (for publishing) + self._group_sequences: Dict[int, int] = {} + + def next_subscribe_id(self) -> int: + """Get the next available subscribe ID. + + Returns: + The next subscribe ID. + """ + sid = self._next_subscribe_id + self._next_subscribe_id += 1 + return sid + + def get_next_group_sequence(self, subscribe_id: int) -> int: + """Get the next group sequence number for publishing. + + Args: + subscribe_id: The subscription ID to get the sequence for. + + Returns: + The next group sequence number. + """ + seq = self._group_sequences.get(subscribe_id, 0) + self._group_sequences[subscribe_id] = seq + 1 + return seq diff --git a/src/pipecat/transports/moq/transport.py b/src/pipecat/transports/moq/transport.py new file mode 100644 index 000000000..6b1406f80 --- /dev/null +++ b/src/pipecat/transports/moq/transport.py @@ -0,0 +1,1072 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""MOQ (Media over QUIC) transport implementation for Pipecat — moq-lite-02. + +This module provides MOQ transport functionality for real-time media streaming +using the QUIC protocol, connecting to MOQ relays for low-latency audio and +video transmission with pub/sub semantics. + +moq-lite-02 uses a stream-per-request model: each operation (setup, +subscribe, announce) opens its own bidirectional QUIC stream, and media +data flows on unidirectional streams as GROUP + FRAME sequences. + +Based on moq-lite-02 (version code 0xff0dad02). +""" + +import asyncio +import json +import ssl +import struct +import time +from typing import Awaitable, Callable, Dict, Optional + +from loguru import logger +from pydantic import BaseModel, ConfigDict + +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + Frame, + InputAudioRawFrame, + InputImageRawFrame, + InputTransportMessageFrame, + InterruptionFrame, + OutputAudioRawFrame, + OutputImageRawFrame, + OutputTransportMessageFrame, + OutputTransportMessageUrgentFrame, + StartFrame, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.transports.base_input import BaseInputTransport +from pipecat.transports.base_output import BaseOutputTransport +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.moq.protocol import ( + CLIENT_SETUP_TYPE, + MOQL_ALPN, + MOQL_VERSION, + SERVER_SETUP_TYPE, + STREAM_TYPE_ANNOUNCE, + STREAM_TYPE_SESSION, + STREAM_TYPE_SUBSCRIBE, + MOQCodec, + MOQRole, + MOQSession, + MOQTrack, + MOQTrackType, +) + +try: + from aioquic.asyncio import connect + from aioquic.asyncio.protocol import QuicConnectionProtocol + from aioquic.quic.configuration import QuicConfiguration + from aioquic.quic.events import ( + ConnectionTerminated, + HandshakeCompleted, + QuicEvent, + StreamDataReceived, + StreamReset, + ) +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use MOQ transport, you need to `pip install pipecat-ai[moq]`.") + raise Exception(f"Missing module: {e}") + + +# Default track settings +DEFAULT_NAMESPACE = "pipecat" +DEFAULT_PARTICIPANT_ID = "bot0" +DEFAULT_PEER_ID = "client0" +DEFAULT_AUDIO_TRACK_NAME = "bot-audio" +DEFAULT_AUDIO_IN_TRACK_NAME = "user-audio" +DEFAULT_VIDEO_TRACK_NAME = "video" +DEFAULT_TRANSCRIPT_TRACK_NAME = "transcript" + + +class MOQParams(TransportParams): + """Configuration parameters for MOQ transport. + + Each MOQ participant publishes under its own broadcast path: + ``/``. The bot subscribes to a peer's broadcast + at ``/``. Tracks live inside a broadcast and are + named by their role (e.g. ``bot-audio``, ``user-audio``, + ``custom-audio``). Using distinct broadcast paths lets the relay route + SUBSCRIBEs to the right participant when multiple bots/clients are on + the same namespace. + + Parameters: + role: The MOQ role (publisher, subscriber, or pubsub). + relay_url: URL of the MOQ relay (e.g., "https://localhost:4080/moq"). + namespace: Top-level namespace (defaults to "pipecat"). + participant_id: This bot's unique suffix inside ``namespace``. + Combined with ``namespace`` gives the broadcast this bot + publishes under, e.g. ``pipecat/bot0``. + peer_id: The peer participant id to subscribe to, e.g. + ``client0`` → bot subscribes to ``pipecat/client0/``. + audio_track_name: Name for the output audio track the bot + publishes inside its broadcast (bot → client). + audio_in_track_name: Name for the input audio track the bot + subscribes to inside the peer's broadcast (client → bot). + video_track_name: Name for the video track inside the bot's + broadcast. + priority: Default publisher priority (lower is higher priority). + verify_ssl: Whether to verify SSL certificates. + connection_timeout: Connection timeout in seconds. + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + role: MOQRole = MOQRole.PUBSUB + relay_url: Optional[str] = None + namespace: str = DEFAULT_NAMESPACE + participant_id: str = DEFAULT_PARTICIPANT_ID + peer_id: str = DEFAULT_PEER_ID + audio_track_name: str = DEFAULT_AUDIO_TRACK_NAME + audio_in_track_name: str = DEFAULT_AUDIO_IN_TRACK_NAME + video_track_name: str = DEFAULT_VIDEO_TRACK_NAME + transcript_track_name: str = DEFAULT_TRANSCRIPT_TRACK_NAME + priority: int = 128 + verify_ssl: bool = True + connection_timeout: float = 30.0 + + +class MOQCallbacks(BaseModel): + """Callback functions for MOQ transport events. + + Parameters: + on_connected: Called when connection is established. + on_disconnected: Called when connection is lost. + on_track_published: Called when a track is successfully published. + on_track_subscribed: Called when a subscription is confirmed. + on_error: Called when an error occurs. + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + on_connected: Callable[[], Awaitable[None]] + on_disconnected: Callable[[], Awaitable[None]] + on_track_published: Callable[[MOQTrack], Awaitable[None]] + on_track_subscribed: Callable[[MOQTrack], Awaitable[None]] + on_error: Callable[[str, Optional[Exception]], Awaitable[None]] + + +class MOQClientProtocol(QuicConnectionProtocol): + """QUIC protocol handler for MOQ client connections (moq-lite-02). + + Uses stream-per-request model: each operation opens its own bidi stream. + Media data flows on unidirectional streams as GROUP + FRAME sequences. + """ + + def __init__(self, *args, **kwargs): + """Initialize the MOQ protocol handler.""" + super().__init__(*args, **kwargs) + self._session: MOQSession = MOQSession() + self._setup_complete = asyncio.Event() + self._connected = False + + # Callbacks for received media and events + self._on_subscribe_received: Optional[Callable[[str], Awaitable[None]]] = None + self._on_audio_frame: Optional[Callable[[bytes, int], Awaitable[None]]] = None + self._on_video_frame: Optional[Callable[[bytes, int, int], Awaitable[None]]] = None + self._on_message: Optional[Callable[[bytes], Awaitable[None]]] = None + + # Subscribe IDs for our subscriptions (receiving data) + self._audio_subscribe_id: Optional[int] = None + self._video_subscribe_id: Optional[int] = None + + # Subscribe ID assigned by relay for our published tracks + self._publish_subscribe_ids: Dict[str, int] = {} + + # Stream buffers for incoming data + self._stream_buffers: Dict[int, bytes] = {} + self._stream_types: Dict[int, Optional[int]] = {} + + # Setup stream + self._setup_stream_id: Optional[int] = None + + # Track which subscribe IDs map to which track names + self._subscribe_id_to_track: Dict[int, str] = {} + + # Track subscribe stream IDs for detecting rejections + self._subscribe_streams: Dict[int, int] = {} # stream_id -> subscribe_id + self._rejected_subscribe_ids: set = set() + + # Namespace + per-participant identity. Broadcast path this client + # publishes under is "/". + self._namespace: str = DEFAULT_NAMESPACE + self._participant_id: str = DEFAULT_PARTICIPANT_ID + self._audio_track_name: str = DEFAULT_AUDIO_TRACK_NAME + self._video_track_name: str = DEFAULT_VIDEO_TRACK_NAME + self._transcript_track_name: str = DEFAULT_TRANSCRIPT_TRACK_NAME + + def set_audio_callback(self, callback: Callable[[bytes, int], Awaitable[None]]): + """Set the callback for received audio frames.""" + self._on_audio_frame = callback + + def set_video_callback(self, callback: Callable[[bytes, int, int], Awaitable[None]]): + """Set the callback for received video frames.""" + self._on_video_frame = callback + + def set_message_callback(self, callback: Callable[[bytes], Awaitable[None]]): + """Set the callback for received messages.""" + self._on_message = callback + + def quic_event_received(self, event: QuicEvent): + """Handle QUIC events.""" + if isinstance(event, HandshakeCompleted): + logger.info( + f"🟠 QUIC handshake completed (alpn={event.alpn_protocol}, " + f"transport=moq-lite-02, version={MOQL_VERSION:#x})" + ) + if event.alpn_protocol != MOQL_ALPN: + logger.error( + f"💔❌💔 UNEXPECTED ALPN! " + f"Expected '{MOQL_ALPN}' but relay negotiated " + f"'{event.alpn_protocol}'. Connection may not work!" + ) + self._connected = True + + elif isinstance(event, StreamDataReceived): + self._handle_stream_data(event.stream_id, event.data, event.end_stream) + + elif isinstance(event, StreamReset): + logger.warning(f"Stream {event.stream_id} reset (error_code={event.error_code})") + # Track which subscribe streams were reset (subscription rejected) + if event.stream_id in self._subscribe_streams: + sub_id = self._subscribe_streams[event.stream_id] + logger.warning(f"Subscribe {sub_id} was rejected (stream reset)") + self._rejected_subscribe_ids.add(sub_id) + if sub_id == self._audio_subscribe_id: + self._audio_subscribe_id = None + + elif isinstance(event, ConnectionTerminated): + logger.info(f"Connection terminated: {event.reason_phrase}") + self._connected = False + + def _is_unidirectional(self, stream_id: int) -> bool: + """Check if a stream ID is unidirectional.""" + return (stream_id & 0x02) != 0 + + def _is_server_initiated(self, stream_id: int) -> bool: + """Check if a stream ID is server-initiated.""" + return (stream_id & 0x01) != 0 + + def _handle_stream_data(self, stream_id: int, data: bytes, end_stream: bool): + """Handle data received on a QUIC stream.""" + if not data and not end_stream: + return + + logger.trace( + f"Stream {stream_id} data ({len(data)} bytes, end={end_stream}): " + f"[{' '.join(f'{b:02x}' for b in data[:40])}]" + ) + + # Buffer data + if stream_id not in self._stream_buffers: + self._stream_buffers[stream_id] = b"" + self._stream_buffers[stream_id] += data + + if self._is_unidirectional(stream_id): + # Uni stream: buffer until FIN, then parse GROUP + FRAMEs + if end_stream: + self._handle_uni_stream_complete(stream_id) + elif stream_id == self._setup_stream_id: + # Our setup stream — look for SERVER_SETUP response + self._try_parse_server_setup(stream_id) + elif self._is_server_initiated(stream_id): + # Server-initiated bidi stream — could be SUBSCRIBE or ANNOUNCE + self._handle_incoming_bidi(stream_id, end_stream) + + def _try_parse_server_setup(self, stream_id: int): + """Try to parse SERVER_SETUP from buffered data on setup stream.""" + buf = self._stream_buffers.get(stream_id, b"") + if len(buf) < 2: + return + + # Look for 0x21 (SERVER_SETUP type byte) + if buf[0] != SERVER_SETUP_TYPE: + logger.warning(f"Expected SERVER_SETUP (0x21), got {buf[0]:#x}") + return + + try: + version, _ = MOQCodec.decode_server_setup(buf, 0) + client_name = "moq-lite-02" + server_name = "moq-lite-02" if version == MOQL_VERSION else f"unknown" + logger.info( + f"🟠 MOQ protocol: client={client_name} ({MOQL_VERSION:#x}), " + f"relay={server_name} ({version:#x})" + ) + if version != MOQL_VERSION: + logger.error( + f"💔❌💔 PROTOCOL MISMATCH! " + f"Bot speaks {client_name} ({MOQL_VERSION:#x}) but " + f"relay responded with {server_name} ({version:#x}). " + f"Connection will likely fail!" + ) + self._session.setup_complete = True + self._setup_complete.set() + except (IndexError, struct.error): + # Incomplete data, wait for more + pass + + def _handle_incoming_bidi(self, stream_id: int, end_stream: bool): + """Handle data on a server-initiated bidi stream.""" + buf = self._stream_buffers.get(stream_id, b"") + if len(buf) < 1: + return + + # Determine stream type from first varint if not yet known + if stream_id not in self._stream_types: + try: + stream_type, _ = MOQCodec.decode_varint(buf, 0) + self._stream_types[stream_id] = stream_type + except (IndexError, struct.error): + return + + stream_type = self._stream_types.get(stream_id) + + if stream_type == STREAM_TYPE_SUBSCRIBE: + self._handle_incoming_subscribe(stream_id, buf) + elif stream_type == STREAM_TYPE_ANNOUNCE: + self._handle_incoming_announce(stream_id, buf) + else: + logger.debug(f"Unknown incoming bidi stream type: {stream_type}") + + def _handle_incoming_subscribe(self, stream_id: int, buf: bytes): + """Handle an incoming SUBSCRIBE from the relay. + + The relay sends SUBSCRIBE when a downstream subscriber wants our track. + We respond with SUBSCRIBE_OK for known tracks, or reset the stream + for unknown tracks so the relay can try other publishers. + """ + try: + # Skip stream type varint + _, offset = MOQCodec.decode_varint(buf, 0) + + # Decode subscribe body + subscribe_id, broadcast_path, track_name, priority, _ = MOQCodec.decode_subscribe( + buf, offset + ) + + logger.debug( + f"Received SUBSCRIBE for {broadcast_path}/{track_name} " + f"(subscribe_id={subscribe_id}, priority={priority})" + ) + + # Only accept subscribes for tracks we actually publish, which + # live inside our broadcast: "/". + our_broadcast = self._namespace + "/" + self._participant_id + known_tracks = { + our_broadcast + "/" + self._audio_track_name, + our_broadcast + "/" + self._video_track_name, + our_broadcast + "/" + self._transcript_track_name, + } + full_track = broadcast_path + "/" + track_name + if full_track not in known_tracks: + logger.debug( + f"Rejecting SUBSCRIBE for unknown track {full_track} " + f"(we only publish: {known_tracks})" + ) + self._quic.reset_stream(stream_id, error_code=0) + self.transmit() + return + + # Store the subscribe_id for publishing to this subscriber + self._publish_subscribe_ids[track_name] = subscribe_id + self._subscribe_id_to_track[subscribe_id] = track_name + + # Send SUBSCRIBE_OK response on the same stream + ok_data = MOQCodec.encode_subscribe_ok() + self._quic.send_stream_data(stream_id, ok_data, end_stream=False) + self.transmit() + + logger.debug(f"Sent SUBSCRIBE_OK for subscribe_id={subscribe_id}") + + # Notify transport that someone subscribed to our track + if self._on_subscribe_received: + asyncio.create_task(self._on_subscribe_received(track_name)) + + except Exception as e: + logger.error(f"Error handling incoming SUBSCRIBE: {e}") + + def _handle_incoming_announce(self, stream_id: int, buf: bytes): + """Handle an incoming ANNOUNCE stream from the relay. + + The relay sends ANNOUNCE_PLEASE with a path prefix, and we respond + with ANNOUNCE_INIT listing our broadcast path suffixes under that prefix. + """ + try: + # Skip stream type varint + _, offset = MOQCodec.decode_varint(buf, 0) + + # This is ANNOUNCE_PLEASE from the relay + path_prefix, _ = MOQCodec.decode_announce_please(buf, offset) + logger.debug(f"Received ANNOUNCE_PLEASE for prefix: '{path_prefix}'") + + # Our broadcast path is "/". Compute + # its suffix relative to the relay-requested prefix. + broadcast = self._namespace + "/" + self._participant_id + if path_prefix and broadcast.startswith(path_prefix): + suffix = broadcast[len(path_prefix) :].lstrip("/") + elif not path_prefix: + suffix = broadcast + else: + # Our broadcast doesn't match the prefix — announce nothing + suffix = None + + if suffix is not None: + logger.debug(f"Announcing broadcast suffix: '{suffix}'") + init_data = MOQCodec.encode_announce_init([suffix]) + else: + logger.debug("No matching broadcasts for prefix, announcing empty") + init_data = MOQCodec.encode_announce_init([]) + + self._quic.send_stream_data(stream_id, init_data) + self.transmit() + + except Exception as e: + logger.error(f"Error handling incoming ANNOUNCE: {e}") + + def _handle_uni_stream_complete(self, stream_id: int): + """Handle a completed unidirectional stream (GROUP + FRAMEs).""" + buf = self._stream_buffers.pop(stream_id, b"") + if not buf: + return + + try: + subscribe_id, group_sequence, offset = MOQCodec.decode_group_header(buf, 0) + frames = MOQCodec.decode_frames(buf, offset) + + track_name = self._subscribe_id_to_track.get(subscribe_id) + + for frame_data in frames: + if not frame_data: + continue + + if subscribe_id == self._audio_subscribe_id and self._on_audio_frame: + asyncio.create_task(self._on_audio_frame(frame_data, 16000)) + elif subscribe_id == self._video_subscribe_id and self._on_video_frame: + asyncio.create_task(self._on_video_frame(frame_data, 0, 0)) + elif self._on_message: + asyncio.create_task(self._on_message(frame_data)) + + except Exception as e: + logger.error(f"Error processing uni stream data: {e}") + + # Clean up + self._stream_types.pop(stream_id, None) + + async def send_client_setup(self, role: MOQRole, path: str = "/moq"): + """Send CLIENT_SETUP message on a new bidi stream.""" + self._setup_stream_id = self._quic.get_next_available_stream_id(is_unidirectional=False) + + setup_msg = MOQCodec.encode_client_setup(role, [MOQL_VERSION], path) + logger.info( + f"CLIENT_SETUP ({len(setup_msg)} bytes) on stream {self._setup_stream_id}: " + f"[{' '.join(f'{b:02x}' for b in setup_msg[:40])}]" + ) + self._quic.send_stream_data(self._setup_stream_id, setup_msg) + self.transmit() + + logger.debug(f"Sent CLIENT_SETUP on stream {self._setup_stream_id}") + + async def wait_for_setup(self, timeout: float = 10.0): + """Wait for setup to complete.""" + try: + await asyncio.wait_for(self._setup_complete.wait(), timeout=timeout) + return True + except asyncio.TimeoutError: + logger.error("Setup timeout") + return False + + async def subscribe_track( + self, namespace: str, track_name: str, track_type: MOQTrackType + ) -> Optional[int]: + """Subscribe to a track by opening a new bidi stream. + + moq-lite-02: Each subscription gets its own bidi stream. + Stream format: varint(2) + SUBSCRIBE body, then read SUBSCRIBE_OK. + """ + subscribe_id = self._session.next_subscribe_id() + + # Open a new bidi stream + stream_id = self._quic.get_next_available_stream_id(is_unidirectional=False) + + # Write stream type + subscribe body + msg = MOQCodec.encode_varint(STREAM_TYPE_SUBSCRIBE) + msg += MOQCodec.encode_subscribe(subscribe_id, namespace, track_name) + + self._quic.send_stream_data(stream_id, msg, end_stream=False) + self.transmit() + + # Track stream for detecting rejections (RESET_STREAM) + self._subscribe_streams[stream_id] = subscribe_id + + # Track the subscribe_id for receiving data + if track_type == MOQTrackType.AUDIO: + self._audio_subscribe_id = subscribe_id + elif track_type == MOQTrackType.VIDEO: + self._video_subscribe_id = subscribe_id + + self._subscribe_id_to_track[subscribe_id] = track_name + + logger.debug( + f"Subscribed to {namespace}/{track_name} " + f"(subscribe_id={subscribe_id}, stream={stream_id})" + ) + return subscribe_id + + async def publish_audio(self, audio: bytes, priority: int = 128): + """Publish audio data on a new unidirectional stream. + + moq-lite-02: Each group is sent on its own uni stream with + GROUP header + FRAME + FIN. + """ + subscribe_id = self._publish_subscribe_ids.get(self._audio_track_name) + if subscribe_id is None: + return + + group_seq = self._session.get_next_group_sequence(subscribe_id) + + # Open uni stream, write GROUP header + FRAME, FIN + stream_id = self._quic.get_next_available_stream_id(is_unidirectional=True) + + data = MOQCodec.encode_group_header(subscribe_id, group_seq) + data += MOQCodec.encode_frame(audio) + + self._quic.send_stream_data(stream_id, data, end_stream=True) + self.transmit() + + async def publish_video(self, image: bytes, priority: int = 129): + """Publish video data on a new unidirectional stream.""" + subscribe_id = self._publish_subscribe_ids.get(self._video_track_name) + if subscribe_id is None: + return + + group_seq = self._session.get_next_group_sequence(subscribe_id) + + stream_id = self._quic.get_next_available_stream_id(is_unidirectional=True) + + data = MOQCodec.encode_group_header(subscribe_id, group_seq) + data += MOQCodec.encode_frame(image) + + self._quic.send_stream_data(stream_id, data, end_stream=True) + self.transmit() + + async def publish_text(self, payload: bytes) -> bool: + """Publish a text/control payload on the transcript track. + + Returns False if no subscriber has SUBSCRIBE'd to our transcript + track yet — caller can drop or buffer. + """ + subscribe_id = self._publish_subscribe_ids.get(self._transcript_track_name) + if subscribe_id is None: + return False + + group_seq = self._session.get_next_group_sequence(subscribe_id) + stream_id = self._quic.get_next_available_stream_id(is_unidirectional=True) + + data = MOQCodec.encode_group_header(subscribe_id, group_seq) + data += MOQCodec.encode_frame(payload) + + self._quic.send_stream_data(stream_id, data, end_stream=True) + self.transmit() + return True + + +class MOQInputTransport(BaseInputTransport): + """MOQ input transport for receiving media from a relay. + + Handles subscribing to tracks and receiving audio/video streams. + """ + + def __init__( + self, + transport: "MOQTransport", + params: MOQParams, + callbacks: MOQCallbacks, + **kwargs, + ): + """Initialize the MOQ input transport.""" + super().__init__(params, **kwargs) + + self._moq_transport = transport + self._params = params + self._callbacks = callbacks + self._protocol: Optional[MOQClientProtocol] = None + self._initialized = False + + async def start(self, frame: StartFrame): + """Start the MOQ input transport and connect to relay.""" + await super().start(frame) + + if self._initialized: + return + + self._initialized = True + + # Auto-connect to relay when pipeline starts + self._moq_transport._connection_task = self.create_task( + self._moq_transport.connect(), "moq_connect" + ) + + await self.set_transport_ready(frame) + + async def stop(self, frame: EndFrame): + """Stop the MOQ input transport.""" + await super().stop(frame) + + async def cancel(self, frame: CancelFrame): + """Cancel the MOQ input transport.""" + await super().cancel(frame) + + async def cleanup(self): + """Cleanup resources.""" + await super().cleanup() + await self._moq_transport.cleanup() + + def set_protocol(self, protocol: MOQClientProtocol): + """Set the MOQ protocol handler.""" + self._protocol = protocol + protocol.set_audio_callback(self._on_audio_received) + protocol.set_video_callback(self._on_video_received) + protocol.set_message_callback(self._on_message_received) + + async def _on_audio_received(self, audio: bytes, sample_rate: int): + """Handle received audio data.""" + frame = InputAudioRawFrame( + audio=audio, + sample_rate=sample_rate, + num_channels=1, + ) + await self.push_audio_frame(frame) + + async def _on_video_received(self, image: bytes, width: int, height: int): + """Handle received video data.""" + frame = InputImageRawFrame( + image=image, + size=(width, height), + format="RGB", + ) + await self.push_video_frame(frame) + + async def _on_message_received(self, message: bytes): + """Handle received transport message.""" + await self.broadcast_frame( + InputTransportMessageFrame, + message=message.decode("utf-8"), + ) + + async def subscribe_audio(self): + """Subscribe to the peer's audio track on the relay. + + Target broadcast is ``/`` so the relay routes + the SUBSCRIBE to the peer (e.g. the browser) and not back to us. + Retries because the peer may not yet be announced when we first try. + """ + if not self._protocol: + return + + broadcast = self._params.namespace + "/" + self._params.peer_id + track = self._params.audio_in_track_name + max_retries = 10 + for attempt in range(max_retries): + if attempt > 0: + await asyncio.sleep(1) + logger.info( + f"Subscribing to client audio: {broadcast}/{track}" + f" (attempt {attempt + 1}/{max_retries})" + ) + sub_id = await self._protocol.subscribe_track( + broadcast, + track, + MOQTrackType.AUDIO, + ) + # Wait for the relay to respond (accept or reset) + await asyncio.sleep(0.5) + # Check if this subscribe was rejected + if sub_id in self._protocol._rejected_subscribe_ids: + logger.warning(f"Subscribe attempt {attempt + 1} rejected, retrying...") + continue + logger.info(f"Subscribed to client audio: {broadcast}/{track}") + return + logger.error(f"Failed to subscribe to client audio after {max_retries} attempts") + + async def subscribe_video(self): + """Subscribe to the peer's video track on the relay.""" + if self._protocol: + await self._protocol.subscribe_track( + self._params.namespace + "/" + self._params.peer_id, + self._params.video_track_name, + MOQTrackType.VIDEO, + ) + + +class MOQOutputTransport(BaseOutputTransport): + """MOQ output transport for publishing media to a relay. + + In moq-lite-02, the announce flow is reversed: the relay/subscriber sends + SUBSCRIBE to us. We register tracks locally and respond to incoming + SUBSCRIBE requests with SUBSCRIBE_OK. + """ + + def __init__( + self, + transport: "MOQTransport", + params: MOQParams, + **kwargs, + ): + """Initialize the MOQ output transport.""" + super().__init__(params, **kwargs) + + self._moq_transport = transport + self._params = params + self._protocol: Optional[MOQClientProtocol] = None + + # Timing for pacing output + self._send_interval = 0 + self._next_send_time = 0 + self._initialized = False + + async def start(self, frame: StartFrame): + """Start the MOQ output transport.""" + await super().start(frame) + + if self._initialized: + return + + self._initialized = True + self._send_interval = (self.audio_chunk_size / self.sample_rate) / 2 + logger.info( + f"MOQ output: sample_rate={self.sample_rate}, " + f"chunk_size={self.audio_chunk_size}, send_interval={self._send_interval:.4f}s" + ) + await self.set_transport_ready(frame) + + async def stop(self, frame: EndFrame): + """Stop the MOQ output transport.""" + await super().stop(frame) + + async def cancel(self, frame: CancelFrame): + """Cancel the MOQ output transport.""" + await super().cancel(frame) + + async def cleanup(self): + """Cleanup resources.""" + await super().cleanup() + await self._moq_transport.cleanup() + + def set_protocol(self, protocol: MOQClientProtocol): + """Set the MOQ protocol handler.""" + self._protocol = protocol + + async def announce_tracks(self): + """Register tracks for publishing. + + In moq-lite-02, we don't proactively announce. The relay forwards + subscriptions to us. We just need to be ready to respond to incoming + SUBSCRIBE requests, which the protocol handler does automatically. + """ + pass + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames.""" + await super().process_frame(frame, direction) + + if isinstance(frame, InterruptionFrame): + self._next_send_time = 0 + + async def send_message( + self, frame: OutputTransportMessageFrame | OutputTransportMessageUrgentFrame + ): + """Send a transport message to subscribers on the transcript track. + + ``RTVIObserver`` (auto-attached to ``PipelineTask``) converts + pipeline frames (transcriptions, bot output, speech events, …) into + RTVI message models and pushes them through this method as + ``OutputTransportMessage[Urgent]Frame`` instances whose ``message`` + is the model's JSON-ready dict. We serialize and send as-is — the + browser parses RTVI message types client-side. + """ + if not self._protocol: + return + payload = frame.message + if not isinstance(payload, (bytes, bytearray)): + payload = json.dumps(payload).encode("utf-8") + try: + await self._protocol.publish_text(payload) + except Exception as e: + logger.warning(f"Failed to publish transport message: {e}") + + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: + """Write an audio frame to the relay.""" + if not self._protocol: + return False + + try: + await self._protocol.publish_audio(frame.audio, self._params.priority) + await self._write_audio_sleep() + return True + except Exception as e: + logger.error(f"Error writing audio frame: {e}") + return False + + async def write_video_frame(self, frame: OutputImageRawFrame) -> bool: + """Write a video frame to the relay.""" + if not self._protocol: + return False + + try: + await self._protocol.publish_video(frame.image, self._params.priority + 1) + return True + except Exception as e: + logger.error(f"Error writing video frame: {e}") + return False + + async def _write_audio_sleep(self): + """Pace audio output.""" + current_time = time.monotonic() + sleep_duration = max(0, self._next_send_time - current_time) + await asyncio.sleep(sleep_duration) + if sleep_duration == 0: + self._next_send_time = time.monotonic() + self._send_interval + else: + self._next_send_time += self._send_interval + + +class MOQTransport(BaseTransport): + """MOQ transport for connecting to a MOQ relay (moq-lite-02). + + Provides a complete MOQ client implementation with separate input and output + transports for publishing and subscribing to media streams over QUIC. + + Example:: + + transport = MOQTransport( + params=MOQParams( + audio_in_enabled=True, + audio_out_enabled=True, + role=MOQRole.PUBSUB, + namespace="my-room", + ), + host="localhost", + port=4080, + ) + + @transport.event_handler("on_connected") + async def on_connected(): + print("Connected to MOQ relay") + + # Connect to relay + await transport.connect() + + Event handlers available: + + - on_connected: Called when connection to relay is established + - on_disconnected: Called when connection is lost + - on_track_published: Called when a track is successfully published + - on_track_subscribed: Called when a subscription is confirmed + - on_error: Called when an error occurs + """ + + def __init__( + self, + params: MOQParams, + host: str = "localhost", + port: int = 4080, + path: str = "/moq", + input_name: Optional[str] = None, + output_name: Optional[str] = None, + ): + """Initialize the MOQ transport. + + Args: + params: MOQ configuration parameters. + host: Relay host address. + port: Relay port number. + path: MOQ endpoint path on the relay. + input_name: Optional name for the input processor. + output_name: Optional name for the output processor. + """ + super().__init__(input_name=input_name, output_name=output_name) + + self._host = host + self._port = port + self._path = path + self._params = params + + self._callbacks = MOQCallbacks( + on_connected=self._on_connected, + on_disconnected=self._on_disconnected, + on_track_published=self._on_track_published, + on_track_subscribed=self._on_track_subscribed, + on_error=self._on_error, + ) + + self._input: Optional[MOQInputTransport] = None + self._output: Optional[MOQOutputTransport] = None + self._protocol: Optional[MOQClientProtocol] = None + self._connection_task: Optional[asyncio.Task] = None + + self._client_connected = False + + # Register event handlers + self._register_event_handler("on_connected") + self._register_event_handler("on_disconnected") + self._register_event_handler("on_client_connected") + self._register_event_handler("on_client_disconnected") + self._register_event_handler("on_track_published") + self._register_event_handler("on_track_subscribed") + self._register_event_handler("on_error") + + def input(self) -> MOQInputTransport: + """Get the input transport for receiving media.""" + if not self._input: + self._input = MOQInputTransport( + self, + self._params, + self._callbacks, + name=self._input_name, + ) + return self._input + + def output(self) -> MOQOutputTransport: + """Get the output transport for sending media.""" + if not self._output: + self._output = MOQOutputTransport( + self, + self._params, + name=self._output_name, + ) + return self._output + + async def connect(self): + """Connect to the MOQ relay using moq-lite-02 protocol.""" + logger.debug("MOQTransport.connect() starting") + + try: + configuration = QuicConfiguration( + alpn_protocols=[MOQL_ALPN], + is_client=True, + ) + + # moq-lite-02 opens many streams + configuration.max_stream_data = 1048576 # 1MB per stream + + if not self._params.verify_ssl: + configuration.verify_mode = ssl.CERT_NONE + + # Force IPv4 for localhost — QUIC/UDP doesn't auto-fallback from IPv6 + connect_host = "127.0.0.1" if self._host == "localhost" else self._host + + # Set SNI explicitly — IP addresses don't carry SNI in TLS, + # so the relay needs the original hostname for certificate lookup + configuration.server_name = self._host + + logger.info(f"Connecting to MOQ relay at {connect_host}:{self._port}{self._path}") + + async with connect( + connect_host, + self._port, + configuration=configuration, + create_protocol=MOQClientProtocol, + ) as protocol: + self._protocol = protocol + + # Store identity on protocol for ANNOUNCE/SUBSCRIBE handling + protocol._namespace = self._params.namespace + protocol._participant_id = self._params.participant_id + protocol._audio_track_name = self._params.audio_track_name + protocol._video_track_name = self._params.video_track_name + protocol._transcript_track_name = self._params.transcript_track_name + protocol._on_subscribe_received = self._on_subscribe_received + + # Set up input/output with protocol + if self._input: + self._input.set_protocol(protocol) + if self._output: + self._output.set_protocol(protocol) + + # Send CLIENT_SETUP + await protocol.send_client_setup(self._params.role, self._path) + + # Wait for SERVER_SETUP + if not await protocol.wait_for_setup(self._params.connection_timeout): + await self._callbacks.on_error("Setup timeout", None) + return + + logger.info("MOQ setup complete, connection established") + + # Don't subscribe yet — wait for a client to connect first. + # Subscribing now would fail with "not found" if no client + # has announced yet. We subscribe in _on_subscribe_received() + # when the relay forwards a SUBSCRIBE for our audio track, + # which signals that a client is present. + + # In moq-lite-02, publishing is reactive: we wait for the relay + # to send us SUBSCRIBE requests for our tracks. The protocol + # handler responds automatically with SUBSCRIBE_OK. + + await self._callbacks.on_connected() + + # Keep connection alive + while protocol._connected: + await asyncio.sleep(0.1) + + await self._callbacks.on_disconnected() + + except Exception as e: + logger.error(f"MOQ connection error: {e}", exc_info=True) + await self._callbacks.on_error(str(e), e) + + async def disconnect(self): + """Disconnect from the MOQ relay.""" + if self._connection_task: + self._connection_task.cancel() + try: + await self._connection_task + except asyncio.CancelledError: + pass + self._connection_task = None + + if self._protocol: + self._protocol.close() + self._protocol = None + + async def _on_connected(self): + """Handle connection established.""" + await self._call_event_handler("on_connected") + + async def _on_disconnected(self): + """Handle connection lost.""" + await self._call_event_handler("on_disconnected") + + async def _on_track_published(self, track: MOQTrack): + """Handle track published.""" + await self._call_event_handler("on_track_published", track) + + async def _on_track_subscribed(self, track: MOQTrack): + """Handle track subscribed.""" + await self._call_event_handler("on_track_subscribed", track) + + async def _on_subscribe_received(self, track_name: str): + """Handle incoming SUBSCRIBE from relay (a client wants our track).""" + if track_name == self._params.audio_track_name and not self._client_connected: + self._client_connected = True + logger.info("Client connected (subscribed to our audio track)") + + # Now that a client is present, subscribe to their audio + if self._params.role in (MOQRole.SUBSCRIBER, MOQRole.PUBSUB): + if self._input: + if self._params.audio_in_enabled: + await self._input.subscribe_audio() + + await self._call_event_handler("on_client_connected") + + async def _on_error(self, message: str, exception: Optional[Exception]): + """Handle error.""" + await self._call_event_handler("on_error", message, exception) diff --git a/uv.lock b/uv.lock index 81c1949a9..00fa25c4b 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.14'", @@ -239,6 +239,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/a1/510b0a7fadc6f43a6ce50152e69dbd86415240835868bb0bd9b5b88b1e06/aioitertools-0.13.0-py3-none-any.whl", hash = "sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be", size = 24182, upload-time = "2025-11-06T22:17:06.502Z" }, ] +[[package]] +name = "aioquic" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "cryptography" }, + { name = "pylsqpack" }, + { name = "pyopenssl" }, + { name = "service-identity" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6b/0c/858bb02e0ff96b40735b09ed7be25690197851e4c1bcde51af3348c851fc/aioquic-1.3.0.tar.gz", hash = "sha256:28d070b2183e3e79afa9d4e7bd558960d0d53aeb98bc0cf0a358b279ba797c92", size = 181923, upload-time = "2025-10-11T09:16:30.91Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/41/9a6cf092f2d21768091969dccd4723270f4cd8138d00097160d9c8eabeb8/aioquic-1.3.0-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:59da070ff0f55a54f5623c9190dbc86638daa0bcf84bbdb11ebe507abc641435", size = 1922701, upload-time = "2025-10-11T09:16:10.971Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ea/ac91850a3e6c915802d8c0ee782f966ddfaeed9f870696c1cdb98b25c9a1/aioquic-1.3.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:48590fa38ec13f01a3d4e44fb3cfd373661094c9c7248f3c54d2d9512b6c3469", size = 2240281, upload-time = "2025-10-11T09:16:12.895Z" }, + { url = "https://files.pythonhosted.org/packages/a8/65/383f3b3921e1d6b9b757bff3c805c24f7180eda690aecb5e8df50eb7b028/aioquic-1.3.0-cp310-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:019b16580d53541b5d77b4a44a61966921156554fad2536d74895713c800caa5", size = 2752433, upload-time = "2025-10-11T09:16:14.724Z" }, + { url = "https://files.pythonhosted.org/packages/b9/00/66f9a2f95db35ccbe1d9384d44beae28072fceec6ca0ffa29f6c640516c2/aioquic-1.3.0-cp310-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:396e5f53f6ddb27713d9b5bb11d8f0f842e42857b7e671c5ae203bf618528550", size = 2445180, upload-time = "2025-10-11T09:16:17.136Z" }, + { url = "https://files.pythonhosted.org/packages/d5/7a/f020815b9fa6ea9b83354deb213b90a25fd01466f5a8e517e1c0e672be8c/aioquic-1.3.0-cp310-abi3-manylinux_2_28_i686.whl", hash = "sha256:4098afc6337adf19bdb54474f6c37983988e7bfa407892a277259c32eb664b00", size = 2361800, upload-time = "2025-10-11T09:16:18.685Z" }, + { url = "https://files.pythonhosted.org/packages/87/be/a141aafe8984ed380e610397d606a9d9818ef30ce352aa9ede048a966d81/aioquic-1.3.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:48292279a248422b6289fffd82159eba8d8b35ff4b1f660b9f74ff85e10ca265", size = 2797515, upload-time = "2025-10-11T09:16:20.451Z" }, + { url = "https://files.pythonhosted.org/packages/52/50/b421e7aedff4a96840bf8734c2c11c18a8434c780c0cb59dff7f0906cee8/aioquic-1.3.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:0538acdfbf839d87b175676664737c248cd51f1a2295c5fef8e131ddde478a86", size = 2388628, upload-time = "2025-10-11T09:16:21.661Z" }, + { url = "https://files.pythonhosted.org/packages/bc/f4/3c674f4608883e7fc7212f067c599d1321b0c5dd45bda5c77ab5a1e73924/aioquic-1.3.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a8881239801279188e33ced6f9849cedf033325a48a6f44d7e55e583abc555a3", size = 2465059, upload-time = "2025-10-11T09:16:23.474Z" }, + { url = "https://files.pythonhosted.org/packages/23/f2/7b1908feffb29b89d2f6d4adc583e83543cd559676354f85c5b4b77a6428/aioquic-1.3.0-cp310-abi3-win32.whl", hash = "sha256:ba30016244e45d9222fdd1fbd4e8b0e5f6811e81a5d0643475ad7024a537274a", size = 1326532, upload-time = "2025-10-11T09:16:25.971Z" }, + { url = "https://files.pythonhosted.org/packages/82/45/4e47404984d65ee31cc9e1370f1fbc4e8c92b25da71f61429dbdba437246/aioquic-1.3.0-cp310-abi3-win_amd64.whl", hash = "sha256:2d7957ba14a6c5efcc14fdc685ccda7ecf0ad048c410a2bdcad1b63bf9527e8e", size = 1675068, upload-time = "2025-10-11T09:16:27.258Z" }, + { url = "https://files.pythonhosted.org/packages/43/60/a8cb5f85c5a6a3cc630124a45644ca5a0ab3eecae2df558b6e0ab7847e1c/aioquic-1.3.0-cp310-abi3-win_arm64.whl", hash = "sha256:9d15a89213d38cbc4679990fa5151af8ea02655a1d6ce5ec972b0a6af74d5f1c", size = 1234825, upload-time = "2025-10-11T09:16:28.994Z" }, +] + [[package]] name = "aiortc" version = "1.14.0" @@ -1898,7 +1924,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/0f/a91f143f356523ff682309732b175765a9bc2836fd7c081c2c67fedc1ad4/greenlet-3.5.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:8f1cc966c126639cd152fdaa52624d2655f492faa79e013fea161de3e6dda082", size = 284726, upload-time = "2026-04-27T12:20:51.402Z" }, { url = "https://files.pythonhosted.org/packages/95/82/800646c7ffc5dbabd75ddd2f6b519bb898c0c9c969e5d0473bfe5d20bcce/greenlet-3.5.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:362624e6a8e5bca3b8233e45eef33903a100e9539a2b995c364d595dbc4018b3", size = 604264, upload-time = "2026-04-27T12:52:39.494Z" }, { url = "https://files.pythonhosted.org/packages/ca/ac/354867c0bba812fc33b15bc55aedafedd0aee3c7dd91dfca22444157dc0c/greenlet-3.5.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5ecd83806b0f4c2f53b1018e0005cd82269ea01d42befc0368730028d850ed1c", size = 616099, upload-time = "2026-04-27T12:59:39.623Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ab/192090c4a5b30df148c22bf4b8895457d739a7c7c5a7b9c41e5dd7f537f2/greenlet-3.5.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa94cb2288681e3a11645958f1871d48ee9211bd2f66628fdace505927d6e564", size = 623976, upload-time = "2026-04-27T13:02:37.363Z" }, { url = "https://files.pythonhosted.org/packages/ff/b0/815bece7399e01cadb69014219eebd0042339875c59a59b0820a46ece356/greenlet-3.5.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0ff251e9a0279522e62f6176412869395a64ddf2b5c5f782ff609a8216a4e662", size = 615198, upload-time = "2026-04-27T12:25:25.928Z" }, + { url = "https://files.pythonhosted.org/packages/24/11/05eb2b9b188c6df7d68a89c99134d644a7af616a40b9808e8e6ced315d5d/greenlet-3.5.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:64d6ac45f7271f48e45f67c95b54ef73534c52ec041fcda8edf520c6d811f4bc", size = 418379, upload-time = "2026-04-27T13:05:12.755Z" }, { url = "https://files.pythonhosted.org/packages/10/80/3b2c0a895d6698f6ddb31b07942ebfa982f3e30888bc5546a5b5990de8b2/greenlet-3.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d874e79afd41a96e11ff4c5d0bc90a80973e476fda1c2c64985667397df432b", size = 1574927, upload-time = "2026-04-27T12:53:25.81Z" }, { url = "https://files.pythonhosted.org/packages/44/0e/f354af514a4c61454dbc68e44d47544a5a4d6317e30b77ddfa3a09f4c5f3/greenlet-3.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0ed006e4b86c59de7467eb2601cd1b77b5a7d657d1ee55e30fe30d76451edba4", size = 1642683, upload-time = "2026-04-27T12:25:23.9Z" }, { url = "https://files.pythonhosted.org/packages/fa/6a/87f38255201e993a1915265ebb80cd7c2c78b04a45744995abbf6b259fd8/greenlet-3.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:703cb211b820dbffbbc55a16bfc6e4583a6e6e990f33a119d2cc8b83211119c8", size = 238115, upload-time = "2026-04-27T12:21:48.845Z" }, @@ -1906,7 +1934,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/32/f2ce6d4cac3e55bc6173f92dbe627e782e1850f89d986c3606feb63aafa7/greenlet-3.5.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:db2910d3c809444e0a20147361f343fe2798e106af8d9d8506f5305302655a9f", size = 286228, upload-time = "2026-04-27T12:20:34.421Z" }, { url = "https://files.pythonhosted.org/packages/b7/aa/caed9e5adf742315fc7be2a84196373aab4816e540e38ba0d76cb7584d68/greenlet-3.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec9ea74e7268ace7f9aab1b1a4e730193fc661b39a993cd91c606c32d4a3628", size = 601775, upload-time = "2026-04-27T12:52:41.045Z" }, { url = "https://files.pythonhosted.org/packages/c7/af/90ae08497400a941595d12774447f752d3dfe0fbb012e35b76bc5c0ff37e/greenlet-3.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54d243512da35485fc7a6bf3c178fdda6327a9d6506fcdd62b1abd1e41b2927b", size = 614436, upload-time = "2026-04-27T12:59:41.595Z" }, + { url = "https://files.pythonhosted.org/packages/3f/e9/4eeadf8cb3403ac274245ba75f07844abc7fa5f6787583fc9156ba741e0f/greenlet-3.5.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:41353ec2ecedf7aa8f682753a41919f8718031a6edac46b8d3dc7ed9e1ceb136", size = 620610, upload-time = "2026-04-27T13:02:39.194Z" }, { url = "https://files.pythonhosted.org/packages/2b/e0/2e13df68f367e2f9960616927d60857dd7e56aaadd59a47c644216b2f920/greenlet-3.5.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d280a7f5c331622c69f97eb167f33577ff2d1df282c41cd15907fc0a3ca198c", size = 611388, upload-time = "2026-04-27T12:25:28.008Z" }, + { url = "https://files.pythonhosted.org/packages/ee/ef/f913b3c0eb7d26d86a2401c5e1546c9d46b657efee724b06f6f4ac5d8824/greenlet-3.5.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:58c1c374fe2b3d852f9b6b11a7dff4c85404e51b9a596fd9e89cf904eb09866d", size = 422775, upload-time = "2026-04-27T13:05:14.261Z" }, { url = "https://files.pythonhosted.org/packages/82/f7/393c64055132ac0d488ef6be549253b7e6274194863967ddc0bc8f5b87b8/greenlet-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1eb67d5adefb5bd2e182d42678a328979a209e4e82eb93575708185d31d1f588", size = 1570768, upload-time = "2026-04-27T12:53:28.099Z" }, { url = "https://files.pythonhosted.org/packages/b8/4b/eaf7735253522cf56d1b74d672a58f54fc114702ceaf05def59aae72f6e1/greenlet-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2628d6c86f6cb0cb45e0c3c54058bbec559f57eaae699447748cb3928150577e", size = 1635983, upload-time = "2026-04-27T12:25:26.903Z" }, { url = "https://files.pythonhosted.org/packages/4c/fe/4fb3a0805bd5165da5ebf858da7cc01cce8061674106d2cf5bdab32cbfde/greenlet-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:d4d9f0624c775f2dfc56ba54d515a8c771044346852a918b405914f6b19d7fd8", size = 238840, upload-time = "2026-04-27T12:23:54.806Z" }, @@ -1914,7 +1944,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/58/fc576f99037ce19c5aa16628e4c3226b6d1419f72a62c79f5f40576e6eb3/greenlet-3.5.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:5a5ed18de6a0f6cc7087f1563f6bd93fc7df1c19165ca01e9bde5a5dc281d106", size = 285066, upload-time = "2026-04-27T12:23:05.033Z" }, { url = "https://files.pythonhosted.org/packages/4a/ba/b28ddbe6bfad6a8ac196ef0e8cff37bc65b79735995b9e410923fffeeb70/greenlet-3.5.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a717fbc46d8a354fa675f7c1e813485b6ba3885f9bef0cd56e5ba27d758ff5b", size = 604414, upload-time = "2026-04-27T12:52:42.358Z" }, { url = "https://files.pythonhosted.org/packages/09/06/4b69f8f0b67603a8be2790e55107a190b376f2627fe0eaf5695d85ffb3cd/greenlet-3.5.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ddc090c5c1792b10246a78e8c2163ebbe04cf877f9d785c230a7b27b39ad038e", size = 617349, upload-time = "2026-04-27T12:59:43.32Z" }, + { url = "https://files.pythonhosted.org/packages/6a/15/a643b4ecd09969e30b8a150d5919960caae0abe4f5af75ab040b1ab85e78/greenlet-3.5.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4964101b8585c144cbda5532b1aa644255126c08a265dae90c16e7a0e63aaa9d", size = 623234, upload-time = "2026-04-27T13:02:40.611Z" }, { url = "https://files.pythonhosted.org/packages/8a/17/a3918541fd0ddefe024a69de6d16aa7b46d36ac19562adaa63c7fa180eff/greenlet-3.5.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2094acd54b272cb6eae8c03dd87b3fa1820a4cef18d6889c378d503500a1dc13", size = 613927, upload-time = "2026-04-27T12:25:30.28Z" }, + { url = "https://files.pythonhosted.org/packages/77/18/3b13d5ef1275b0ffaf933b05efa21408ac4ca95823c7411d79682e4fdcff/greenlet-3.5.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:7022615368890680e67b9965d33f5773aade330d5343bbe25560135aaa849eae", size = 425243, upload-time = "2026-04-27T13:05:15.689Z" }, { url = "https://files.pythonhosted.org/packages/ee/e1/bd0af6213c7dd33175d8a462d4c1fe1175124ebed4855bc1475a5b5242c2/greenlet-3.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5e05ba267789ea87b5a155cf0e810b1ab88bf18e9e8740813945ceb8ee4350ba", size = 1570893, upload-time = "2026-04-27T12:53:29.483Z" }, { url = "https://files.pythonhosted.org/packages/9b/2a/0789702f864f5382cb476b93d7a9c823c10472658102ccd65f415747d2e2/greenlet-3.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0ecec963079cd58cbd14723582384f11f166fd58883c15dcbfb342e0bc9b5846", size = 1636060, upload-time = "2026-04-27T12:25:28.845Z" }, { url = "https://files.pythonhosted.org/packages/b2/8f/22bf9df92bbff0eb07842b60f7e63bf7675a9742df628437a9f02d09137f/greenlet-3.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:728d9667d8f2f586644b748dbd9bb67e50d6a9381767d1357714ea6825bb3bf5", size = 238740, upload-time = "2026-04-27T12:24:01.341Z" }, @@ -1922,7 +1954,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/5e/a70f31e3e8d961c4ce589c15b28e4225d63704e431a23932a3808cbcc867/greenlet-3.5.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:f35807464c4c58c55f0d31dfa83c541a5615d825c2fe3d2b95360cf7c4e3c0a8", size = 285564, upload-time = "2026-04-27T12:23:08.555Z" }, { url = "https://files.pythonhosted.org/packages/af/a6/046c0a28e21833e4086918218cfb3d8bed51c075a1b700f20b9d7861c0f4/greenlet-3.5.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55fa7ea52771be44af0de27d8b80c02cd18c2c3cddde6c847ecebdf72418b6a1", size = 651166, upload-time = "2026-04-27T12:52:43.644Z" }, { url = "https://files.pythonhosted.org/packages/47/f8/4af27f71c5ff32a7fbc516adb46370d9c4ae2bc7bd3dc7d066ac542b4b15/greenlet-3.5.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a97e4821aa710603f94de0da25f25096454d78ffdace5dc77f3a006bc01abba3", size = 663792, upload-time = "2026-04-27T12:59:44.93Z" }, + { url = "https://files.pythonhosted.org/packages/fb/89/2dadb89793c37ee8b4c237857188293e9060dc085f19845c292e00f8e091/greenlet-3.5.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf2d8a80bec89ab46221ae45c5373d5ba0bd36c19aa8508e85c6cd7e5106cd37", size = 668086, upload-time = "2026-04-27T13:02:42.314Z" }, { url = "https://files.pythonhosted.org/packages/a3/59/1bd6d7428d6ed9106efbb8c52310c60fd04f6672490f452aeaa3829aa436/greenlet-3.5.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f52a464e4ed91780bdfbbdd2b97197f3accaa629b98c200f4dffada759f3ae7", size = 660933, upload-time = "2026-04-27T12:25:33.276Z" }, + { url = "https://files.pythonhosted.org/packages/82/35/75722be7e26a2af4cbd2dc35b0ed382dacf9394b7e75551f76ed1abe87f2/greenlet-3.5.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:1bae92a1dd94c5f9d9493c3a212dd874c202442047cf96446412c862feca83a2", size = 470799, upload-time = "2026-04-27T13:05:17.094Z" }, { url = "https://files.pythonhosted.org/packages/83/e4/b903e5a5fae1e8a28cdd32a0cfbfd560b668c25b692f67768822ddc5f40f/greenlet-3.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:762612baf1161ccb8437c0161c668a688223cba28e1bf038f4eb47b13e39ccdf", size = 1618401, upload-time = "2026-04-27T12:53:31.062Z" }, { url = "https://files.pythonhosted.org/packages/0e/e3/5ec408a329acb854fb607a122e1ee5fb3ff649f9a97952948a90803c0d8e/greenlet-3.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:57a43c6079a89713522bc4bcb9f75070ecf5d3dbad7792bfe42239362cbf2a16", size = 1682038, upload-time = "2026-04-27T12:25:31.838Z" }, { url = "https://files.pythonhosted.org/packages/91/20/6b165108058767ee643c55c5c4904d591a830ee2b3c7dbd359828fbc829f/greenlet-3.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:3bc59be3945ae9750b9e7d45067d01ae3fe90ea5f9ade99239dabdd6e28a5033", size = 239835, upload-time = "2026-04-27T12:24:54.136Z" }, @@ -1930,7 +1964,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/a8/4522939255bb5409af4e87132f915446bf3622c2c292d14d3c38d128ae82/greenlet-3.5.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:a10a732421ab4fec934783ce3e54763470d0181db6e3468f9103a275c3ed1853", size = 293614, upload-time = "2026-04-27T12:24:12.874Z" }, { url = "https://files.pythonhosted.org/packages/15/5e/8744c52e2c027b5a8772a01561934c8835f869733e101f62075c60430340/greenlet-3.5.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7fc391b1566f2907d17aaebe78f8855dc45675159a775fcf9e61f8ee0078e87f", size = 650723, upload-time = "2026-04-27T12:52:45.412Z" }, { url = "https://files.pythonhosted.org/packages/00/ef/7b4c39c03cf46ceca512c5d3f914afd85aa30b2cc9a93015b0dd73e4be6c/greenlet-3.5.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:680bd0e7ad5e8daa8a4aa89f68fd6adc834b8a8036dc256533f7e08f4a4b01f7", size = 656529, upload-time = "2026-04-27T12:59:46.295Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5c/0602239503b124b70e39355cbdb39361ecfe65b87a5f2f63752c32f5286f/greenlet-3.5.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1aa4ce8debcd4ea7fb2e150f3036588c41493d1d52c43538924ae1819003f4ce", size = 657015, upload-time = "2026-04-27T13:02:43.973Z" }, { url = "https://files.pythonhosted.org/packages/0b/b5/c7768f352f5c010f92064d0063f987e7dc0cd290a6d92a34109015ce4aa1/greenlet-3.5.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ddb36c7d6c9c0a65f18c7258634e0c416c6ab59caac8c987b96f80c2ebda0112", size = 654364, upload-time = "2026-04-27T12:25:35.64Z" }, + { url = "https://files.pythonhosted.org/packages/38/51/8699f865f125dc952384cb432b0f7138aa4d8f2969a7d12d0df5b94d054d/greenlet-3.5.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:728a73687e39ae9ca34e4694cbf2f049d3fbc7174639468d0f67200a97d8f9e2", size = 488275, upload-time = "2026-04-27T13:05:18.28Z" }, { url = "https://files.pythonhosted.org/packages/ef/d0/079ebe12e4b1fc758857ce5be1a5e73f06870f2101e52611d1e71925ce54/greenlet-3.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e5ddf316ced87539144621453c3aef229575825fe60c604e62bedc4003f372b2", size = 1614204, upload-time = "2026-04-27T12:53:32.618Z" }, { url = "https://files.pythonhosted.org/packages/6d/89/6c2fb63df3596552d20e58fb4d96669243388cf680cff222758812c7bfaa/greenlet-3.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4a448128607be0de65342dc9b31be7f948ef4cc0bc8832069350abefd310a8f2", size = 1675480, upload-time = "2026-04-27T12:25:34.168Z" }, { url = "https://files.pythonhosted.org/packages/15/32/77ee8a6c1564fc345a491a4e85b3bf360e4cf26eac98c4532d2fdb96e01f/greenlet-3.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d60097128cb0a1cab9ea541186ea13cd7b847b8449a7787c2e2350da0cb82d86", size = 245324, upload-time = "2026-04-27T12:24:40.295Z" }, @@ -4326,6 +4362,10 @@ moondream = [ { name = "timm" }, { name = "transformers" }, ] +moq = [ + { name = "aioquic" }, + { name = "cryptography" }, +] neuphonic = [ { name = "websockets" }, ] @@ -4446,6 +4486,7 @@ requires-dist = [ { name = "aioboto3", marker = "extra == 'aws'", specifier = ">=15.5.0,<16" }, { name = "aiofiles", specifier = ">=24.1.0,<27" }, { name = "aiohttp", specifier = ">=3.11.12,<4" }, + { name = "aioquic", marker = "extra == 'moq'", specifier = ">=1.2.0,<2" }, { name = "aiortc", marker = "extra == 'webrtc'", specifier = ">=1.14.0,<2" }, { name = "anthropic", marker = "extra == 'anthropic'", specifier = ">=0.49.0,<1" }, { name = "audioop-lts", marker = "python_full_version >= '3.13'", specifier = "~=0.2.1" }, @@ -4454,6 +4495,7 @@ requires-dist = [ { name = "azure-cognitiveservices-speech", marker = "extra == 'azure'", specifier = ">=1.47.0,<2" }, { name = "camb-sdk", marker = "extra == 'camb'", specifier = ">=1.5.4,<2" }, { name = "coremltools", marker = "extra == 'local-smart-turn'", specifier = ">=8.0" }, + { name = "cryptography", marker = "extra == 'moq'", specifier = ">=43.0.0" }, { name = "daily-python", marker = "extra == 'daily'", specifier = "~=0.28.0" }, { name = "deepgram-sdk", marker = "extra == 'deepgram'", specifier = ">=6.1.1,<7" }, { name = "docstring-parser", specifier = ">=0.16,<1" }, @@ -4550,7 +4592,7 @@ requires-dist = [ { name = "wait-for2", marker = "python_full_version < '3.12'", specifier = ">=0.4.1,<1" }, { name = "websockets", marker = "extra == 'websockets-base'", specifier = ">=13.1,<16.0" }, ] -provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "camb", "cerebras", "daily", "deepgram", "deepseek", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "gradium", "grok", "groq", "gstreamer", "heygen", "hume", "inworld", "koala", "kokoro", "langchain", "lemonslice", "livekit", "lmnt", "local", "local-smart-turn", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "nebius", "neuphonic", "novita", "nvidia", "openai", "rnnoise", "openrouter", "perplexity", "piper", "qwen", "resembleai", "rime", "runner", "sagemaker", "sambanova", "sarvam", "sentry", "silero", "simli", "smallest", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper", "xai"] +provides-extras = ["aic", "anthropic", "assemblyai", "asyncai", "aws", "aws-nova-sonic", "azure", "cartesia", "camb", "cerebras", "daily", "deepgram", "deepseek", "elevenlabs", "fal", "fireworks", "fish", "gladia", "google", "gradium", "grok", "groq", "gstreamer", "heygen", "hume", "inworld", "koala", "kokoro", "langchain", "lemonslice", "livekit", "lmnt", "local", "local-smart-turn", "mcp", "mem0", "mistral", "mlx-whisper", "moondream", "moq", "nebius", "neuphonic", "novita", "nvidia", "openai", "rnnoise", "openrouter", "perplexity", "piper", "qwen", "resembleai", "rime", "runner", "sagemaker", "sambanova", "sarvam", "sentry", "silero", "simli", "smallest", "soniox", "soundfile", "speechmatics", "strands", "tavus", "together", "tracing", "ultravox", "webrtc", "websocket", "websockets-base", "whisper", "xai"] [package.metadata.requires-dev] dev = [ @@ -5148,6 +5190,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/b6/65a49a05614b2548edbba3aab118f2ebe7441dfd778accdcdce9f6567f20/pyloudnorm-0.2.0-py3-none-any.whl", hash = "sha256:9bb69afb904f59d007a7f9ba3d75d16fb8aeef35c44d6df822a9f192d69cf13f", size = 10879, upload-time = "2026-01-04T11:43:34.534Z" }, ] +[[package]] +name = "pylsqpack" +version = "0.3.24" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/a0/20b34e654b911a9abb736b242cc0a11912bc79ea3e911f139ea756e39ea2/pylsqpack-0.3.24.tar.gz", hash = "sha256:8ec455f44614228f89e38d40c1b1e37895620e20ec6b21e3b562fa8b79a23890", size = 677187, upload-time = "2026-03-29T15:42:40.136Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/88/71b79d334f67dd595fbed5f3a337e2aa997a96e452bb1b64120bccf5679d/pylsqpack-0.3.24-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:8edf48d0a023cd3629b2c4aaccac9b79a46d566c0f61e7416b5678228433763d", size = 162525, upload-time = "2026-03-29T15:42:25.436Z" }, + { url = "https://files.pythonhosted.org/packages/4e/96/f0a7625075394e93db42bd476abb7240ff1a474acd1ad404158baf68dc6a/pylsqpack-0.3.24-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e7d956dbc8f7d597b237b9157d0a16bc7c655a1b031239763c18dc8582aff8cc", size = 168643, upload-time = "2026-03-29T15:42:26.744Z" }, + { url = "https://files.pythonhosted.org/packages/42/de/49ec59856ea41468ed879ec143fc429729e37e4860b2119959a2a66fb652/pylsqpack-0.3.24-cp310-abi3-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:b6a8bb42127d5ece8d301a673c8205df25b73b69f8c46b9f0c3034588de1789a", size = 246930, upload-time = "2026-03-29T15:42:28.136Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d3/3e748fa5317782bfe68a7eaf890524aee48281c59f07e9bdfd7774f158db/pylsqpack-0.3.24-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3f977d419c60c1d6c2240e6d7a52df820d37eb8c36b4057113bcd7859f53e2c", size = 249234, upload-time = "2026-03-29T15:42:29.583Z" }, + { url = "https://files.pythonhosted.org/packages/22/5b/06f5e354ed882ce036ed65f2a393c98d0f6c71a23fa64b53251ddeb40a7b/pylsqpack-0.3.24-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6024854eb16d32803d4890fb90a73b9348c74b61c0770680aefaaa75f8456e8c", size = 250274, upload-time = "2026-03-29T15:42:31.03Z" }, + { url = "https://files.pythonhosted.org/packages/61/0e/c95cae2817a5c272b7a3132376165aa16875efcccbbd3e6608f5082770cc/pylsqpack-0.3.24-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:54978a9879471596d84bbad5e67d727014048926bc5bb2dac0eb3701b48c5ac9", size = 246966, upload-time = "2026-03-29T15:42:32.035Z" }, + { url = "https://files.pythonhosted.org/packages/40/fe/d5e84c3b4b2fa716df9e95aeb40d3bfb4de50c21cccccd66e194cfc084ac/pylsqpack-0.3.24-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:caf63ddc2e581c764d17432893acce02c5c29ff879d77c2abf1e26aa4eeb831b", size = 246546, upload-time = "2026-03-29T15:42:33.105Z" }, + { url = "https://files.pythonhosted.org/packages/65/f5/88e442ced83c0305f50f45bf521bbce3344ef0c29c3442f010086ff0c124/pylsqpack-0.3.24-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e3dc5f146fd456b50b227858aed59faa0ff8445aa426e69bb4e50d46c487aab0", size = 248517, upload-time = "2026-03-29T15:42:34.237Z" }, + { url = "https://files.pythonhosted.org/packages/a6/c2/886348974bba20db2a80cf37e97203d7334223b3c1c1babe4159dd12626d/pylsqpack-0.3.24-cp310-abi3-win32.whl", hash = "sha256:8da12be7b35b7c9a8cf73a4c077f72e5022a311f80a401c79904213376f2d767", size = 153483, upload-time = "2026-03-29T15:42:35.214Z" }, + { url = "https://files.pythonhosted.org/packages/0d/22/adbce7adfb41b8f5f222195f7f4f5e58655aa3e83f525bc5f3882b07d6e8/pylsqpack-0.3.24-cp310-abi3-win_amd64.whl", hash = "sha256:c3e2327af25ee616ce4483a8748f0957cf017cbca82d58ed15efea68f70f94ff", size = 156145, upload-time = "2026-03-29T15:42:36.902Z" }, + { url = "https://files.pythonhosted.org/packages/a5/2e/6fb6d797ce88741a0e18984bbab69160abc0971a41f4478cab6c8255a8dc/pylsqpack-0.3.24-cp310-abi3-win_arm64.whl", hash = "sha256:23b4d8af48836893beac356c10ca268161953de5bf9ed691526a93f5c82433e9", size = 153424, upload-time = "2026-03-29T15:42:38.73Z" }, +] + [[package]] name = "pyopenssl" version = "26.2.0" @@ -6022,6 +6083,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/29/41/f2b800b7f12a05dd48c2a6280d4dd812d1425fc66ed3fe3fd99420c41d1a/sentry_sdk-2.60.0-py3-none-any.whl", hash = "sha256:28a536c03291c8bcb363cf35c611b32738ec118ff64d8d6383b096448ac4c803", size = 475616, upload-time = "2026-05-13T13:34:50.259Z" }, ] +[[package]] +name = "service-identity" +version = "24.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "cryptography" }, + { name = "pyasn1" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/a5/dfc752b979067947261dbbf2543470c58efe735c3c1301dd870ef27830ee/service_identity-24.2.0.tar.gz", hash = "sha256:b8683ba13f0d39c6cd5d625d2c5f65421d6d707b013b375c355751557cbe8e09", size = 39245, upload-time = "2024-10-26T07:21:57.736Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/2c/ca6dd598b384bc1ce581e24aaae0f2bed4ccac57749d5c3befbb5e742081/service_identity-24.2.0-py3-none-any.whl", hash = "sha256:6b047fbd8a84fd0bb0d55ebce4031e400562b9196e1e0d3e0fe2b8a59f6d4a85", size = 11364, upload-time = "2024-10-26T07:21:56.302Z" }, +] + [[package]] name = "setuptools" version = "78.1.1"