/** * Deixis — vanilla JS client. * * Same base wiring as pointing (PipecatClient + * managed snapshot streaming + bot audio sink), with three command * handlers: ``scroll_to``, ``highlight``, and ``select_text``. * * The interesting one is ``select_text``: it puts the OS-level text * selection on the referenced element, so when the agent says * "this paragraph here" the user sees exactly which paragraph it * means. The READ direction (user selection) flows the other way — * Managed snapshot streaming automatically captures * ``window.getSelection()`` and emits a ``... * `` block in the snapshot the server sees. */ import { PipecatClient, RTVIEvent, findElementByRef, } from "@pipecat-ai/client-js"; import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport"; const BOT_URL = "http://localhost:7860/api/offer"; const connectButton = document.getElementById("connect"); const status = document.getElementById("status"); const botAudio = document.getElementById("bot-audio"); let client; let unsubscribes = []; function setStatus(text, autoHideMs = 0) { status.textContent = text; status.dataset.show = text ? "1" : "0"; if (text && autoHideMs > 0) { setTimeout(() => { if (status.textContent === text) status.dataset.show = "0"; }, autoHideMs); } } function resolveTarget(payload) { if (payload?.ref) { const el = findElementByRef(payload.ref); if (el) return el; } if (payload?.target_id) { return document.getElementById(payload.target_id); } return null; } function handleScrollTo(payload) { const el = resolveTarget(payload); if (!el) return; const behavior = payload?.behavior === "instant" || payload?.behavior === "smooth" ? payload.behavior : "smooth"; el.scrollIntoView({ behavior, block: "center", inline: "nearest" }); } function handleHighlight(payload) { // Pointing's pulse style isn't appropriate for an article // (paragraphs don't want to scale). Use a brief background flash // by briefly setting a class. Highlight is used here mostly for // emphasizing single phrases the agent named — see CSS. const el = resolveTarget(payload); if (!el) return; el.classList.remove("flash"); void el.offsetWidth; el.classList.add("flash"); const duration = payload?.duration_ms ?? 1500; setTimeout(() => el.classList.remove("flash"), duration); } /** * Programmatically select an element's text on the page. * * Build a ``Range`` covering the element's children, replace the * window selection with it, and scroll the element into view. This * is the WRITE side of the deixis story: the agent says "this * paragraph" and the page shows the text actually selected. */ function handleSelectText(payload) { const el = resolveTarget(payload); if (!el) return; const range = document.createRange(); range.selectNodeContents(el); const sel = window.getSelection(); if (!sel) return; sel.removeAllRanges(); sel.addRange(range); // Scroll the selection into view if it isn't already, so the user // actually sees the agent's pointer. el.scrollIntoView({ behavior: "smooth", block: "center" }); } function onUICommand(command, handler) { const listener = (data) => { if (data.command !== command) return; handler(data.payload); }; client.on(RTVIEvent.UICommand, listener); return () => client.off(RTVIEvent.UICommand, listener); } async function connect() { connectButton.disabled = true; setStatus("Connecting…"); client = new PipecatClient({ transport: new SmallWebRTCTransport(), enableMic: true, enableCam: false, }); client.on(RTVIEvent.BotConnected, () => setStatus("Bot connected", 1500)); client.on(RTVIEvent.Disconnected, () => { setStatus("Disconnected", 2000); connectButton.dataset.state = ""; connectButton.textContent = "Connect"; connectButton.disabled = false; teardownUI(); }); client.on(RTVIEvent.TrackStarted, (track, participant) => { if (track.kind !== "audio") return; if (participant?.local) return; botAudio.srcObject = new MediaStream([track]); }); unsubscribes = [ onUICommand("scroll_to", handleScrollTo), onUICommand("highlight", handleHighlight), onUICommand("select_text", handleSelectText), ]; try { await client.connect({ webrtcUrl: BOT_URL }); client.startUISnapshotStream(); connectButton.dataset.state = "connected"; connectButton.textContent = "Disconnect"; connectButton.disabled = false; setStatus("Connected. Try selecting a paragraph and asking 'explain this.'", 5000); } catch (err) { console.error("Connect failed:", err); setStatus(`Connect failed: ${err.message ?? err}`, 4000); teardownUI(); connectButton.disabled = false; } } async function disconnect() { connectButton.disabled = true; setStatus("Disconnecting…"); try { await client?.disconnect(); } finally { teardownUI(); connectButton.dataset.state = ""; connectButton.textContent = "Connect"; connectButton.disabled = false; } } function teardownUI() { client?.stopUISnapshotStream(); unsubscribes.forEach((unsubscribe) => unsubscribe()); unsubscribes = []; if (botAudio.srcObject) botAudio.srcObject = null; client = undefined; } connectButton.addEventListener("click", () => { if (connectButton.dataset.state === "connected") { disconnect(); } else { connect(); } });