2254a8d0a2
Smallest UIWorker demo: a voice LLM in the main pipeline delegates screen-relevant utterances to a UIWorker via a respond job; the UIWorker auto-injects the current <ui_state> and answers grounded in what's on screen. Includes a vanilla-JS client that streams accessibility snapshots over RTVI.
111 lines
3.4 KiB
JavaScript
111 lines
3.4 KiB
JavaScript
/**
|
|
* Hello UIAgent — vanilla JS client.
|
|
*
|
|
* Wires three pieces of the SDK end to end:
|
|
* 1. PipecatClient + SmallWebRTCTransport for the voice session.
|
|
* 2. PipecatClient-managed accessibility snapshot streaming on
|
|
* every meaningful change (DOM mutations, focus, scroll-end,
|
|
* resize, visibility, selection).
|
|
*
|
|
* The agent has no tools — the snapshot is the entire input. The
|
|
* server's ``UIAgent`` auto-injects the latest ``<ui_state>`` block
|
|
* into the LLM context at the start of every turn, so the agent
|
|
* always answers grounded in what's currently on screen.
|
|
*/
|
|
|
|
import { PipecatClient, RTVIEvent } from "@pipecat-ai/client-js";
|
|
import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport";
|
|
|
|
const BOT_URL = "http://localhost:7860/api/offer";
|
|
|
|
const connectButton = document.getElementById("connect");
|
|
const status = document.getElementById("status");
|
|
const botAudio = document.getElementById("bot-audio");
|
|
|
|
let client;
|
|
|
|
function setStatus(text, autoHideMs = 0) {
|
|
status.textContent = text;
|
|
status.dataset.show = text ? "1" : "0";
|
|
if (text && autoHideMs > 0) {
|
|
setTimeout(() => {
|
|
if (status.textContent === text) status.dataset.show = "0";
|
|
}, autoHideMs);
|
|
}
|
|
}
|
|
|
|
async function connect() {
|
|
connectButton.disabled = true;
|
|
setStatus("Connecting…");
|
|
|
|
// 1. Construct the Pipecat client with the WebRTC transport.
|
|
client = new PipecatClient({
|
|
transport: new SmallWebRTCTransport(),
|
|
enableMic: true,
|
|
enableCam: false,
|
|
});
|
|
|
|
client.on(RTVIEvent.BotConnected, () => setStatus("Bot connected", 1500));
|
|
client.on(RTVIEvent.Disconnected, () => {
|
|
setStatus("Disconnected", 2000);
|
|
connectButton.dataset.state = "";
|
|
connectButton.textContent = "Connect";
|
|
connectButton.disabled = false;
|
|
teardownUI();
|
|
});
|
|
|
|
// Pipe the bot's audio track into the <audio> sink so the user
|
|
// hears it. Without this, the WebRTC track is alive but never
|
|
// routed to a playback element. The React kit ships
|
|
// `PipecatClientAudio` to do the same thing — this is the vanilla
|
|
// equivalent.
|
|
client.on(RTVIEvent.TrackStarted, (track, participant) => {
|
|
if (track.kind !== "audio") return;
|
|
if (participant?.local) return;
|
|
botAudio.srcObject = new MediaStream([track]);
|
|
});
|
|
|
|
// 3. Connect to the bot.
|
|
try {
|
|
await client.connect({ webrtcUrl: BOT_URL });
|
|
// 2. Start the managed snapshot stream once the RTVI transport is ready.
|
|
client.startUISnapshotStream();
|
|
connectButton.dataset.state = "connected";
|
|
connectButton.textContent = "Disconnect";
|
|
connectButton.disabled = false;
|
|
setStatus("Connected. Try asking the agent what's on screen.", 4000);
|
|
} catch (err) {
|
|
console.error("Connect failed:", err);
|
|
setStatus(`Connect failed: ${err.message ?? err}`, 4000);
|
|
teardownUI();
|
|
connectButton.disabled = false;
|
|
}
|
|
}
|
|
|
|
async function disconnect() {
|
|
connectButton.disabled = true;
|
|
setStatus("Disconnecting…");
|
|
try {
|
|
await client?.disconnect();
|
|
} finally {
|
|
teardownUI();
|
|
connectButton.dataset.state = "";
|
|
connectButton.textContent = "Connect";
|
|
connectButton.disabled = false;
|
|
}
|
|
}
|
|
|
|
function teardownUI() {
|
|
client?.stopUISnapshotStream();
|
|
if (botAudio.srcObject) botAudio.srcObject = null;
|
|
client = undefined;
|
|
}
|
|
|
|
connectButton.addEventListener("click", () => {
|
|
if (connectButton.dataset.state === "connected") {
|
|
disconnect();
|
|
} else {
|
|
connect();
|
|
}
|
|
});
|