Add hello-snapshot UIWorker example

Smallest UIWorker demo: a voice LLM in the main pipeline delegates screen-relevant utterances to a UIWorker via a respond job; the UIWorker auto-injects the current <ui_state> and answers grounded in what's on screen. Includes a vanilla-JS client that streams accessibility snapshots over RTVI.
2026-05-21 16:51:38 -04:00
parent f1f5a986e8
commit 2254a8d0a2
8 changed files with 1828 additions and 0 deletions
--- a/examples/multi-worker/ui-worker/hello-snapshot/client/index.html
+++ b/examples/multi-worker/ui-worker/hello-snapshot/client/index.html
@@ -0,0 +1,73 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Hello UIAgent</title>
+    <link rel="stylesheet" href="./styles.css" />
+  </head>
+  <body>
+    <header>
+      <h1>Local News Today</h1>
+      <button id="connect" type="button">Connect</button>
+    </header>
+
+    <main>
+      <section aria-label="Top stories">
+        <h2>Top stories</h2>
+
+        <article id="story-mediterranean" class="card">
+          <h3>Mediterranean diet linked to longer cognitive lifespan</h3>
+          <p>
+            A 12-year study following 50,000 adults across Italy and Spain
+            found that strict adherence to the Mediterranean diet
+            correlated with a 28% lower incidence of age-related cognitive
+            decline. Researchers attribute the effect to anti-inflammatory
+            compounds in olive oil and oily fish.
+          </p>
+        </article>
+
+        <article id="story-atacama" class="card">
+          <h3>Atacama solar field crosses 5 GW capacity</h3>
+          <p>
+            Chile's flagship desert solar project added another 600 MW of
+            generation this quarter, pushing total capacity past five
+            gigawatts. The expansion, completed three months ahead of
+            schedule, makes Atacama the largest solar generation site in
+            South America.
+          </p>
+        </article>
+
+        <article id="story-mars" class="card">
+          <h3>Mars sample return mission re-scoped to 2031</h3>
+          <p>
+            NASA and ESA jointly announced a revised timeline for the Mars
+            Sample Return mission, citing budget pressure and the need to
+            consolidate launch logistics. The first samples are now
+            expected back on Earth in late 2031, a two-year delay.
+          </p>
+        </article>
+      </section>
+
+      <aside aria-label="Trending tags">
+        <h2>Trending</h2>
+        <ul>
+          <li>climate</li>
+          <li>health</li>
+          <li>space</li>
+          <li>energy</li>
+          <li>longevity</li>
+        </ul>
+      </aside>
+    </main>
+
+    <div id="status" aria-live="polite"></div>
+
+    <!-- Bot audio sink. main.js attaches the bot's audio track to
+         this element on RTVIEvent.TrackStarted. Hidden via
+         data-a11y-exclude so it doesn't show up in the snapshot. -->
+    <audio id="bot-audio" autoplay data-a11y-exclude></audio>
+
+    <script type="module" src="./main.js"></script>
+  </body>
+</html>
--- a/examples/multi-worker/ui-worker/hello-snapshot/client/main.js
+++ b/examples/multi-worker/ui-worker/hello-snapshot/client/main.js
@@ -0,0 +1,110 @@
+/**
+ * Hello UIAgent — vanilla JS client.
+ *
+ * Wires three pieces of the SDK end to end:
+ *   1. PipecatClient + SmallWebRTCTransport for the voice session.
+ *   2. PipecatClient-managed accessibility snapshot streaming on
+ *      every meaningful change (DOM mutations, focus, scroll-end,
+ *      resize, visibility, selection).
+ *
+ * The agent has no tools — the snapshot is the entire input. The
+ * server's ``UIAgent`` auto-injects the latest ``<ui_state>`` block
+ * into the LLM context at the start of every turn, so the agent
+ * always answers grounded in what's currently on screen.
+ */
+
+import { PipecatClient, RTVIEvent } from "@pipecat-ai/client-js";
+import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport";
+
+const BOT_URL = "http://localhost:7860/api/offer";
+
+const connectButton = document.getElementById("connect");
+const status = document.getElementById("status");
+const botAudio = document.getElementById("bot-audio");
+
+let client;
+
+function setStatus(text, autoHideMs = 0) {
+  status.textContent = text;
+  status.dataset.show = text ? "1" : "0";
+  if (text && autoHideMs > 0) {
+    setTimeout(() => {
+      if (status.textContent === text) status.dataset.show = "0";
+    }, autoHideMs);
+  }
+}
+
+async function connect() {
+  connectButton.disabled = true;
+  setStatus("Connecting…");
+
+  // 1. Construct the Pipecat client with the WebRTC transport.
+  client = new PipecatClient({
+    transport: new SmallWebRTCTransport(),
+    enableMic: true,
+    enableCam: false,
+  });
+
+  client.on(RTVIEvent.BotConnected, () => setStatus("Bot connected", 1500));
+  client.on(RTVIEvent.Disconnected, () => {
+    setStatus("Disconnected", 2000);
+    connectButton.dataset.state = "";
+    connectButton.textContent = "Connect";
+    connectButton.disabled = false;
+    teardownUI();
+  });
+
+  // Pipe the bot's audio track into the <audio> sink so the user
+  // hears it. Without this, the WebRTC track is alive but never
+  // routed to a playback element. The React kit ships
+  // `PipecatClientAudio` to do the same thing — this is the vanilla
+  // equivalent.
+  client.on(RTVIEvent.TrackStarted, (track, participant) => {
+    if (track.kind !== "audio") return;
+    if (participant?.local) return;
+    botAudio.srcObject = new MediaStream([track]);
+  });
+
+  // 3. Connect to the bot.
+  try {
+    await client.connect({ webrtcUrl: BOT_URL });
+    // 2. Start the managed snapshot stream once the RTVI transport is ready.
+    client.startUISnapshotStream();
+    connectButton.dataset.state = "connected";
+    connectButton.textContent = "Disconnect";
+    connectButton.disabled = false;
+    setStatus("Connected. Try asking the agent what's on screen.", 4000);
+  } catch (err) {
+    console.error("Connect failed:", err);
+    setStatus(`Connect failed: ${err.message ?? err}`, 4000);
+    teardownUI();
+    connectButton.disabled = false;
+  }
+}
+
+async function disconnect() {
+  connectButton.disabled = true;
+  setStatus("Disconnecting…");
+  try {
+    await client?.disconnect();
+  } finally {
+    teardownUI();
+    connectButton.dataset.state = "";
+    connectButton.textContent = "Connect";
+    connectButton.disabled = false;
+  }
+}
+
+function teardownUI() {
+  client?.stopUISnapshotStream();
+  if (botAudio.srcObject) botAudio.srcObject = null;
+  client = undefined;
+}
+
+connectButton.addEventListener("click", () => {
+  if (connectButton.dataset.state === "connected") {
+    disconnect();
+  } else {
+    connect();
+  }
+});
--- a/examples/multi-worker/ui-worker/hello-snapshot/client/package-lock.json
+++ b/examples/multi-worker/ui-worker/hello-snapshot/client/package-lock.json
--- a/examples/multi-worker/ui-worker/hello-snapshot/client/package.json
+++ b/examples/multi-worker/ui-worker/hello-snapshot/client/package.json
@@ -0,0 +1,18 @@
+{
+  "name": "hello-snapshot-client",
+  "private": true,
+  "version": "0.1.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@pipecat-ai/client-js": "1.9.0",
+    "@pipecat-ai/small-webrtc-transport": "^1.10.2"
+  },
+  "devDependencies": {
+    "vite": "^8"
+  }
+}
--- a/examples/multi-worker/ui-worker/hello-snapshot/client/styles.css
+++ b/examples/multi-worker/ui-worker/hello-snapshot/client/styles.css
@@ -0,0 +1,129 @@
+:root {
+  color-scheme: light;
+  font-family: system-ui, -apple-system, sans-serif;
+  --border: #d4d4d8;
+  --muted: #71717a;
+}
+
+* {
+  box-sizing: border-box;
+}
+
+body {
+  margin: 0;
+  background: #fafafa;
+  color: #18181b;
+}
+
+header {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  padding: 1rem 1.5rem;
+  border-bottom: 1px solid var(--border);
+  background: #fff;
+}
+
+header h1 {
+  font-size: 1.25rem;
+  margin: 0;
+}
+
+#connect {
+  padding: 0.5rem 1rem;
+  border: 1px solid var(--border);
+  background: #fff;
+  border-radius: 6px;
+  cursor: pointer;
+  font-size: 0.875rem;
+}
+
+#connect:hover {
+  background: #f4f4f5;
+}
+
+#connect[data-state="connected"] {
+  background: #ef4444;
+  color: white;
+  border-color: #ef4444;
+}
+
+main {
+  display: grid;
+  grid-template-columns: 1fr 240px;
+  gap: 2rem;
+  padding: 1.5rem;
+  max-width: 1100px;
+  margin: 0 auto;
+}
+
+section,
+aside {
+  display: flex;
+  flex-direction: column;
+  gap: 1rem;
+}
+
+section h2,
+aside h2 {
+  font-size: 0.875rem;
+  text-transform: uppercase;
+  color: var(--muted);
+  letter-spacing: 0.04em;
+  margin: 0;
+}
+
+.card {
+  background: #fff;
+  border: 1px solid var(--border);
+  border-radius: 8px;
+  padding: 1rem;
+}
+
+.card h3 {
+  margin: 0 0 0.5rem;
+  font-size: 1rem;
+}
+
+.card p {
+  margin: 0;
+  font-size: 0.875rem;
+  line-height: 1.5;
+  color: #3f3f46;
+}
+
+aside ul {
+  list-style: none;
+  padding: 0;
+  margin: 0;
+  display: flex;
+  flex-direction: column;
+  gap: 0.5rem;
+}
+
+aside li {
+  font-size: 0.875rem;
+  padding: 0.375rem 0.75rem;
+  background: #fff;
+  border: 1px solid var(--border);
+  border-radius: 999px;
+  align-self: flex-start;
+}
+
+#status {
+  position: fixed;
+  bottom: 1rem;
+  right: 1rem;
+  padding: 0.5rem 0.75rem;
+  border-radius: 6px;
+  font-size: 0.8125rem;
+  background: #18181b;
+  color: white;
+  opacity: 0;
+  transition: opacity 0.2s;
+  pointer-events: none;
+}
+
+#status[data-show="1"] {
+  opacity: 1;
+}
--- a/examples/multi-worker/ui-worker/hello-snapshot/client/vite.config.js
+++ b/examples/multi-worker/ui-worker/hello-snapshot/client/vite.config.js
@@ -0,0 +1,7 @@
+import { defineConfig } from "vite";
+
+export default defineConfig({
+  server: {
+    port: 5173,
+  },
+});