Add pointing UIWorker example

The voice LLM delegates to a ReplyToolMixin UIWorker that scrolls offscreen items into view and highlights the phones it names — exercising the scroll_to / highlight UI commands and the [offscreen] state tag.
2026-05-21 17:21:08 -04:00
parent 2254a8d0a2
commit 81b956d963
8 changed files with 1952 additions and 0 deletions
--- a/examples/multi-worker/ui-worker/pointing/client/index.html
+++ b/examples/multi-worker/ui-worker/pointing/client/index.html
@@ -0,0 +1,123 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Pointing — UIAgent demo</title>
+    <link rel="stylesheet" href="./styles.css" />
+  </head>
+  <body>
+    <header>
+      <h1>Phone catalog</h1>
+      <button id="connect" type="button">Connect</button>
+    </header>
+
+    <main aria-label="Phones">
+      <div class="phone-card" id="iphone-17">
+        <h2>iPhone 17</h2>
+        <p class="brand">Apple · 2025</p>
+        <p class="desc">Titanium frame, A19 chip, 48MP triple camera.</p>
+      </div>
+      <div class="phone-card" id="iphone-17-pro">
+        <h2>iPhone 17 Pro</h2>
+        <p class="brand">Apple · 2025</p>
+        <p class="desc">ProMotion 120Hz, A19 Pro, periscope zoom.</p>
+      </div>
+      <div class="phone-card" id="pixel-9">
+        <h2>Pixel 9</h2>
+        <p class="brand">Google · 2024</p>
+        <p class="desc">Tensor G4, 6.3-inch OLED, magic eraser.</p>
+      </div>
+      <div class="phone-card" id="pixel-9-pro">
+        <h2>Pixel 9 Pro</h2>
+        <p class="brand">Google · 2024</p>
+        <p class="desc">Triple lens, 5x telephoto, Tensor G4.</p>
+      </div>
+      <div class="phone-card" id="galaxy-s25">
+        <h2>Galaxy S25</h2>
+        <p class="brand">Samsung · 2025</p>
+        <p class="desc">Snapdragon 8 Gen 4, 200MP main camera.</p>
+      </div>
+      <div class="phone-card" id="galaxy-s25-ultra">
+        <h2>Galaxy S25 Ultra</h2>
+        <p class="brand">Samsung · 2025</p>
+        <p class="desc">S Pen, 6.9-inch Dynamic AMOLED, titanium.</p>
+      </div>
+      <div class="phone-card" id="oneplus-13">
+        <h2>OnePlus 13</h2>
+        <p class="brand">OnePlus · 2025</p>
+        <p class="desc">Hasselblad cameras, 100W charging, 6000mAh.</p>
+      </div>
+      <div class="phone-card" id="xiaomi-15">
+        <h2>Xiaomi 15</h2>
+        <p class="brand">Xiaomi · 2024</p>
+        <p class="desc">Leica optics, Snapdragon 8 Elite.</p>
+      </div>
+      <div class="phone-card" id="nothing-3">
+        <h2>Nothing Phone 3</h2>
+        <p class="brand">Nothing · 2025</p>
+        <p class="desc">Glyph interface, transparent back, MediaTek 8300.</p>
+      </div>
+      <div class="phone-card" id="motorola-edge-60">
+        <h2>Motorola Edge 60</h2>
+        <p class="brand">Motorola · 2025</p>
+        <p class="desc">Curved pOLED, 125W charging, slim 7.6mm body.</p>
+      </div>
+      <div class="phone-card" id="oppo-find-x8">
+        <h2>OPPO Find X8</h2>
+        <p class="brand">OPPO · 2024</p>
+        <p class="desc">Hasselblad triple zoom, MediaTek Dimensity 9400.</p>
+      </div>
+      <div class="phone-card" id="vivo-x200">
+        <h2>Vivo X200</h2>
+        <p class="brand">Vivo · 2024</p>
+        <p class="desc">Zeiss optics, 6000mAh battery, AMOLED.</p>
+      </div>
+      <div class="phone-card" id="honor-magic-7">
+        <h2>Honor Magic 7</h2>
+        <p class="brand">Honor · 2025</p>
+        <p class="desc">AI-driven imaging, 6.8-inch quad-curve display.</p>
+      </div>
+      <div class="phone-card" id="zenfone-12">
+        <h2>Zenfone 12</h2>
+        <p class="brand">ASUS · 2025</p>
+        <p class="desc">Compact 5.9-inch flagship, Snapdragon 8 Elite.</p>
+      </div>
+      <div class="phone-card" id="iphone-16e">
+        <h2>iPhone 16e</h2>
+        <p class="brand">Apple · 2025</p>
+        <p class="desc">Budget A18 chip, single-lens 48MP, USB-C.</p>
+      </div>
+      <div class="phone-card" id="pixel-9a">
+        <h2>Pixel 9a</h2>
+        <p class="brand">Google · 2025</p>
+        <p class="desc">Mid-range Tensor G4, 6.1-inch OLED.</p>
+      </div>
+      <div class="phone-card" id="galaxy-a56">
+        <h2>Galaxy A56</h2>
+        <p class="brand">Samsung · 2025</p>
+        <p class="desc">5,000mAh battery, Exynos 1580, mid-tier flagship.</p>
+      </div>
+      <div class="phone-card" id="redmi-note-14-pro">
+        <h2>Redmi Note 14 Pro</h2>
+        <p class="brand">Xiaomi · 2024</p>
+        <p class="desc">200MP camera, Dimensity 7300, AMOLED.</p>
+      </div>
+      <div class="phone-card" id="oneplus-nord-4">
+        <h2>OnePlus Nord 4</h2>
+        <p class="brand">OnePlus · 2024</p>
+        <p class="desc">Metal unibody, Snapdragon 7 Gen 3, 100W charging.</p>
+      </div>
+      <div class="phone-card" id="poco-f7">
+        <h2>POCO F7</h2>
+        <p class="brand">Xiaomi · 2025</p>
+        <p class="desc">Flagship-grade Snapdragon, 120Hz AMOLED, gamer focus.</p>
+      </div>
+    </main>
+
+    <div id="status" aria-live="polite"></div>
+    <audio id="bot-audio" autoplay data-a11y-exclude></audio>
+
+    <script type="module" src="./main.js"></script>
+  </body>
+</html>
--- a/examples/multi-worker/ui-worker/pointing/client/main.js
+++ b/examples/multi-worker/ui-worker/pointing/client/main.js
@@ -0,0 +1,170 @@
+/**
+ * Pointing — vanilla JS client.
+ *
+ * Builds on the hello-snapshot wiring (PipecatClient +
+ * managed snapshot streaming + bot audio sink) and adds two command
+ * handlers: ``scroll_to`` and ``highlight``. Both resolve the target
+ * element via ``findElementByRef`` (the snapshot ref system the
+ * walker assigns) and act on the live DOM node.
+ *
+ * The React SDK ships ``useDefaultScrollToHandler`` and
+ * ``useDefaultHighlightHandler`` that do this same work in hooks.
+ * Vanilla apps subscribe to ``RTVIEvent.UICommand`` and filter by
+ * command name.
+ */
+
+import {
+  PipecatClient,
+  RTVIEvent,
+  findElementByRef,
+} from "@pipecat-ai/client-js";
+import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport";
+
+const BOT_URL = "http://localhost:7860/api/offer";
+
+const connectButton = document.getElementById("connect");
+const status = document.getElementById("status");
+const botAudio = document.getElementById("bot-audio");
+
+let client;
+let unsubscribes = [];
+
+function setStatus(text, autoHideMs = 0) {
+  status.textContent = text;
+  status.dataset.show = text ? "1" : "0";
+  if (text && autoHideMs > 0) {
+    setTimeout(() => {
+      if (status.textContent === text) status.dataset.show = "0";
+    }, autoHideMs);
+  }
+}
+
+/**
+ * Resolve a payload that carries either ``ref`` (snapshot id) or
+ * ``target_id`` (DOM element id). Match what the standard React
+ * handlers do: prefer ref, fall back to target_id.
+ */
+function resolveTarget(payload) {
+  if (payload?.ref) {
+    const el = findElementByRef(payload.ref);
+    if (el) return el;
+  }
+  if (payload?.target_id) {
+    return document.getElementById(payload.target_id);
+  }
+  return null;
+}
+
+function handleScrollTo(payload) {
+  const el = resolveTarget(payload);
+  if (!el) return;
+  const behavior =
+    payload?.behavior === "instant" || payload?.behavior === "smooth"
+      ? payload.behavior
+      : "smooth";
+  el.scrollIntoView({ behavior, block: "center", inline: "nearest" });
+}
+
+function handleHighlight(payload) {
+  const el = resolveTarget(payload);
+  if (!el) return;
+  // The page CSS defines ``.ui-highlight`` as a keyframe pulse —
+  // scale + glow + tint, settling back. The animation duration is
+  // driven by the ``--highlight-duration`` CSS variable so the
+  // server-supplied ``duration_ms`` actually controls it.
+  const duration = payload?.duration_ms ?? 1500;
+  el.style.setProperty("--highlight-duration", `${duration}ms`);
+  // Re-trigger the animation cleanly if a previous highlight is
+  // still running on this element.
+  el.classList.remove("ui-highlight");
+  void el.offsetWidth; // force reflow so removing + re-adding restarts
+  el.classList.add("ui-highlight");
+  setTimeout(() => {
+    el.classList.remove("ui-highlight");
+    el.style.removeProperty("--highlight-duration");
+  }, duration);
+}
+
+function onUICommand(command, handler) {
+  const listener = (data) => {
+    if (data.command !== command) return;
+    handler(data.payload);
+  };
+  client.on(RTVIEvent.UICommand, listener);
+  return () => client.off(RTVIEvent.UICommand, listener);
+}
+
+async function connect() {
+  connectButton.disabled = true;
+  setStatus("Connecting…");
+
+  client = new PipecatClient({
+    transport: new SmallWebRTCTransport(),
+    enableMic: true,
+    enableCam: false,
+  });
+
+  client.on(RTVIEvent.BotConnected, () => setStatus("Bot connected", 1500));
+  client.on(RTVIEvent.Disconnected, () => {
+    setStatus("Disconnected", 2000);
+    connectButton.dataset.state = "";
+    connectButton.textContent = "Connect";
+    connectButton.disabled = false;
+    teardownUI();
+  });
+
+  // Pipe the bot's audio track into the <audio> sink.
+  client.on(RTVIEvent.TrackStarted, (track, participant) => {
+    if (track.kind !== "audio") return;
+    if (participant?.local) return;
+    botAudio.srcObject = new MediaStream([track]);
+  });
+
+  unsubscribes = [
+    onUICommand("scroll_to", handleScrollTo),
+    onUICommand("highlight", handleHighlight),
+  ];
+
+  try {
+    await client.connect({ webrtcUrl: BOT_URL });
+    client.startUISnapshotStream();
+    connectButton.dataset.state = "connected";
+    connectButton.textContent = "Disconnect";
+    connectButton.disabled = false;
+    setStatus("Connected. Try: 'where's the Pixel 9?'", 4000);
+  } catch (err) {
+    console.error("Connect failed:", err);
+    setStatus(`Connect failed: ${err.message ?? err}`, 4000);
+    teardownUI();
+    connectButton.disabled = false;
+  }
+}
+
+async function disconnect() {
+  connectButton.disabled = true;
+  setStatus("Disconnecting…");
+  try {
+    await client?.disconnect();
+  } finally {
+    teardownUI();
+    connectButton.dataset.state = "";
+    connectButton.textContent = "Connect";
+    connectButton.disabled = false;
+  }
+}
+
+function teardownUI() {
+  client?.stopUISnapshotStream();
+  unsubscribes.forEach((unsubscribe) => unsubscribe());
+  unsubscribes = [];
+  if (botAudio.srcObject) botAudio.srcObject = null;
+  client = undefined;
+}
+
+connectButton.addEventListener("click", () => {
+  if (connectButton.dataset.state === "connected") {
+    disconnect();
+  } else {
+    connect();
+  }
+});
--- a/examples/multi-worker/ui-worker/pointing/client/package-lock.json
+++ b/examples/multi-worker/ui-worker/pointing/client/package-lock.json
--- a/examples/multi-worker/ui-worker/pointing/client/package.json
+++ b/examples/multi-worker/ui-worker/pointing/client/package.json
@@ -0,0 +1,18 @@
+{
+  "name": "pointing-client",
+  "private": true,
+  "version": "0.1.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@pipecat-ai/client-js": "1.9.0",
+    "@pipecat-ai/small-webrtc-transport": "^1.10.2"
+  },
+  "devDependencies": {
+    "vite": "^8"
+  }
+}
--- a/examples/multi-worker/ui-worker/pointing/client/styles.css
+++ b/examples/multi-worker/ui-worker/pointing/client/styles.css
@@ -0,0 +1,151 @@
+:root {
+  color-scheme: light;
+  font-family:
+    system-ui,
+    -apple-system,
+    sans-serif;
+  --border: #d4d4d8;
+  --muted: #71717a;
+  --highlight: #fbbf24;
+}
+
+* {
+  box-sizing: border-box;
+}
+
+body {
+  margin: 0;
+  background: #fafafa;
+  color: #18181b;
+}
+
+header {
+  position: sticky;
+  top: 0;
+  z-index: 10;
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  padding: 1rem 1.5rem;
+  border-bottom: 1px solid var(--border);
+  background: #fff;
+}
+
+header h1 {
+  font-size: 1.25rem;
+  margin: 0;
+}
+
+#connect {
+  padding: 0.5rem 1rem;
+  border: 1px solid var(--border);
+  background: #fff;
+  border-radius: 6px;
+  cursor: pointer;
+  font-size: 0.875rem;
+}
+
+#connect:hover {
+  background: #f4f4f5;
+}
+
+#connect[data-state='connected'] {
+  background: #ef4444;
+  color: white;
+  border-color: #ef4444;
+}
+
+main {
+  display: grid;
+  grid-template-columns: repeat(auto-fill, minmax(280px, 1fr));
+  gap: 1rem;
+  padding: 1.5rem;
+  max-width: 1200px;
+  margin: 0 auto;
+}
+
+.phone-card {
+  background: #fff;
+  border: 1px solid var(--border);
+  border-radius: 8px;
+  padding: 1rem;
+  /* Tall enough that, combined with N rows, the bottom of the grid
+     scrolls below typical viewport heights — exercises offscreen
+     detection and scroll_to. */
+  min-height: 130px;
+  scroll-margin-top: 5rem;
+  /* Position relative + z-index so the lifted card during the
+     highlight pulse renders above its neighbors. */
+  position: relative;
+  z-index: 0;
+}
+
+.phone-card h2 {
+  margin: 0 0 0.25rem;
+  font-size: 1rem;
+}
+
+.phone-card .brand {
+  margin: 0 0 0.5rem;
+  font-size: 0.75rem;
+  color: var(--muted);
+  text-transform: uppercase;
+  letter-spacing: 0.04em;
+}
+
+.phone-card .desc {
+  margin: 0;
+  font-size: 0.875rem;
+  line-height: 1.45;
+  color: #3f3f46;
+}
+
+/* Highlight pulse: card scales up briefly, glows, tints, then
+   settles back. The animation duration is keyed off a CSS variable
+   so the handler can drive it from the server-supplied ``duration_ms``.
+   The visual restyle is entirely client-side — ``highlight`` is
+   just a command name, the page decides what "highlight" looks like. */
+@keyframes ui-highlight-pulse {
+  0% {
+    transform: scale(1);
+    box-shadow: 0 0 0 0 rgba(251, 191, 36, 0);
+    background: #fff;
+    border-color: var(--border);
+  }
+  35% {
+    transform: scale(1.04);
+    box-shadow: 0 0 36px 8px rgba(251, 191, 36, 0.55);
+    background: #fffbeb;
+    border-color: var(--highlight);
+  }
+  100% {
+    transform: scale(1);
+    box-shadow: 0 0 0 0 rgba(251, 191, 36, 0);
+    background: #fff;
+    border-color: var(--border);
+  }
+}
+
+.ui-highlight {
+  animation: ui-highlight-pulse var(--highlight-duration, 2500ms)
+    cubic-bezier(0.2, 0.8, 0.2, 1);
+  z-index: 2;
+}
+
+#status {
+  position: fixed;
+  bottom: 1rem;
+  right: 1rem;
+  padding: 0.5rem 0.75rem;
+  border-radius: 6px;
+  font-size: 0.8125rem;
+  background: #18181b;
+  color: white;
+  opacity: 0;
+  transition: opacity 0.2s;
+  pointer-events: none;
+}
+
+#status[data-show='1'] {
+  opacity: 1;
+}
--- a/examples/multi-worker/ui-worker/pointing/client/vite.config.js
+++ b/examples/multi-worker/ui-worker/pointing/client/vite.config.js
@@ -0,0 +1,7 @@
+import { defineConfig } from "vite";
+
+export default defineConfig({
+  server: {
+    port: 5173,
+  },
+});