Add form-fill UIWorker example

A ReplyToolMixin UIWorker that fills inputs (fills) and toggles checkboxes / presses submit (click) by voice — the state-changing half of the standard action set.
2026-05-21 17:21:08 -04:00
parent f826da9ac9
commit 6b0e204d66
8 changed files with 2004 additions and 0 deletions
--- a/examples/multi-worker/ui-worker/form-fill/client/index.html
+++ b/examples/multi-worker/ui-worker/form-fill/client/index.html
@@ -0,0 +1,88 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Form fill — UIAgent demo</title>
+    <link rel="stylesheet" href="./styles.css" />
+  </head>
+  <body>
+    <header>
+      <h1>Application form</h1>
+      <button id="connect" type="button">Connect</button>
+    </header>
+
+    <main>
+      <form id="application-form" aria-label="Job application">
+        <h2>Apply for: Software Engineer</h2>
+        <p class="hint">
+          Tell the assistant what to put in any field. Try: "my name
+          is John Smith, my email is john at gmail dot com". When
+          you're ready, say "submit".
+        </p>
+
+        <fieldset>
+          <legend>Your details</legend>
+
+          <div class="field">
+            <label for="first-name">First name</label>
+            <input id="first-name" name="first_name" type="text" autocomplete="given-name" />
+          </div>
+
+          <div class="field">
+            <label for="last-name">Last name</label>
+            <input id="last-name" name="last_name" type="text" autocomplete="family-name" />
+          </div>
+
+          <div class="field">
+            <label for="email">Email address</label>
+            <input id="email" name="email" type="email" autocomplete="email" />
+          </div>
+
+          <div class="field">
+            <label for="phone">Phone (optional)</label>
+            <input id="phone" name="phone" type="tel" autocomplete="tel" />
+          </div>
+        </fieldset>
+
+        <fieldset>
+          <legend>About you</legend>
+
+          <div class="field">
+            <label for="experience">Years of relevant experience</label>
+            <input id="experience" name="experience" type="text" inputmode="numeric" />
+          </div>
+
+          <div class="field">
+            <label for="cover">Why are you interested?</label>
+            <textarea id="cover" name="cover" rows="5"></textarea>
+          </div>
+        </fieldset>
+
+        <fieldset>
+          <legend>Confirm</legend>
+
+          <div class="field checkbox-field">
+            <input id="terms" name="terms" type="checkbox" />
+            <label for="terms">I agree to the terms of service.</label>
+          </div>
+
+          <div class="field checkbox-field">
+            <input id="newsletter" name="newsletter" type="checkbox" />
+            <label for="newsletter">Send me product updates.</label>
+          </div>
+        </fieldset>
+
+        <div class="actions">
+          <button id="submit" type="submit">Submit application</button>
+          <span id="form-status" aria-live="polite"></span>
+        </div>
+      </form>
+    </main>
+
+    <div id="status" aria-live="polite"></div>
+    <audio id="bot-audio" autoplay data-a11y-exclude></audio>
+
+    <script type="module" src="./main.js"></script>
+  </body>
+</html>
--- a/examples/multi-worker/ui-worker/form-fill/client/main.js
+++ b/examples/multi-worker/ui-worker/form-fill/client/main.js
@@ -0,0 +1,202 @@
+/**
+ * Form fill — vanilla JS client.
+ *
+ * Same base wiring as pointing/deixis (PipecatClient
+ * + managed snapshot streaming + bot audio sink). Three command
+ * handlers: ``scroll_to``, ``set_input_value``, and ``click``.
+ *
+ * ``set_input_value`` writes a string into an ``<input>`` /
+ * ``<textarea>``. Crucially it dispatches ``input`` and ``change``
+ * events so React-controlled or other frameworks pick up the change
+ * naturally; the React standard handler does the same. ``click``
+ * is the catch-all for checkboxes, radios, and submit buttons.
+ *
+ * The submit button intercepts the form's submit event so the demo
+ * stays on-page after the agent submits. Real apps would let it
+ * through.
+ */
+
+import {
+  PipecatClient,
+  RTVIEvent,
+  findElementByRef,
+} from "@pipecat-ai/client-js";
+import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport";
+
+const BOT_URL = "http://localhost:7860/api/offer";
+
+const connectButton = document.getElementById("connect");
+const status = document.getElementById("status");
+const botAudio = document.getElementById("bot-audio");
+const form = document.getElementById("application-form");
+const formStatus = document.getElementById("form-status");
+
+let client;
+let unsubscribes = [];
+
+function setStatus(text, autoHideMs = 0) {
+  status.textContent = text;
+  status.dataset.show = text ? "1" : "0";
+  if (text && autoHideMs > 0) {
+    setTimeout(() => {
+      if (status.textContent === text) status.dataset.show = "0";
+    }, autoHideMs);
+  }
+}
+
+function resolveTarget(payload) {
+  if (payload?.ref) {
+    const el = findElementByRef(payload.ref);
+    if (el) return el;
+  }
+  if (payload?.target_id) {
+    return document.getElementById(payload.target_id);
+  }
+  return null;
+}
+
+function handleScrollTo(payload) {
+  const el = resolveTarget(payload);
+  if (!el) return;
+  const behavior =
+    payload?.behavior === "instant" || payload?.behavior === "smooth"
+      ? payload.behavior
+      : "smooth";
+  el.scrollIntoView({ behavior, block: "center", inline: "nearest" });
+}
+
+/**
+ * Write ``payload.value`` into the targeted input/textarea.
+ *
+ * Skips ``disabled`` / ``readonly`` / ``type="hidden"`` targets so
+ * the agent can't bypass UI affordances. Dispatches ``input`` and
+ * ``change`` events so framework-controlled inputs (React, Vue, etc.)
+ * notice the change. Briefly flashes the field so the user sees
+ * what the agent wrote.
+ */
+function handleSetInputValue(payload) {
+  const el = resolveTarget(payload);
+  if (!el) return;
+  if (!(el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement))
+    return;
+  if (el.disabled || el.readOnly) return;
+  if (el.type === "hidden") return;
+
+  const value = String(payload?.value ?? "");
+  const replace = payload?.replace !== false;
+  el.value = replace ? value : (el.value || "") + value;
+
+  el.dispatchEvent(new Event("input", { bubbles: true }));
+  el.dispatchEvent(new Event("change", { bubbles: true }));
+
+  // Visual confirmation: a brief background flash so the user
+  // notices the write happened.
+  el.classList.remove("fill-flash");
+  void el.offsetWidth;
+  el.classList.add("fill-flash");
+  setTimeout(() => el.classList.remove("fill-flash"), 1200);
+}
+
+/**
+ * Click the targeted element. Skips ``disabled`` targets so the
+ * agent can't bypass disabled affordances; the standard React
+ * handler does the same.
+ */
+function handleClick(payload) {
+  const el = resolveTarget(payload);
+  if (!el) return;
+  if ("disabled" in el && el.disabled) return;
+  el.click();
+}
+
+// Don't actually submit the form on the demo; the agent says "I
+// submitted it" and we show a status message instead.
+form.addEventListener("submit", (e) => {
+  e.preventDefault();
+  formStatus.textContent = "Submitted (demo only — no network call).";
+  formStatus.style.color = "#16a34a";
+});
+
+function onUICommand(command, handler) {
+  const listener = (data) => {
+    if (data.command !== command) return;
+    handler(data.payload);
+  };
+  client.on(RTVIEvent.UICommand, listener);
+  return () => client.off(RTVIEvent.UICommand, listener);
+}
+
+async function connect() {
+  connectButton.disabled = true;
+  setStatus("Connecting…");
+
+  client = new PipecatClient({
+    transport: new SmallWebRTCTransport(),
+    enableMic: true,
+    enableCam: false,
+  });
+
+  client.on(RTVIEvent.BotConnected, () => setStatus("Bot connected", 1500));
+  client.on(RTVIEvent.Disconnected, () => {
+    setStatus("Disconnected", 2000);
+    connectButton.dataset.state = "";
+    connectButton.textContent = "Connect";
+    connectButton.disabled = false;
+    teardownUI();
+  });
+
+  client.on(RTVIEvent.TrackStarted, (track, participant) => {
+    if (track.kind !== "audio") return;
+    if (participant?.local) return;
+    botAudio.srcObject = new MediaStream([track]);
+  });
+
+  unsubscribes = [
+    onUICommand("scroll_to", handleScrollTo),
+    onUICommand("set_input_value", handleSetInputValue),
+    onUICommand("click", handleClick),
+  ];
+
+  try {
+    await client.connect({ webrtcUrl: BOT_URL });
+    client.startUISnapshotStream();
+    connectButton.dataset.state = "connected";
+    connectButton.textContent = "Disconnect";
+    connectButton.disabled = false;
+    setStatus("Connected. Try: 'My name is John Smith'", 5000);
+  } catch (err) {
+    console.error("Connect failed:", err);
+    setStatus(`Connect failed: ${err.message ?? err}`, 4000);
+    teardownUI();
+    connectButton.disabled = false;
+  }
+}
+
+async function disconnect() {
+  connectButton.disabled = true;
+  setStatus("Disconnecting…");
+  try {
+    await client?.disconnect();
+  } finally {
+    teardownUI();
+    connectButton.dataset.state = "";
+    connectButton.textContent = "Connect";
+    connectButton.disabled = false;
+  }
+}
+
+function teardownUI() {
+  client?.stopUISnapshotStream();
+  unsubscribes.forEach((unsubscribe) => unsubscribe());
+  unsubscribes = [];
+  if (botAudio.srcObject) botAudio.srcObject = null;
+  client = undefined;
+}
+
+connectButton.addEventListener("click", () => {
+  if (connectButton.dataset.state === "connected") {
+    disconnect();
+  } else {
+    connect();
+  }
+});
--- a/examples/multi-worker/ui-worker/form-fill/client/package-lock.json
+++ b/examples/multi-worker/ui-worker/form-fill/client/package-lock.json
--- a/examples/multi-worker/ui-worker/form-fill/client/package.json
+++ b/examples/multi-worker/ui-worker/form-fill/client/package.json
@@ -0,0 +1,18 @@
+{
+  "name": "form-fill-client",
+  "private": true,
+  "version": "0.1.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@pipecat-ai/client-js": "1.9.0",
+    "@pipecat-ai/small-webrtc-transport": "^1.10.2"
+  },
+  "devDependencies": {
+    "vite": "^8"
+  }
+}
--- a/examples/multi-worker/ui-worker/form-fill/client/styles.css
+++ b/examples/multi-worker/ui-worker/form-fill/client/styles.css
@@ -0,0 +1,207 @@
+:root {
+  color-scheme: light;
+  font-family: system-ui, -apple-system, sans-serif;
+  --border: #d4d4d8;
+  --border-focus: #3b82f6;
+  --muted: #71717a;
+  --bg: #fafafa;
+}
+
+* {
+  box-sizing: border-box;
+}
+
+body {
+  margin: 0;
+  background: var(--bg);
+  color: #18181b;
+}
+
+header {
+  position: sticky;
+  top: 0;
+  z-index: 10;
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  padding: 1rem 1.5rem;
+  border-bottom: 1px solid var(--border);
+  background: #fff;
+}
+
+header h1 {
+  font-size: 1.125rem;
+  margin: 0;
+}
+
+#connect {
+  padding: 0.5rem 1rem;
+  border: 1px solid var(--border);
+  background: #fff;
+  border-radius: 6px;
+  cursor: pointer;
+  font-size: 0.875rem;
+}
+
+#connect:hover {
+  background: #f4f4f5;
+}
+
+#connect[data-state="connected"] {
+  background: #ef4444;
+  color: white;
+  border-color: #ef4444;
+}
+
+main {
+  max-width: 640px;
+  margin: 0 auto;
+  padding: 2rem 1.5rem 4rem;
+}
+
+form h2 {
+  margin: 0 0 0.5rem;
+  font-size: 1.5rem;
+}
+
+.hint {
+  margin: 0 0 1.5rem;
+  font-size: 0.9375rem;
+  line-height: 1.5;
+  color: var(--muted);
+}
+
+fieldset {
+  margin: 0 0 1.5rem;
+  padding: 1rem 1.25rem 1.25rem;
+  border: 1px solid var(--border);
+  border-radius: 8px;
+  background: #fff;
+}
+
+legend {
+  padding: 0 0.5rem;
+  font-size: 0.8125rem;
+  font-weight: 600;
+  text-transform: uppercase;
+  letter-spacing: 0.04em;
+  color: var(--muted);
+}
+
+.field {
+  display: flex;
+  flex-direction: column;
+  gap: 0.25rem;
+  margin-top: 0.875rem;
+}
+
+.field:first-child {
+  margin-top: 0;
+}
+
+.field label {
+  font-size: 0.875rem;
+  font-weight: 500;
+  color: #27272a;
+}
+
+.field input[type="text"],
+.field input[type="email"],
+.field input[type="tel"],
+.field textarea {
+  font: inherit;
+  font-size: 1rem;
+  padding: 0.5rem 0.625rem;
+  border: 1px solid var(--border);
+  border-radius: 6px;
+  background: #fff;
+  width: 100%;
+  scroll-margin-top: 5rem;
+  transition:
+    border-color 0.15s,
+    box-shadow 0.15s,
+    background 0.4s;
+}
+
+.field input:focus,
+.field textarea:focus {
+  outline: none;
+  border-color: var(--border-focus);
+  box-shadow: 0 0 0 3px rgba(59, 130, 246, 0.2);
+}
+
+/* Brief flash when the agent fills a field, so the user notices the
+   write. The CSS class is added by the set_input_value handler and
+   removed after the animation. */
+@keyframes field-fill-flash {
+  0% {
+    background: #fef3c7;
+  }
+  100% {
+    background: #fff;
+  }
+}
+
+.field input.fill-flash,
+.field textarea.fill-flash {
+  animation: field-fill-flash 1.2s ease-out;
+}
+
+.field textarea {
+  resize: vertical;
+}
+
+.checkbox-field {
+  flex-direction: row;
+  align-items: center;
+  gap: 0.5rem;
+}
+
+.checkbox-field label {
+  font-weight: 400;
+}
+
+.actions {
+  display: flex;
+  gap: 1rem;
+  align-items: center;
+}
+
+#submit {
+  font: inherit;
+  font-size: 0.9375rem;
+  font-weight: 500;
+  padding: 0.625rem 1.25rem;
+  background: #18181b;
+  color: white;
+  border: 1px solid #18181b;
+  border-radius: 6px;
+  cursor: pointer;
+}
+
+#submit:hover {
+  background: #27272a;
+}
+
+#form-status {
+  font-size: 0.875rem;
+  color: var(--muted);
+}
+
+#status {
+  position: fixed;
+  bottom: 1rem;
+  right: 1rem;
+  padding: 0.5rem 0.75rem;
+  border-radius: 6px;
+  font-size: 0.8125rem;
+  background: #18181b;
+  color: white;
+  opacity: 0;
+  transition: opacity 0.2s;
+  pointer-events: none;
+}
+
+#status[data-show="1"] {
+  opacity: 1;
+}
--- a/examples/multi-worker/ui-worker/form-fill/client/vite.config.js
+++ b/examples/multi-worker/ui-worker/form-fill/client/vite.config.js
@@ -0,0 +1,7 @@
+import { defineConfig } from "vite";
+
+export default defineConfig({
+  server: {
+    port: 5173,
+  },
+});