Add document-review UIWorker example
Synthesis example: a ReplyToolMixin UIWorker adds a start_review tool that fans out to clarity/tone peers via start_user_job_group, translates each reviewer response into an add_note command in on_job_response, handles a client note_click event via @on_ui_event, and keeps history across turns.
This commit is contained in:
Mark Backman
475
examples/multi-worker/ui-worker/document-review/client/main.js
Normal file
475
examples/multi-worker/ui-worker/document-review/client/main.js
Normal file
@@ -0,0 +1,475 @@
|
||||
/**
|
||||
* Document review — vanilla JS client.
|
||||
*
|
||||
* Combines the patterns from every prior demo into one workspace:
|
||||
*
|
||||
* - Snapshot streaming (every demo).
|
||||
* - ``scroll_to`` and ``select_text`` for the agent to point back at
|
||||
* paragraphs (pointing + deixis).
|
||||
* - ``set_input_value`` and ``click`` for dictating notes (form-fill).
|
||||
* - ``ui-task`` envelopes for the in-flight review card with
|
||||
* per-worker progress and a Cancel button (async-tasks).
|
||||
* - One **custom command**, ``add_note``, registered locally.
|
||||
* - One **client-emitted event**, ``note_click``, sent when the user
|
||||
* clicks a note in the panel. The agent's
|
||||
* ``@on_ui_event("note_click")`` handler drives ``select_text`` to
|
||||
* navigate.
|
||||
*/
|
||||
|
||||
import {
|
||||
PipecatClient,
|
||||
RTVIEvent,
|
||||
findElementByRef,
|
||||
findRefForElement,
|
||||
} from "@pipecat-ai/client-js";
|
||||
import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport";
|
||||
|
||||
const BOT_URL = "http://localhost:7860/api/offer";
|
||||
|
||||
const connectButton = document.getElementById("connect");
|
||||
const status = document.getElementById("status");
|
||||
const botAudio = document.getElementById("bot-audio");
|
||||
const noteInput = document.getElementById("note-input");
|
||||
const noteForm = document.getElementById("note-form");
|
||||
const notesList = document.getElementById("notes-list");
|
||||
const notesEmpty = document.getElementById("notes-empty");
|
||||
const articleEl = document.querySelector("article");
|
||||
|
||||
let client;
|
||||
let unsubscribes = [];
|
||||
|
||||
// In-flight review groups, keyed by task_id. Rendered as cards above
|
||||
// the notes list while running.
|
||||
const reviewGroups = new Map();
|
||||
|
||||
// All notes ever added in this session (transient — not persisted).
|
||||
// We use refs to find them and to drive the has-notes paragraph styling.
|
||||
const notes = [];
|
||||
|
||||
// The last article paragraph the user selected. Tracked separately
|
||||
// from window.getSelection() because the textarea steals selection
|
||||
// focus when the user (or the agent) types into it. Updated only
|
||||
// when the selection lands inside the article.
|
||||
let lastArticleRef = null;
|
||||
|
||||
// Walk up from a node looking for the first ancestor that has a
|
||||
// snapshot ref assigned. Used both at submit time and from the
|
||||
// selection-tracker below.
|
||||
function findRefForAncestor(node) {
|
||||
let el = node && node.nodeType === 1 ? node : node?.parentElement ?? null;
|
||||
while (el && el !== document.body) {
|
||||
const ref = findRefForElement(el);
|
||||
if (ref) return { ref, element: el };
|
||||
el = el.parentElement;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
document.addEventListener("selectionchange", () => {
|
||||
const sel = document.getSelection();
|
||||
if (!sel || sel.isCollapsed || !sel.anchorNode) return;
|
||||
const found = findRefForAncestor(sel.anchorNode);
|
||||
if (!found) return;
|
||||
// Only remember selections inside the article column. Textarea /
|
||||
// notes-pane selections shouldn't override it.
|
||||
if (articleEl && articleEl.contains(found.element)) {
|
||||
lastArticleRef = found.ref;
|
||||
}
|
||||
});
|
||||
|
||||
function setStatus(text, autoHideMs = 0) {
|
||||
status.textContent = text;
|
||||
status.dataset.show = text ? "1" : "0";
|
||||
if (text && autoHideMs > 0) {
|
||||
setTimeout(() => {
|
||||
if (status.textContent === text) status.dataset.show = "0";
|
||||
}, autoHideMs);
|
||||
}
|
||||
}
|
||||
|
||||
function refreshEmptyState() {
|
||||
notesEmpty.hidden = notesList.children.length > 0 || reviewGroups.size > 0;
|
||||
}
|
||||
|
||||
function resolveTarget(payload) {
|
||||
if (payload?.ref) {
|
||||
const el = findElementByRef(payload.ref);
|
||||
if (el) return el;
|
||||
}
|
||||
if (payload?.target_id) {
|
||||
return document.getElementById(payload.target_id);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────
|
||||
// Standard command handlers
|
||||
// ─────────────────────────────────────────────
|
||||
|
||||
function handleScrollTo(payload) {
|
||||
const el = resolveTarget(payload);
|
||||
if (!el) return;
|
||||
el.scrollIntoView({ behavior: "smooth", block: "center" });
|
||||
}
|
||||
|
||||
function handleSelectText(payload) {
|
||||
const el = resolveTarget(payload);
|
||||
if (!el) return;
|
||||
const range = document.createRange();
|
||||
range.selectNodeContents(el);
|
||||
const sel = window.getSelection();
|
||||
if (!sel) return;
|
||||
sel.removeAllRanges();
|
||||
sel.addRange(range);
|
||||
el.scrollIntoView({ behavior: "smooth", block: "center" });
|
||||
}
|
||||
|
||||
function handleSetInputValue(payload) {
|
||||
const el = resolveTarget(payload);
|
||||
if (!el) return;
|
||||
if (!(el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement))
|
||||
return;
|
||||
if (el.disabled || el.readOnly || el.type === "hidden") return;
|
||||
const value = String(payload?.value ?? "");
|
||||
const replace = payload?.replace !== false;
|
||||
el.value = replace ? value : (el.value || "") + value;
|
||||
el.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
el.dispatchEvent(new Event("change", { bubbles: true }));
|
||||
el.classList.remove("fill-flash");
|
||||
void el.offsetWidth;
|
||||
el.classList.add("fill-flash");
|
||||
setTimeout(() => el.classList.remove("fill-flash"), 1200);
|
||||
}
|
||||
|
||||
function handleClick(payload) {
|
||||
const el = resolveTarget(payload);
|
||||
if (!el) return;
|
||||
if ("disabled" in el && el.disabled) return;
|
||||
el.click();
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────
|
||||
// Custom command: add_note
|
||||
//
|
||||
// Server emits this when a worker produces feedback, when the user's
|
||||
// dictated note is committed, etc. Payload: {source, ref?, text}.
|
||||
// We render a clickable card that — when clicked — sends a note_click
|
||||
// UI event back to the server so the agent can respond by selecting
|
||||
// the related paragraph.
|
||||
// ─────────────────────────────────────────────
|
||||
|
||||
function handleAddNote(payload) {
|
||||
const source = payload?.source ?? "me";
|
||||
const ref = payload?.ref ?? null;
|
||||
const text = String(payload?.text ?? "").trim();
|
||||
if (!text) return;
|
||||
|
||||
const note = { source, ref, text };
|
||||
notes.push(note);
|
||||
|
||||
const li = document.createElement("li");
|
||||
li.className = "note";
|
||||
if (ref) {
|
||||
li.dataset.ref = ref;
|
||||
li.tabIndex = 0;
|
||||
li.title = "Click to jump to the paragraph";
|
||||
}
|
||||
|
||||
const meta = document.createElement("div");
|
||||
meta.className = "note-meta";
|
||||
|
||||
const sourceEl = document.createElement("span");
|
||||
sourceEl.className = "note-source";
|
||||
sourceEl.dataset.source = source;
|
||||
sourceEl.textContent = source;
|
||||
meta.appendChild(sourceEl);
|
||||
|
||||
if (ref) {
|
||||
const refEl = document.createElement("span");
|
||||
refEl.className = "note-ref";
|
||||
refEl.textContent = `¶ ${ref}`;
|
||||
meta.appendChild(refEl);
|
||||
}
|
||||
|
||||
li.appendChild(meta);
|
||||
|
||||
const body = document.createElement("div");
|
||||
body.className = "note-text";
|
||||
body.textContent = text;
|
||||
li.appendChild(body);
|
||||
|
||||
// Send a UI event when the user clicks the note. The server's
|
||||
// @on_ui_event("note_click") handler turns it into a select_text
|
||||
// command back to us — full round-trip, agent-driven.
|
||||
if (ref) {
|
||||
li.addEventListener("click", () => {
|
||||
client?.sendUIEvent("note_click", { ref });
|
||||
});
|
||||
}
|
||||
|
||||
notesList.prepend(li);
|
||||
refreshEmptyState();
|
||||
|
||||
// Mark the paragraph as having notes so it stands out in the
|
||||
// document column.
|
||||
if (ref) {
|
||||
const para = findElementByRef(ref);
|
||||
if (para) para.classList.add("has-notes");
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────
|
||||
// In-flight review card (ui-task envelopes)
|
||||
// ─────────────────────────────────────────────
|
||||
|
||||
function renderReviewCard(group) {
|
||||
const card = document.createElement("div");
|
||||
card.className = "review-card";
|
||||
card.dataset.taskId = group.task_id;
|
||||
|
||||
const header = document.createElement("div");
|
||||
header.className = "review-card-header";
|
||||
|
||||
const label = document.createElement("div");
|
||||
label.className = "review-card-label";
|
||||
label.textContent = group.label ?? `Review ${group.task_id.slice(0, 6)}`;
|
||||
header.appendChild(label);
|
||||
|
||||
if (group.cancellable) {
|
||||
const cancel = document.createElement("button");
|
||||
cancel.type = "button";
|
||||
cancel.className = "review-card-cancel";
|
||||
cancel.textContent = "Cancel";
|
||||
cancel.addEventListener("click", () => {
|
||||
cancel.disabled = true;
|
||||
cancel.textContent = "Cancelling…";
|
||||
client?.cancelUITask(group.task_id, "user requested");
|
||||
});
|
||||
group.cancelButton = cancel;
|
||||
header.appendChild(cancel);
|
||||
}
|
||||
card.appendChild(header);
|
||||
|
||||
const ul = document.createElement("ul");
|
||||
ul.className = "review-workers";
|
||||
for (const agent of group.agents) {
|
||||
const li = document.createElement("li");
|
||||
li.dataset.agent = agent;
|
||||
|
||||
const name = document.createElement("span");
|
||||
name.className = "review-worker-name";
|
||||
name.textContent = agent;
|
||||
li.appendChild(name);
|
||||
|
||||
const update = document.createElement("span");
|
||||
update.className = "review-worker-update";
|
||||
update.textContent = "starting…";
|
||||
li.appendChild(update);
|
||||
|
||||
const stat = document.createElement("span");
|
||||
stat.className = "review-worker-status";
|
||||
stat.dataset.status = "running";
|
||||
stat.textContent = "running";
|
||||
li.appendChild(stat);
|
||||
|
||||
ul.appendChild(li);
|
||||
}
|
||||
card.appendChild(ul);
|
||||
|
||||
group.cardEl = card;
|
||||
group.listEl = ul;
|
||||
return card;
|
||||
}
|
||||
|
||||
function updateWorkerRow(group, agentName, { update, statusValue }) {
|
||||
const li = group.listEl.querySelector(
|
||||
`li[data-agent="${CSS.escape(agentName)}"]`,
|
||||
);
|
||||
if (!li) return;
|
||||
if (update !== undefined) {
|
||||
li.querySelector(".review-worker-update").textContent = update;
|
||||
}
|
||||
if (statusValue !== undefined) {
|
||||
const stat = li.querySelector(".review-worker-status");
|
||||
stat.dataset.status = statusValue;
|
||||
stat.textContent = statusValue;
|
||||
}
|
||||
}
|
||||
|
||||
function handleTaskEnvelope(env) {
|
||||
switch (env.kind) {
|
||||
case "group_started": {
|
||||
const group = {
|
||||
task_id: env.task_id,
|
||||
label: env.label,
|
||||
cancellable: env.cancellable,
|
||||
agents: env.agents,
|
||||
ref: extractRefFromLabel(env.label),
|
||||
};
|
||||
reviewGroups.set(env.task_id, group);
|
||||
// Place the in-flight card just below the new-note form so it
|
||||
// sits visibly above the existing notes.
|
||||
noteForm.insertAdjacentElement("afterend", renderReviewCard(group));
|
||||
// Mark the paragraph as under review.
|
||||
if (group.ref) {
|
||||
const para = findElementByRef(group.ref);
|
||||
if (para) para.classList.add("under-review");
|
||||
}
|
||||
refreshEmptyState();
|
||||
break;
|
||||
}
|
||||
case "task_update": {
|
||||
const group = reviewGroups.get(env.task_id);
|
||||
if (!group) break;
|
||||
const text = env.data?.text ?? JSON.stringify(env.data);
|
||||
updateWorkerRow(group, env.agent_name, { update: text });
|
||||
break;
|
||||
}
|
||||
case "task_completed": {
|
||||
const group = reviewGroups.get(env.task_id);
|
||||
if (!group) break;
|
||||
updateWorkerRow(group, env.agent_name, {
|
||||
update: env.status === "completed" ? "✓ done" : env.status,
|
||||
statusValue: env.status,
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "group_completed": {
|
||||
const group = reviewGroups.get(env.task_id);
|
||||
if (!group) break;
|
||||
// Drop the in-flight card; the notes that arrived via add_note
|
||||
// remain in the list.
|
||||
group.cardEl.remove();
|
||||
reviewGroups.delete(env.task_id);
|
||||
if (group.ref) {
|
||||
const para = findElementByRef(group.ref);
|
||||
if (para) para.classList.remove("under-review");
|
||||
}
|
||||
refreshEmptyState();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function extractRefFromLabel(label) {
|
||||
// The server sends labels like "Reviewing ¶ e5". Extract the ref so
|
||||
// we can mark the paragraph as under-review while workers run.
|
||||
const m = (label ?? "").match(/¶\s+(\S+)/);
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
function onUICommand(command, handler) {
|
||||
const listener = (data) => {
|
||||
if (data.command !== command) return;
|
||||
handler(data.payload);
|
||||
};
|
||||
client.on(RTVIEvent.UICommand, listener);
|
||||
return () => client.off(RTVIEvent.UICommand, listener);
|
||||
}
|
||||
|
||||
function onUITask(handler) {
|
||||
client.on(RTVIEvent.UITask, handler);
|
||||
return () => client.off(RTVIEvent.UITask, handler);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────
|
||||
// Form behavior
|
||||
// ─────────────────────────────────────────────
|
||||
|
||||
// The user (or the agent via fills + click) submits a note. Pull the
|
||||
// textarea content into a synthetic add_note so it shows up in the
|
||||
// list, then clear the textarea. The note attaches to whichever
|
||||
// article paragraph the user last selected (tracked via
|
||||
// selectionchange above) — this works for both flows because the
|
||||
// textarea's selection focus does NOT overwrite ``lastArticleRef``.
|
||||
noteForm.addEventListener("submit", (e) => {
|
||||
e.preventDefault();
|
||||
const text = noteInput.value.trim();
|
||||
if (!text) return;
|
||||
handleAddNote({ source: "me", ref: lastArticleRef, text });
|
||||
noteInput.value = "";
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────
|
||||
// Connection lifecycle
|
||||
// ─────────────────────────────────────────────
|
||||
|
||||
async function connect() {
|
||||
connectButton.disabled = true;
|
||||
setStatus("Connecting…");
|
||||
|
||||
client = new PipecatClient({
|
||||
transport: new SmallWebRTCTransport(),
|
||||
enableMic: true,
|
||||
enableCam: false,
|
||||
});
|
||||
|
||||
client.on(RTVIEvent.BotConnected, () => setStatus("Bot connected", 1500));
|
||||
client.on(RTVIEvent.Disconnected, () => {
|
||||
setStatus("Disconnected", 2000);
|
||||
connectButton.dataset.state = "";
|
||||
connectButton.textContent = "Connect";
|
||||
connectButton.disabled = false;
|
||||
teardownUI();
|
||||
});
|
||||
|
||||
client.on(RTVIEvent.TrackStarted, (track, participant) => {
|
||||
if (track.kind !== "audio") return;
|
||||
if (participant?.local) return;
|
||||
botAudio.srcObject = new MediaStream([track]);
|
||||
});
|
||||
|
||||
unsubscribes = [
|
||||
onUICommand("scroll_to", handleScrollTo),
|
||||
onUICommand("select_text", handleSelectText),
|
||||
onUICommand("set_input_value", handleSetInputValue),
|
||||
onUICommand("click", handleClick),
|
||||
onUICommand("add_note", handleAddNote),
|
||||
onUITask(handleTaskEnvelope),
|
||||
];
|
||||
|
||||
try {
|
||||
await client.connect({ webrtcUrl: BOT_URL });
|
||||
client.startUISnapshotStream();
|
||||
connectButton.dataset.state = "connected";
|
||||
connectButton.textContent = "Disconnect";
|
||||
connectButton.disabled = false;
|
||||
setStatus("Connected. Select a paragraph and ask 'review this'.", 5000);
|
||||
} catch (err) {
|
||||
console.error("Connect failed:", err);
|
||||
setStatus(`Connect failed: ${err.message ?? err}`, 4000);
|
||||
teardownUI();
|
||||
connectButton.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function disconnect() {
|
||||
connectButton.disabled = true;
|
||||
setStatus("Disconnecting…");
|
||||
try {
|
||||
await client?.disconnect();
|
||||
} finally {
|
||||
teardownUI();
|
||||
connectButton.dataset.state = "";
|
||||
connectButton.textContent = "Connect";
|
||||
connectButton.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
function teardownUI() {
|
||||
client?.stopUISnapshotStream();
|
||||
unsubscribes.forEach((unsubscribe) => unsubscribe());
|
||||
unsubscribes = [];
|
||||
if (botAudio.srcObject) botAudio.srcObject = null;
|
||||
client = undefined;
|
||||
}
|
||||
|
||||
connectButton.addEventListener("click", () => {
|
||||
if (connectButton.dataset.state === "connected") {
|
||||
disconnect();
|
||||
} else {
|
||||
connect();
|
||||
}
|
||||
});
|
||||
|
||||
refreshEmptyState();
|
||||
Reference in New Issue
Block a user