950fc10f05
Synthesis example: a ReplyToolMixin UIWorker adds a start_review tool that fans out to clarity/tone peers via start_user_job_group, translates each reviewer response into an add_note command in on_job_response, handles a client note_click event via @on_ui_event, and keeps history across turns.
476 lines
15 KiB
JavaScript
476 lines
15 KiB
JavaScript
/**
|
|
* Document review — vanilla JS client.
|
|
*
|
|
* Combines the patterns from every prior demo into one workspace:
|
|
*
|
|
* - Snapshot streaming (every demo).
|
|
* - ``scroll_to`` and ``select_text`` for the agent to point back at
|
|
* paragraphs (pointing + deixis).
|
|
* - ``set_input_value`` and ``click`` for dictating notes (form-fill).
|
|
* - ``ui-task`` envelopes for the in-flight review card with
|
|
* per-worker progress and a Cancel button (async-tasks).
|
|
* - One **custom command**, ``add_note``, registered locally.
|
|
* - One **client-emitted event**, ``note_click``, sent when the user
|
|
* clicks a note in the panel. The agent's
|
|
* ``@on_ui_event("note_click")`` handler drives ``select_text`` to
|
|
* navigate.
|
|
*/
|
|
|
|
import {
|
|
PipecatClient,
|
|
RTVIEvent,
|
|
findElementByRef,
|
|
findRefForElement,
|
|
} from "@pipecat-ai/client-js";
|
|
import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport";
|
|
|
|
const BOT_URL = "http://localhost:7860/api/offer";
|
|
|
|
const connectButton = document.getElementById("connect");
|
|
const status = document.getElementById("status");
|
|
const botAudio = document.getElementById("bot-audio");
|
|
const noteInput = document.getElementById("note-input");
|
|
const noteForm = document.getElementById("note-form");
|
|
const notesList = document.getElementById("notes-list");
|
|
const notesEmpty = document.getElementById("notes-empty");
|
|
const articleEl = document.querySelector("article");
|
|
|
|
let client;
|
|
let unsubscribes = [];
|
|
|
|
// In-flight review groups, keyed by task_id. Rendered as cards above
|
|
// the notes list while running.
|
|
const reviewGroups = new Map();
|
|
|
|
// All notes ever added in this session (transient — not persisted).
|
|
// We use refs to find them and to drive the has-notes paragraph styling.
|
|
const notes = [];
|
|
|
|
// The last article paragraph the user selected. Tracked separately
|
|
// from window.getSelection() because the textarea steals selection
|
|
// focus when the user (or the agent) types into it. Updated only
|
|
// when the selection lands inside the article.
|
|
let lastArticleRef = null;
|
|
|
|
// Walk up from a node looking for the first ancestor that has a
|
|
// snapshot ref assigned. Used both at submit time and from the
|
|
// selection-tracker below.
|
|
function findRefForAncestor(node) {
|
|
let el = node && node.nodeType === 1 ? node : node?.parentElement ?? null;
|
|
while (el && el !== document.body) {
|
|
const ref = findRefForElement(el);
|
|
if (ref) return { ref, element: el };
|
|
el = el.parentElement;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
document.addEventListener("selectionchange", () => {
|
|
const sel = document.getSelection();
|
|
if (!sel || sel.isCollapsed || !sel.anchorNode) return;
|
|
const found = findRefForAncestor(sel.anchorNode);
|
|
if (!found) return;
|
|
// Only remember selections inside the article column. Textarea /
|
|
// notes-pane selections shouldn't override it.
|
|
if (articleEl && articleEl.contains(found.element)) {
|
|
lastArticleRef = found.ref;
|
|
}
|
|
});
|
|
|
|
function setStatus(text, autoHideMs = 0) {
|
|
status.textContent = text;
|
|
status.dataset.show = text ? "1" : "0";
|
|
if (text && autoHideMs > 0) {
|
|
setTimeout(() => {
|
|
if (status.textContent === text) status.dataset.show = "0";
|
|
}, autoHideMs);
|
|
}
|
|
}
|
|
|
|
function refreshEmptyState() {
|
|
notesEmpty.hidden = notesList.children.length > 0 || reviewGroups.size > 0;
|
|
}
|
|
|
|
function resolveTarget(payload) {
|
|
if (payload?.ref) {
|
|
const el = findElementByRef(payload.ref);
|
|
if (el) return el;
|
|
}
|
|
if (payload?.target_id) {
|
|
return document.getElementById(payload.target_id);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
// ─────────────────────────────────────────────
|
|
// Standard command handlers
|
|
// ─────────────────────────────────────────────
|
|
|
|
function handleScrollTo(payload) {
|
|
const el = resolveTarget(payload);
|
|
if (!el) return;
|
|
el.scrollIntoView({ behavior: "smooth", block: "center" });
|
|
}
|
|
|
|
function handleSelectText(payload) {
|
|
const el = resolveTarget(payload);
|
|
if (!el) return;
|
|
const range = document.createRange();
|
|
range.selectNodeContents(el);
|
|
const sel = window.getSelection();
|
|
if (!sel) return;
|
|
sel.removeAllRanges();
|
|
sel.addRange(range);
|
|
el.scrollIntoView({ behavior: "smooth", block: "center" });
|
|
}
|
|
|
|
function handleSetInputValue(payload) {
|
|
const el = resolveTarget(payload);
|
|
if (!el) return;
|
|
if (!(el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement))
|
|
return;
|
|
if (el.disabled || el.readOnly || el.type === "hidden") return;
|
|
const value = String(payload?.value ?? "");
|
|
const replace = payload?.replace !== false;
|
|
el.value = replace ? value : (el.value || "") + value;
|
|
el.dispatchEvent(new Event("input", { bubbles: true }));
|
|
el.dispatchEvent(new Event("change", { bubbles: true }));
|
|
el.classList.remove("fill-flash");
|
|
void el.offsetWidth;
|
|
el.classList.add("fill-flash");
|
|
setTimeout(() => el.classList.remove("fill-flash"), 1200);
|
|
}
|
|
|
|
function handleClick(payload) {
|
|
const el = resolveTarget(payload);
|
|
if (!el) return;
|
|
if ("disabled" in el && el.disabled) return;
|
|
el.click();
|
|
}
|
|
|
|
// ─────────────────────────────────────────────
|
|
// Custom command: add_note
|
|
//
|
|
// Server emits this when a worker produces feedback, when the user's
|
|
// dictated note is committed, etc. Payload: {source, ref?, text}.
|
|
// We render a clickable card that — when clicked — sends a note_click
|
|
// UI event back to the server so the agent can respond by selecting
|
|
// the related paragraph.
|
|
// ─────────────────────────────────────────────
|
|
|
|
function handleAddNote(payload) {
|
|
const source = payload?.source ?? "me";
|
|
const ref = payload?.ref ?? null;
|
|
const text = String(payload?.text ?? "").trim();
|
|
if (!text) return;
|
|
|
|
const note = { source, ref, text };
|
|
notes.push(note);
|
|
|
|
const li = document.createElement("li");
|
|
li.className = "note";
|
|
if (ref) {
|
|
li.dataset.ref = ref;
|
|
li.tabIndex = 0;
|
|
li.title = "Click to jump to the paragraph";
|
|
}
|
|
|
|
const meta = document.createElement("div");
|
|
meta.className = "note-meta";
|
|
|
|
const sourceEl = document.createElement("span");
|
|
sourceEl.className = "note-source";
|
|
sourceEl.dataset.source = source;
|
|
sourceEl.textContent = source;
|
|
meta.appendChild(sourceEl);
|
|
|
|
if (ref) {
|
|
const refEl = document.createElement("span");
|
|
refEl.className = "note-ref";
|
|
refEl.textContent = `¶ ${ref}`;
|
|
meta.appendChild(refEl);
|
|
}
|
|
|
|
li.appendChild(meta);
|
|
|
|
const body = document.createElement("div");
|
|
body.className = "note-text";
|
|
body.textContent = text;
|
|
li.appendChild(body);
|
|
|
|
// Send a UI event when the user clicks the note. The server's
|
|
// @on_ui_event("note_click") handler turns it into a select_text
|
|
// command back to us — full round-trip, agent-driven.
|
|
if (ref) {
|
|
li.addEventListener("click", () => {
|
|
client?.sendUIEvent("note_click", { ref });
|
|
});
|
|
}
|
|
|
|
notesList.prepend(li);
|
|
refreshEmptyState();
|
|
|
|
// Mark the paragraph as having notes so it stands out in the
|
|
// document column.
|
|
if (ref) {
|
|
const para = findElementByRef(ref);
|
|
if (para) para.classList.add("has-notes");
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────
|
|
// In-flight review card (ui-task envelopes)
|
|
// ─────────────────────────────────────────────
|
|
|
|
function renderReviewCard(group) {
|
|
const card = document.createElement("div");
|
|
card.className = "review-card";
|
|
card.dataset.taskId = group.task_id;
|
|
|
|
const header = document.createElement("div");
|
|
header.className = "review-card-header";
|
|
|
|
const label = document.createElement("div");
|
|
label.className = "review-card-label";
|
|
label.textContent = group.label ?? `Review ${group.task_id.slice(0, 6)}`;
|
|
header.appendChild(label);
|
|
|
|
if (group.cancellable) {
|
|
const cancel = document.createElement("button");
|
|
cancel.type = "button";
|
|
cancel.className = "review-card-cancel";
|
|
cancel.textContent = "Cancel";
|
|
cancel.addEventListener("click", () => {
|
|
cancel.disabled = true;
|
|
cancel.textContent = "Cancelling…";
|
|
client?.cancelUITask(group.task_id, "user requested");
|
|
});
|
|
group.cancelButton = cancel;
|
|
header.appendChild(cancel);
|
|
}
|
|
card.appendChild(header);
|
|
|
|
const ul = document.createElement("ul");
|
|
ul.className = "review-workers";
|
|
for (const agent of group.agents) {
|
|
const li = document.createElement("li");
|
|
li.dataset.agent = agent;
|
|
|
|
const name = document.createElement("span");
|
|
name.className = "review-worker-name";
|
|
name.textContent = agent;
|
|
li.appendChild(name);
|
|
|
|
const update = document.createElement("span");
|
|
update.className = "review-worker-update";
|
|
update.textContent = "starting…";
|
|
li.appendChild(update);
|
|
|
|
const stat = document.createElement("span");
|
|
stat.className = "review-worker-status";
|
|
stat.dataset.status = "running";
|
|
stat.textContent = "running";
|
|
li.appendChild(stat);
|
|
|
|
ul.appendChild(li);
|
|
}
|
|
card.appendChild(ul);
|
|
|
|
group.cardEl = card;
|
|
group.listEl = ul;
|
|
return card;
|
|
}
|
|
|
|
function updateWorkerRow(group, agentName, { update, statusValue }) {
|
|
const li = group.listEl.querySelector(
|
|
`li[data-agent="${CSS.escape(agentName)}"]`,
|
|
);
|
|
if (!li) return;
|
|
if (update !== undefined) {
|
|
li.querySelector(".review-worker-update").textContent = update;
|
|
}
|
|
if (statusValue !== undefined) {
|
|
const stat = li.querySelector(".review-worker-status");
|
|
stat.dataset.status = statusValue;
|
|
stat.textContent = statusValue;
|
|
}
|
|
}
|
|
|
|
function handleTaskEnvelope(env) {
|
|
switch (env.kind) {
|
|
case "group_started": {
|
|
const group = {
|
|
task_id: env.task_id,
|
|
label: env.label,
|
|
cancellable: env.cancellable,
|
|
agents: env.agents,
|
|
ref: extractRefFromLabel(env.label),
|
|
};
|
|
reviewGroups.set(env.task_id, group);
|
|
// Place the in-flight card just below the new-note form so it
|
|
// sits visibly above the existing notes.
|
|
noteForm.insertAdjacentElement("afterend", renderReviewCard(group));
|
|
// Mark the paragraph as under review.
|
|
if (group.ref) {
|
|
const para = findElementByRef(group.ref);
|
|
if (para) para.classList.add("under-review");
|
|
}
|
|
refreshEmptyState();
|
|
break;
|
|
}
|
|
case "task_update": {
|
|
const group = reviewGroups.get(env.task_id);
|
|
if (!group) break;
|
|
const text = env.data?.text ?? JSON.stringify(env.data);
|
|
updateWorkerRow(group, env.agent_name, { update: text });
|
|
break;
|
|
}
|
|
case "task_completed": {
|
|
const group = reviewGroups.get(env.task_id);
|
|
if (!group) break;
|
|
updateWorkerRow(group, env.agent_name, {
|
|
update: env.status === "completed" ? "✓ done" : env.status,
|
|
statusValue: env.status,
|
|
});
|
|
break;
|
|
}
|
|
case "group_completed": {
|
|
const group = reviewGroups.get(env.task_id);
|
|
if (!group) break;
|
|
// Drop the in-flight card; the notes that arrived via add_note
|
|
// remain in the list.
|
|
group.cardEl.remove();
|
|
reviewGroups.delete(env.task_id);
|
|
if (group.ref) {
|
|
const para = findElementByRef(group.ref);
|
|
if (para) para.classList.remove("under-review");
|
|
}
|
|
refreshEmptyState();
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
function extractRefFromLabel(label) {
|
|
// The server sends labels like "Reviewing ¶ e5". Extract the ref so
|
|
// we can mark the paragraph as under-review while workers run.
|
|
const m = (label ?? "").match(/¶\s+(\S+)/);
|
|
return m ? m[1] : null;
|
|
}
|
|
|
|
function onUICommand(command, handler) {
|
|
const listener = (data) => {
|
|
if (data.command !== command) return;
|
|
handler(data.payload);
|
|
};
|
|
client.on(RTVIEvent.UICommand, listener);
|
|
return () => client.off(RTVIEvent.UICommand, listener);
|
|
}
|
|
|
|
function onUITask(handler) {
|
|
client.on(RTVIEvent.UITask, handler);
|
|
return () => client.off(RTVIEvent.UITask, handler);
|
|
}
|
|
|
|
// ─────────────────────────────────────────────
|
|
// Form behavior
|
|
// ─────────────────────────────────────────────
|
|
|
|
// The user (or the agent via fills + click) submits a note. Pull the
|
|
// textarea content into a synthetic add_note so it shows up in the
|
|
// list, then clear the textarea. The note attaches to whichever
|
|
// article paragraph the user last selected (tracked via
|
|
// selectionchange above) — this works for both flows because the
|
|
// textarea's selection focus does NOT overwrite ``lastArticleRef``.
|
|
noteForm.addEventListener("submit", (e) => {
|
|
e.preventDefault();
|
|
const text = noteInput.value.trim();
|
|
if (!text) return;
|
|
handleAddNote({ source: "me", ref: lastArticleRef, text });
|
|
noteInput.value = "";
|
|
});
|
|
|
|
// ─────────────────────────────────────────────
|
|
// Connection lifecycle
|
|
// ─────────────────────────────────────────────
|
|
|
|
async function connect() {
|
|
connectButton.disabled = true;
|
|
setStatus("Connecting…");
|
|
|
|
client = new PipecatClient({
|
|
transport: new SmallWebRTCTransport(),
|
|
enableMic: true,
|
|
enableCam: false,
|
|
});
|
|
|
|
client.on(RTVIEvent.BotConnected, () => setStatus("Bot connected", 1500));
|
|
client.on(RTVIEvent.Disconnected, () => {
|
|
setStatus("Disconnected", 2000);
|
|
connectButton.dataset.state = "";
|
|
connectButton.textContent = "Connect";
|
|
connectButton.disabled = false;
|
|
teardownUI();
|
|
});
|
|
|
|
client.on(RTVIEvent.TrackStarted, (track, participant) => {
|
|
if (track.kind !== "audio") return;
|
|
if (participant?.local) return;
|
|
botAudio.srcObject = new MediaStream([track]);
|
|
});
|
|
|
|
unsubscribes = [
|
|
onUICommand("scroll_to", handleScrollTo),
|
|
onUICommand("select_text", handleSelectText),
|
|
onUICommand("set_input_value", handleSetInputValue),
|
|
onUICommand("click", handleClick),
|
|
onUICommand("add_note", handleAddNote),
|
|
onUITask(handleTaskEnvelope),
|
|
];
|
|
|
|
try {
|
|
await client.connect({ webrtcUrl: BOT_URL });
|
|
client.startUISnapshotStream();
|
|
connectButton.dataset.state = "connected";
|
|
connectButton.textContent = "Disconnect";
|
|
connectButton.disabled = false;
|
|
setStatus("Connected. Select a paragraph and ask 'review this'.", 5000);
|
|
} catch (err) {
|
|
console.error("Connect failed:", err);
|
|
setStatus(`Connect failed: ${err.message ?? err}`, 4000);
|
|
teardownUI();
|
|
connectButton.disabled = false;
|
|
}
|
|
}
|
|
|
|
async function disconnect() {
|
|
connectButton.disabled = true;
|
|
setStatus("Disconnecting…");
|
|
try {
|
|
await client?.disconnect();
|
|
} finally {
|
|
teardownUI();
|
|
connectButton.dataset.state = "";
|
|
connectButton.textContent = "Connect";
|
|
connectButton.disabled = false;
|
|
}
|
|
}
|
|
|
|
function teardownUI() {
|
|
client?.stopUISnapshotStream();
|
|
unsubscribes.forEach((unsubscribe) => unsubscribe());
|
|
unsubscribes = [];
|
|
if (botAudio.srcObject) botAudio.srcObject = null;
|
|
client = undefined;
|
|
}
|
|
|
|
connectButton.addEventListener("click", () => {
|
|
if (connectButton.dataset.state === "connected") {
|
|
disconnect();
|
|
} else {
|
|
connect();
|
|
}
|
|
});
|
|
|
|
refreshEmptyState();
|