From 2254a8d0a23746980ffef15b67cc3c56c5440b06 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 21 May 2026 16:51:38 -0400 Subject: [PATCH] Add hello-snapshot UIWorker example Smallest UIWorker demo: a voice LLM in the main pipeline delegates screen-relevant utterances to a UIWorker via a respond job; the UIWorker auto-injects the current and answers grounded in what's on screen. Includes a vanilla-JS client that streams accessibility snapshots over RTVI. --- .../ui-worker/hello-snapshot/README.md | 89 ++ .../ui-worker/hello-snapshot/bot.py | 274 ++++ .../hello-snapshot/client/index.html | 73 ++ .../ui-worker/hello-snapshot/client/main.js | 110 ++ .../hello-snapshot/client/package-lock.json | 1128 +++++++++++++++++ .../hello-snapshot/client/package.json | 18 + .../hello-snapshot/client/styles.css | 129 ++ .../hello-snapshot/client/vite.config.js | 7 + 8 files changed, 1828 insertions(+) create mode 100644 examples/multi-worker/ui-worker/hello-snapshot/README.md create mode 100644 examples/multi-worker/ui-worker/hello-snapshot/bot.py create mode 100644 examples/multi-worker/ui-worker/hello-snapshot/client/index.html create mode 100644 examples/multi-worker/ui-worker/hello-snapshot/client/main.js create mode 100644 examples/multi-worker/ui-worker/hello-snapshot/client/package-lock.json create mode 100644 examples/multi-worker/ui-worker/hello-snapshot/client/package.json create mode 100644 examples/multi-worker/ui-worker/hello-snapshot/client/styles.css create mode 100644 examples/multi-worker/ui-worker/hello-snapshot/client/vite.config.js diff --git a/examples/multi-worker/ui-worker/hello-snapshot/README.md b/examples/multi-worker/ui-worker/hello-snapshot/README.md new file mode 100644 index 000000000..cde3b4528 --- /dev/null +++ b/examples/multi-worker/ui-worker/hello-snapshot/README.md @@ -0,0 +1,89 @@ +# hello-snapshot + +The smallest possible `UIWorker` example. A static HTML page with a few +news cards and a sidebar. The user speaks; the worker answers grounded +in whatever's currently on screen. + +## What it shows + +- The accessibility-snapshot pipeline: the client walks the DOM and + streams a snapshot, which the `UIWorker` injects into its LLM context + as ``. +- The UIWorker delegate setup: the main pipeline's LLM (the + conversational layer) delegates every utterance to a `HelloWorker` + (`UIWorker`) via the `answer_about_screen` tool + (`params.pipeline_worker.job("hello", name="respond", ...)`) and speaks + the result. +- The native RTVI⇄bus UI bridge built into `PipelineWorker`: with + `enable_rtvi=True` (the default), inbound `ui-snapshot` messages are + broadcast on the bus and the `UIWorker` stores them — no decorator or + manual wiring. + +## Architecture + +``` +Main worker (PipelineWorker, owns transport + RTVI): + transport.in → STT → user_agg → LLM → TTS → transport.out → assistant_agg + └── answer_about_screen(query) tool + └── params.pipeline_worker.job("hello", name="respond", payload={query}) + +HelloWorker (UIWorker): + └── @tool answer(text) +``` + +## Run + +Two terminals. + +**Terminal 1 — bot:** + +```bash +cd examples/multi-worker/ui-worker/hello-snapshot +uv run python bot.py +``` + +The bot starts on `http://localhost:7860`. + +**Terminal 2 — client:** + +```bash +cd examples/multi-worker/ui-worker/hello-snapshot/client +npm install # one-time +npm run dev +``` + +Open `http://localhost:5173` and click **Connect**. + +## What to try + +Once connected, ask the worker: + +- _"What's on this page?"_ — it summarizes the layout (heading, three + stories, trending tags sidebar). +- _"What was the second story about?"_ — sibling order in the snapshot + matches reading order, so "second" resolves cleanly. +- _"Which story was about energy?"_ — the worker grounds against the + actual content, not just titles. +- _"What tags are trending?"_ — exercises sidebar reading. +- _"What's the capital of France?"_ — the worker answers from general + knowledge when the question has nothing to do with the page. + +If you scroll the page (in a smaller window) or resize, the snapshot +re-emits. Off-screen elements get an `[offscreen]` tag the worker +respects when answering positional questions like "what do I see right +now." + +## Requirements + +- `OPENAI_API_KEY` +- `DEEPGRAM_API_KEY` +- `CARTESIA_API_KEY` + +A `.env` in the example folder is the easiest way to set these (see +`examples/multi-worker/env.example`). + +## What this example _doesn't_ show + +The read-side foundation only — no acting on the page (`scroll_to`, +`highlight`, ...), form filling, selection-based deixis, or async task +cards. Those build on this same skeleton. diff --git a/examples/multi-worker/ui-worker/hello-snapshot/bot.py b/examples/multi-worker/ui-worker/hello-snapshot/bot.py new file mode 100644 index 000000000..93b7aa018 --- /dev/null +++ b/examples/multi-worker/ui-worker/hello-snapshot/bot.py @@ -0,0 +1,274 @@ +# +# Copyright (c) 2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Hello UIWorker — the smallest possible accessibility-snapshot demo. + +A voice bot whose LLM delegates every screen-relevant utterance to a +``UIWorker`` that sees the page and writes the spoken answer. + +Architecture:: + + Main worker (PipelineWorker, owns transport + RTVI): + transport.in → STT → user_agg → LLM → TTS → transport.out → assistant_agg + └── answer_about_screen(query) tool + └── params.pipeline_worker.job("hello", name="respond", payload={query}) + + HelloWorker (UIWorker): + └── @tool answer(text) + +The main LLM is the conversational layer: it forwards every utterance +to the UI worker via the ``answer_about_screen`` tool and speaks the +result. The UI worker's built-in ``respond`` job fires, which +auto-injects the latest ```` block into its LLM context. The +UI worker's LLM picks the ``answer`` tool with a spoken reply grounded +in what's on screen. + +The RTVI⇄bus UI bridge is built into ``PipelineWorker`` (active because +``enable_rtvi=True``), so inbound ``ui-snapshot`` messages from the +client are broadcast on the bus and the ``UIWorker`` stores them — no +decorator or manual wiring needed. + +Why two LLMs for "hello world": this is the pattern UIWorker's +auto-inject is built for. The UI worker auto-injects the current screen +at the start of every delegated job, so the conversational LLM stays +small and screen-unaware. Later examples (pointing, form-fill, deixis, +async-tasks) compose new tools onto the same skeleton. + +Run:: + + uv run python bot.py + +Then open the client at ``http://localhost:5173`` (see ``README.md``). + +Requirements: + +- OPENAI_API_KEY +- DEEPGRAM_API_KEY +- CARTESIA_API_KEY +""" + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.job_context import JobError +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.worker import PipelineParams, PipelineWorker +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.llm_service import FunctionCallParams +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.workers.llm import tool +from pipecat.workers.ui import UIWorker + +load_dotenv(override=True) + +MAIN_NAME = "main" + +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +VOICE_PROMPT = """\ +You are the voice layer of a screen-aware assistant. A separate UI \ +layer sees the page the user is looking at and writes the spoken \ +reply for any question that could plausibly involve the page. + +## Routing rule +For every user utterance that could involve the page in any way — \ +"what's on screen", "what does this say", "is X on the page", \ +factual questions, navigational questions, anything where the page \ +content might matter — call ``answer_about_screen`` with the user's \ +request verbatim. The tool's response is the spoken reply, already \ +TTS-ready; pass it through without paraphrasing. + +If the request has nothing to do with the page, still call the \ +tool — the UI layer falls back to general knowledge. + +## When to answer directly +Only respond directly for pure pleasantries that don't need any \ +content awareness: + +- Greetings ("hi", "hello"). +- Acknowledgements ("thanks", "got it"). +- Goodbyes ("bye", "see you"). + +Keep direct replies to one short spoken sentence. No markdown, no \ +lists, no symbols.""" + + +# The UI wire-format guide (UI_STATE_PROMPT_GUIDE) is appended to the LLM's +# system instruction automatically by UIWorker, so this prompt only needs the +# app-specific behavior. +HELLO_PROMPT = """\ +You answer the user's question grounded in the page they're looking \ +at. The current ```` block is in your context — use it for \ +anything the user could be asking about on screen. + +Always call exactly one tool: ``answer(text)``. Put the spoken reply \ +in ``text``. Plain language, one or two short sentences, no markdown \ +or symbols. + +When the question is about something on the page, ground claims in \ +the ```` content. When it's general knowledge with no \ +on-page referent (history, geography, definitions), answer from your \ +own knowledge. Don't tell the user what you can't see — just answer \ +or admit you don't know.""" + + +class HelloWorker(UIWorker): + """Snapshot-aware layer. Answers grounded in ````. + + ``UIWorker`` defaults to ``active=True`` (unlike ``LLMWorker``) + because the canonical UIWorker role is an always-on delegate, so it + is online to receive snapshots and ``respond`` jobs as soon as its + pipeline starts. + """ + + def __init__(self): + llm = OpenAILLMService( + api_key=os.environ["OPENAI_API_KEY"], + settings=OpenAILLMService.Settings(system_instruction=HELLO_PROMPT), + ) + super().__init__("hello", llm=llm) + + @tool + async def answer(self, params: FunctionCallParams, text: str): + """Speak ``text`` back to the user. + + Args: + text: The spoken reply in plain language. One or two short + sentences. No markdown, no symbols, no lists. + """ + logger.info(f"{self}: answer('{text[:80]}…')") + await self.respond_to_job(speak=text) + await params.result_callback(None) + + +async def answer_about_screen(params: FunctionCallParams, query: str): + """Ask the screen-aware UI layer to answer about the current page. + + Args: + query (str): The user's request, passed verbatim. + """ + logger.info(f"answer_about_screen('{query}')") + try: + async with params.pipeline_worker.job( + "hello", name="respond", payload={"query": query}, timeout=30 + ) as t: + pass + except JobError as e: + logger.warning(f"hello job failed: {e}") + await params.result_callback("Something went wrong on my side.") + return + + speak = (t.response or {}).get("speak") + await params.result_callback(speak or "I'm not sure how to answer that.") + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info("Starting hello-snapshot bot") + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + stt = DeepgramSTTService(api_key=os.environ["DEEPGRAM_API_KEY"]) + tts = CartesiaTTSService( + api_key=os.environ["CARTESIA_API_KEY"], + settings=CartesiaTTSService.Settings( + voice=os.getenv("CARTESIA_VOICE_ID", "71a7ad14-091c-4e8e-a314-022ece01c121"), + ), + ) + llm = OpenAILLMService( + api_key=os.environ["OPENAI_API_KEY"], + settings=OpenAILLMService.Settings(system_instruction=VOICE_PROMPT), + ) + llm.register_direct_function(answer_about_screen, cancel_on_interruption=False, timeout_secs=60) + + context = LLMContext(tools=ToolsSchema(standard_tools=[answer_about_screen])) + aggregators = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + ) + + pipeline = Pipeline( + [ + transport.input(), + stt, + aggregators.user(), + llm, + tts, + transport.output(), + aggregators.assistant(), + ] + ) + + worker = PipelineWorker( + pipeline, + name=MAIN_NAME, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info("Client connected") + context.add_message( + { + "role": "developer", + "content": ( + "Greet the user briefly. Tell them they can ask about " + "anything on this page. One short sentence." + ), + } + ) + await worker.queue_frame(LLMRunFrame()) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info("Client disconnected") + await runner.cancel() + + await runner.launch_worker(HelloWorker()) + await runner.launch_worker(worker) + + await runner.run() + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/multi-worker/ui-worker/hello-snapshot/client/index.html b/examples/multi-worker/ui-worker/hello-snapshot/client/index.html new file mode 100644 index 000000000..1bb1bbd67 --- /dev/null +++ b/examples/multi-worker/ui-worker/hello-snapshot/client/index.html @@ -0,0 +1,73 @@ + + + + + + Hello UIAgent + + + +
+

Local News Today

+ +
+ +
+
+

Top stories

+ +
+

Mediterranean diet linked to longer cognitive lifespan

+

+ A 12-year study following 50,000 adults across Italy and Spain + found that strict adherence to the Mediterranean diet + correlated with a 28% lower incidence of age-related cognitive + decline. Researchers attribute the effect to anti-inflammatory + compounds in olive oil and oily fish. +

+
+ +
+

Atacama solar field crosses 5 GW capacity

+

+ Chile's flagship desert solar project added another 600 MW of + generation this quarter, pushing total capacity past five + gigawatts. The expansion, completed three months ahead of + schedule, makes Atacama the largest solar generation site in + South America. +

+
+ +
+

Mars sample return mission re-scoped to 2031

+

+ NASA and ESA jointly announced a revised timeline for the Mars + Sample Return mission, citing budget pressure and the need to + consolidate launch logistics. The first samples are now + expected back on Earth in late 2031, a two-year delay. +

+
+
+ + +
+ +
+ + + + + + + diff --git a/examples/multi-worker/ui-worker/hello-snapshot/client/main.js b/examples/multi-worker/ui-worker/hello-snapshot/client/main.js new file mode 100644 index 000000000..4493e0d50 --- /dev/null +++ b/examples/multi-worker/ui-worker/hello-snapshot/client/main.js @@ -0,0 +1,110 @@ +/** + * Hello UIAgent — vanilla JS client. + * + * Wires three pieces of the SDK end to end: + * 1. PipecatClient + SmallWebRTCTransport for the voice session. + * 2. PipecatClient-managed accessibility snapshot streaming on + * every meaningful change (DOM mutations, focus, scroll-end, + * resize, visibility, selection). + * + * The agent has no tools — the snapshot is the entire input. The + * server's ``UIAgent`` auto-injects the latest ```` block + * into the LLM context at the start of every turn, so the agent + * always answers grounded in what's currently on screen. + */ + +import { PipecatClient, RTVIEvent } from "@pipecat-ai/client-js"; +import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport"; + +const BOT_URL = "http://localhost:7860/api/offer"; + +const connectButton = document.getElementById("connect"); +const status = document.getElementById("status"); +const botAudio = document.getElementById("bot-audio"); + +let client; + +function setStatus(text, autoHideMs = 0) { + status.textContent = text; + status.dataset.show = text ? "1" : "0"; + if (text && autoHideMs > 0) { + setTimeout(() => { + if (status.textContent === text) status.dataset.show = "0"; + }, autoHideMs); + } +} + +async function connect() { + connectButton.disabled = true; + setStatus("Connecting…"); + + // 1. Construct the Pipecat client with the WebRTC transport. + client = new PipecatClient({ + transport: new SmallWebRTCTransport(), + enableMic: true, + enableCam: false, + }); + + client.on(RTVIEvent.BotConnected, () => setStatus("Bot connected", 1500)); + client.on(RTVIEvent.Disconnected, () => { + setStatus("Disconnected", 2000); + connectButton.dataset.state = ""; + connectButton.textContent = "Connect"; + connectButton.disabled = false; + teardownUI(); + }); + + // Pipe the bot's audio track into the