Files
pipecat/tests/test_rtvi_ui.py
Mark Backman 43abca0b06 feat(rtvi): add UI Agent Protocol as first-class RTVI message types
The UI Agent Protocol lets server-side AI agents observe and drive
a GUI app on the client side through structured RTVI messages.
Five new top-level RTVI types in kebab-case, in line with the rest
of the protocol:

  ui-event         client → server  (named event with payload)
  ui-command       server → client  (named command with payload)
  ui-snapshot      client → server  (accessibility tree of the page)
  ui-cancel-task   client → server  (cancel an in-flight task group)
  ui-task          server → client  (task lifecycle envelope)

Each ships paired ``*Data`` / ``*Message`` pydantic models in
``rtvi.models``, following the existing RTVI envelope convention
(``BotReady`` / ``BotReadyData``, ``Error`` / ``ErrorData``, etc.).
Built-in command payload models (``Toast``, ``Navigate``,
``ScrollTo``, ``Highlight``, ``Focus``, ``Click``, ``SetInputValue``,
``SelectText``) ship alongside; matching default React handlers
live in ``@pipecat-ai/client-react``.

Bumps the RTVI ``PROTOCOL_VERSION`` from ``1.2.0`` to ``1.3.0``.
Purely additive: only new top-level message types are introduced;
no existing wire shapes are changed. The major-version
compatibility check on ``client-ready`` still passes for older
1.x clients, so old clients continue to connect without warning;
they simply will not exercise the new types.

The ``RTVIProcessor`` registers a new ``on_ui_message`` event
handler that fires for inbound ``ui-event`` / ``ui-snapshot`` /
``ui-cancel-task`` with the parsed Message envelope, mirroring how
``on_client_message`` works for ``client-message``.

Five new pipeline frames let pipeline observers and processors see
UI traffic the same way they see other RTVI messages, mirroring
the frame-and-event pattern used by ``client-message``:

  RTVIUICommandFrame(command_name, payload)
    Pushed by downstream code (e.g. ``pipecat-ai-subagents``'s
    bridge) to send a UI command to the client. Wrapped by the
    observer into a ``UICommandMessage`` envelope.

  RTVIUITaskFrame(data: UITaskData)
    Same shape but for ``ui-task``; wrapped into ``UITaskMessage``.
    ``UITaskData`` is a discriminated union of the four lifecycle
    kinds (group_started / task_update / task_completed /
    group_completed).

  RTVIUIEventFrame(msg_id, event_name, payload)
  RTVIUISnapshotFrame(msg_id, tree)
  RTVIUICancelTaskFrame(msg_id, task_id, reason)
    Pushed by ``RTVIProcessor._handle_message`` whenever the
    matching inbound message arrives, alongside firing
    ``on_ui_message``. Pipeline observers and processors can match
    on the frame; subscribers like the subagents bridge keep using
    the event handler.

The data layer is the canonical authority for the wire format:
higher-level frameworks like ``pipecat-ai-subagents`` build the
agent abstractions on top, and single-LLM Pipecat apps can target
the same wire format directly via custom tools that emit these
typed messages.
2026-05-02 12:09:01 -04:00

192 lines
5.9 KiB
Python

#
# Copyright (c) 2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Smoke tests for the UI Agent Protocol wire format.
The module under test is data only (constants, payload models, and
envelope classes), so the goal is to pin the shapes: any accidental
rename or type change to a wire-format field would break compatibility
with existing client code, and we want a test that fails loudly.
"""
import unittest
from pipecat.processors.frameworks.rtvi.models import (
Click,
Focus,
Highlight,
Navigate,
ScrollTo,
SelectText,
SetInputValue,
Toast,
UICancelTaskData,
UICancelTaskMessage,
UICommandData,
UICommandMessage,
UIEventData,
UIEventMessage,
UISnapshotData,
UISnapshotMessage,
UITaskCompletedData,
UITaskGroupCompletedData,
UITaskGroupStartedData,
UITaskMessage,
UITaskUpdateData,
)
class TestEnvelopeMessages(unittest.TestCase):
"""Pin the on-the-wire envelope shapes for each first-class UI message."""
def test_ui_event_envelope(self):
msg = UIEventMessage(id="m1", data=UIEventData(name="nav_click", payload={"view": "home"}))
self.assertEqual(
msg.model_dump(),
{
"label": "rtvi-ai",
"type": "ui-event",
"id": "m1",
"data": {"name": "nav_click", "payload": {"view": "home"}},
},
)
def test_ui_command_envelope_no_id(self):
# Server-to-client push: no id field on the envelope (matches
# ServerMessage / LLMFunctionCallMessage shape).
msg = UICommandMessage(data=UICommandData(name="toast", payload={"title": "Saved"}))
self.assertEqual(
msg.model_dump(),
{
"label": "rtvi-ai",
"type": "ui-command",
"data": {"name": "toast", "payload": {"title": "Saved"}},
},
)
def test_ui_snapshot_envelope(self):
msg = UISnapshotMessage(id="m2", data=UISnapshotData(tree={"root": "..."}))
self.assertEqual(
msg.model_dump(),
{
"label": "rtvi-ai",
"type": "ui-snapshot",
"id": "m2",
"data": {"tree": {"root": "..."}},
},
)
def test_ui_cancel_task_envelope(self):
msg = UICancelTaskMessage(id="m3", data=UICancelTaskData(task_id="t-99", reason="user"))
self.assertEqual(
msg.model_dump(),
{
"label": "rtvi-ai",
"type": "ui-cancel-task",
"id": "m3",
"data": {"task_id": "t-99", "reason": "user"},
},
)
def test_ui_task_group_started(self):
msg = UITaskMessage(
data=UITaskGroupStartedData(task_id="t-1", agents=["a", "b"], label="Search", at=42)
)
self.assertEqual(msg.type, "ui-task")
self.assertEqual(msg.data.kind, "group_started")
self.assertEqual(msg.data.task_id, "t-1")
def test_ui_task_update(self):
msg = UITaskMessage(
data=UITaskUpdateData(task_id="t-1", agent_name="a", data={"progress": 0.5}, at=43)
)
self.assertEqual(msg.data.kind, "task_update")
self.assertEqual(msg.data.agent_name, "a")
def test_ui_task_completed(self):
msg = UITaskMessage(
data=UITaskCompletedData(
task_id="t-1", agent_name="a", status="completed", response={"ok": True}, at=44
)
)
self.assertEqual(msg.data.kind, "task_completed")
self.assertEqual(msg.data.status, "completed")
def test_ui_task_group_completed(self):
msg = UITaskMessage(data=UITaskGroupCompletedData(task_id="t-1", at=45))
self.assertEqual(msg.data.kind, "group_completed")
class TestPayloadShapes(unittest.TestCase):
"""Pin the on-the-wire dict shape of each command payload."""
def test_toast_required_only(self):
self.assertEqual(
dict(Toast(title="Saved")),
{
"title": "Saved",
"subtitle": None,
"description": None,
"image_url": None,
"duration_ms": None,
},
)
def test_navigate(self):
self.assertEqual(
dict(Navigate(view="home", params={"id": "42"})),
{"view": "home", "params": {"id": "42"}},
)
def test_scroll_to_with_ref(self):
self.assertEqual(
dict(ScrollTo(ref="e42", behavior="smooth")),
{"ref": "e42", "target_id": None, "behavior": "smooth"},
)
def test_highlight_with_target_id(self):
self.assertEqual(
dict(Highlight(target_id="cta", duration_ms=2000)),
{"ref": None, "target_id": "cta", "duration_ms": 2000},
)
def test_focus(self):
self.assertEqual(
dict(Focus(ref="e7")),
{"ref": "e7", "target_id": None},
)
def test_click(self):
self.assertEqual(
dict(Click(ref="e9")),
{"ref": "e9", "target_id": None},
)
def test_set_input_value_default_replace_true(self):
self.assertEqual(
dict(SetInputValue(ref="e3", value="Marie Curie")),
{"value": "Marie Curie", "ref": "e3", "target_id": None, "replace": True},
)
def test_set_input_value_append(self):
payload = dict(SetInputValue(ref="e3", value="more", replace=False))
self.assertFalse(payload["replace"])
def test_select_text_full(self):
self.assertEqual(
dict(SelectText(ref="e15", start_offset=4, end_offset=12)),
{"ref": "e15", "target_id": None, "start_offset": 4, "end_offset": 12},
)
def test_select_text_whole_target_when_offsets_omitted(self):
payload = dict(SelectText(ref="e15"))
self.assertIsNone(payload["start_offset"])
self.assertIsNone(payload["end_offset"])
if __name__ == "__main__":
unittest.main()