diff --git a/changelog/4407.added.md b/changelog/4407.added.md new file mode 100644 index 000000000..dcae4bf31 --- /dev/null +++ b/changelog/4407.added.md @@ -0,0 +1,6 @@ +- Added the UI Agent Protocol to `pipecat.processors.frameworks.rtvi.models`: + - Five first-class RTVI message types carry the wire format: `ui-event` (client → server), `ui-command` (server → client), `ui-snapshot` (client → server, accessibility tree), `ui-cancel-task` (client → server), and `ui-task` (server → client task lifecycle envelopes). Each ships a paired `*Data` / `*Message` pydantic model following the existing RTVI convention. + - Built-in command payload models (`Toast`, `Navigate`, `ScrollTo`, `Highlight`, `Focus`, `Click`, `SetInputValue`, `SelectText`) ship alongside; matching default handlers live in `@pipecat-ai/client-react`. + - The `RTVIProcessor` registers a new `on_ui_message` event handler that fires for inbound `ui-event` / `ui-snapshot` / `ui-cancel-task`. + - Five new pipeline frames let pipeline observers and processors see UI traffic the same way they see other RTVI messages: `RTVIUICommandFrame` and `RTVIUITaskFrame` are pushed by downstream code to be wrapped by the observer into outbound `UICommandMessage` / `UITaskMessage` envelopes; `RTVIUIEventFrame`, `RTVIUISnapshotFrame`, and `RTVIUICancelTaskFrame` are pushed by the processor on inbound, alongside firing `on_ui_message`, mirroring the frame-and-event pattern used by `client-message`. + - Bumps the RTVI `PROTOCOL_VERSION` from `1.2.0` to `1.3.0`. diff --git a/src/pipecat/processors/frameworks/rtvi/__init__.py b/src/pipecat/processors/frameworks/rtvi/__init__.py index 6c5225906..647dda416 100644 --- a/src/pipecat/processors/frameworks/rtvi/__init__.py +++ b/src/pipecat/processors/frameworks/rtvi/__init__.py @@ -10,6 +10,11 @@ from pipecat.processors.frameworks.rtvi.frames import ( RTVIClientMessageFrame, RTVIServerMessageFrame, RTVIServerResponseFrame, + RTVIUICancelTaskFrame, + RTVIUICommandFrame, + RTVIUIEventFrame, + RTVIUISnapshotFrame, + RTVIUITaskFrame, ) from pipecat.processors.frameworks.rtvi.observer import ( RTVIFunctionCallReportLevel, @@ -26,4 +31,9 @@ __all__ = [ "RTVIProcessor", "RTVIServerMessageFrame", "RTVIServerResponseFrame", + "RTVIUICancelTaskFrame", + "RTVIUICommandFrame", + "RTVIUIEventFrame", + "RTVIUISnapshotFrame", + "RTVIUITaskFrame", ] diff --git a/src/pipecat/processors/frameworks/rtvi/frames.py b/src/pipecat/processors/frameworks/rtvi/frames.py index 092755510..e44de7f1d 100644 --- a/src/pipecat/processors/frameworks/rtvi/frames.py +++ b/src/pipecat/processors/frameworks/rtvi/frames.py @@ -10,6 +10,7 @@ from dataclasses import dataclass from typing import Any from pipecat.frames.frames import SystemFrame +from pipecat.processors.frameworks.rtvi.models import UITaskData @dataclass @@ -27,6 +28,134 @@ class RTVIServerMessageFrame(SystemFrame): return f"{self.name}(data: {self.data})" +@dataclass +class RTVIUICommandFrame(SystemFrame): + """A frame for sending a UI command to the client. + + Pipeline-side counterpart of the ``ui-command`` RTVI message. + The observer wraps the ``command_name`` + ``payload`` into a + ``UICommandMessage`` envelope before pushing it to the transport, + so the wire shape is: + ``{label, type: "ui-command", data: {name, payload}}``. + + Parameters: + command_name: App-defined command name (e.g. ``"toast"``, + ``"navigate"``, or any app-specific name). The wire + field is ``data.name``; this avoids shadowing + ``SystemFrame.name`` (the per-instance debug label). + payload: App-defined payload. Pydantic command models + (``Toast``, ``Navigate``, ``ScrollTo``, ...) should be + converted to a plain dict via ``model_dump()`` before + being placed here; an arbitrary dict works as well. + """ + + command_name: str = "" + payload: Any = None + + def __str__(self): + """String representation of the UI command frame.""" + return f"{self.name}(command: {self.command_name})" + + +@dataclass +class RTVIUITaskFrame(SystemFrame): + """A frame for sending a UI task lifecycle envelope to the client. + + Pipeline-side counterpart of the ``ui-task`` RTVI message. The + observer wraps the ``data`` into a ``UITaskMessage`` envelope + before pushing it to the transport, so the wire shape is: + ``{label, type: "ui-task", data: }``. + + Parameters: + data: One of the four task-lifecycle data models from + ``rtvi.models`` (``UITaskGroupStartedData``, + ``UITaskUpdateData``, ``UITaskCompletedData``, or + ``UITaskGroupCompletedData``). The ``kind`` field on + each discriminates which lifecycle phase this is. + """ + + data: UITaskData | None = None + + def __str__(self): + """String representation of the UI task frame.""" + kind = getattr(self.data, "kind", "?") + return f"{self.name}(kind: {kind})" + + +@dataclass +class RTVIUIEventFrame(SystemFrame): + """An inbound UI event from the client. + + Pushed downstream by ``RTVIProcessor`` whenever a ``ui-event`` + message arrives from the client, alongside firing the + ``on_ui_message`` event handler. Mirrors the + frame-and-event pattern used by ``client-message``: pipeline + observers and processors that want to react to UI events at the + pipeline level can match on this frame; code that subscribes to + events instead (like the bridge in ``pipecat-ai-subagents``) + keeps using the event handler. + + Parameters: + msg_id: The RTVI message id, as set by the client. + event_name: App-defined event name (the ``data.name`` field). + payload: App-defined payload (the ``data.payload`` field). + """ + + msg_id: str = "" + event_name: str = "" + payload: Any = None + + def __str__(self): + """String representation of the UI event frame.""" + return f"{self.name}(event: {self.event_name})" + + +@dataclass +class RTVIUISnapshotFrame(SystemFrame): + """An inbound accessibility-snapshot from the client. + + Pushed downstream by ``RTVIProcessor`` whenever a ``ui-snapshot`` + message arrives, alongside firing ``on_ui_message``. Carries + the serialized accessibility tree the client took of its DOM. + + Parameters: + msg_id: The RTVI message id, as set by the client. + tree: The serialized accessibility tree. + """ + + msg_id: str = "" + tree: Any = None + + def __str__(self): + """String representation of the UI snapshot frame.""" + return f"{self.name}" + + +@dataclass +class RTVIUICancelTaskFrame(SystemFrame): + """An inbound user-task-group cancellation request from the client. + + Pushed downstream by ``RTVIProcessor`` whenever a + ``ui-cancel-task`` message arrives, alongside firing + ``on_ui_message``. The server-side framework should look up the + matching task group and cancel it (subject to whatever + cancellable policy the group was registered with). + + Parameters: + msg_id: The RTVI message id, as set by the client. + task_id: The task group id the client wants cancelled. + reason: Optional human-readable reason. + """ + + msg_id: str = "" + task_id: str = "" + reason: str | None = None + + def __str__(self): + """String representation of the UI cancel-task frame.""" + return f"{self.name}(task_id: {self.task_id})" + + @dataclass class RTVIClientMessageFrame(SystemFrame): """A frame for sending messages from the client to the RTVI server. diff --git a/src/pipecat/processors/frameworks/rtvi/models.py b/src/pipecat/processors/frameworks/rtvi/models.py index 81c1b2aae..d814fa3e8 100644 --- a/src/pipecat/processors/frameworks/rtvi/models.py +++ b/src/pipecat/processors/frameworks/rtvi/models.py @@ -27,7 +27,7 @@ from pipecat.frames.frames import ( ) # -- Constants -- -PROTOCOL_VERSION = "1.2.0" +PROTOCOL_VERSION = "1.3.0" MESSAGE_LABEL = "rtvi-ai" MessageLiteral = Literal["rtvi-ai"] @@ -549,3 +549,401 @@ class SystemLogMessage(BaseModel): label: MessageLiteral = MESSAGE_LABEL type: Literal["system-log"] = "system-log" data: TextMessageData + + +# -- UI Agent Protocol ------------------------------------------------------- +# +# A structured RTVI message vocabulary that lets server-side AI agents +# observe and drive a GUI app on the client side. The protocol covers +# five first-class RTVI message types: +# +# ui-event client-to-server event message +# ui-command server-to-client command message +# ui-snapshot client-to-server accessibility snapshot +# ui-cancel-task client-to-server cancellation request +# ui-task server-to-client task lifecycle envelope +# +# This section is data only (constants and payload models, no +# behavior). Higher-level frameworks like ``pipecat-ai-subagents`` +# build the agent abstractions on top, and single-LLM Pipecat apps can +# target the same wire format directly via custom tools that emit +# typed RTVI messages with these types. The matching client-side +# implementation lives in ``@pipecat-ai/client-js`` and +# ``@pipecat-ai/client-react``. + +# The wire-format ``type`` strings (``"ui-event"``, ``"ui-command"``, +# ``"ui-snapshot"``, ``"ui-cancel-task"``, ``"ui-task"``) are pinned +# as ``Literal[...]`` field defaults on the corresponding ``*Message`` +# pydantic class below, matching the convention used for every other +# RTVI message type in this module. + +# Each ``ui-task`` envelope carries a ``kind`` field that the client's +# task reducer dispatches on. The four kinds form the lifecycle of a +# user-facing task group: +# +# group_started → task_update* → task_completed × N → group_completed +# +# where N is the number of workers in the group. The kind strings are +# pinned as ``Literal[...]`` defaults on the matching ``UITask*Data`` +# class below. + + +# -- UI envelope data classes -- + + +class UIEventData(BaseModel): + """Inner ``data`` for a ``ui-event`` message. + + Parameters: + name: App-defined event name. + payload: App-defined payload, schemaless by design. + """ + + name: str + payload: Any | None = None + + +class UICommandData(BaseModel): + """Inner ``data`` for a ``ui-command`` message. + + Parameters: + name: App-defined command name. + payload: App-defined payload (already a plain dict by the + time it lands on the wire). The standard payload models + below produce the right shape via ``model_dump()``. + """ + + name: str + payload: Any | None = None + + +class UISnapshotData(BaseModel): + """Inner ``data`` for a ``ui-snapshot`` message. + + The accessibility snapshot tree is opaque on the server side. + The client owns its shape; the server stores it as-is for + rendering into the LLM context. + + Parameters: + tree: The serialized accessibility tree. + """ + + tree: Any | None = None + + +class UICancelTaskData(BaseModel): + """Inner ``data`` for a ``ui-cancel-task`` message. + + Parameters: + task_id: The task group id the client wants cancelled. + reason: Optional human-readable reason. + """ + + task_id: str + reason: str | None = None + + +class UITaskGroupStartedData(BaseModel): + """``data`` for a ``ui-task`` envelope with kind ``group_started``. + + Parameters: + kind: Always ``"group_started"``. + task_id: Shared task identifier for the group. + agents: Names of the agents the work was dispatched to. + label: Optional human-readable label for the group. + cancellable: Whether the client may request cancellation. + at: Epoch milliseconds when the group started. + """ + + kind: Literal["group_started"] = "group_started" + task_id: str + agents: list[str] | None = None + label: str | None = None + cancellable: bool = True + at: int = 0 + + +class UITaskUpdateData(BaseModel): + """``data`` for a ``ui-task`` envelope with kind ``task_update``. + + Parameters: + kind: Always ``"task_update"``. + task_id: The shared task identifier. + agent_name: The worker that produced the update. + data: The worker's update payload, forwarded verbatim. + at: Epoch milliseconds when the update was emitted. + """ + + kind: Literal["task_update"] = "task_update" + task_id: str + agent_name: str + data: Any | None = None + at: int = 0 + + +class UITaskCompletedData(BaseModel): + """``data`` for a ``ui-task`` envelope with kind ``task_completed``. + + Parameters: + kind: Always ``"task_completed"``. + task_id: The shared task identifier. + agent_name: The worker that produced the response. + status: Completion status string. + response: The worker's response payload. + at: Epoch milliseconds when the response was received. + """ + + kind: Literal["task_completed"] = "task_completed" + task_id: str + agent_name: str + status: str + response: Any | None = None + at: int = 0 + + +class UITaskGroupCompletedData(BaseModel): + """``data`` for a ``ui-task`` envelope with kind ``group_completed``. + + Parameters: + kind: Always ``"group_completed"``. + task_id: The shared task identifier. + at: Epoch milliseconds when the group completed. + """ + + kind: Literal["group_completed"] = "group_completed" + task_id: str + at: int = 0 + + +#: Discriminated union over the four task-lifecycle data shapes, +#: keyed by the ``kind`` field. +UITaskData = ( + UITaskGroupStartedData | UITaskUpdateData | UITaskCompletedData | UITaskGroupCompletedData +) + + +# -- UI envelope message classes -- + + +class UIEventMessage(BaseModel): + """RTVI ``ui-event`` message (client → server).""" + + label: MessageLiteral = MESSAGE_LABEL + type: Literal["ui-event"] = "ui-event" + id: str + data: UIEventData + + +class UICommandMessage(BaseModel): + """RTVI ``ui-command`` message (server → client).""" + + label: MessageLiteral = MESSAGE_LABEL + type: Literal["ui-command"] = "ui-command" + data: UICommandData + + +class UISnapshotMessage(BaseModel): + """RTVI ``ui-snapshot`` message (client → server).""" + + label: MessageLiteral = MESSAGE_LABEL + type: Literal["ui-snapshot"] = "ui-snapshot" + id: str + data: UISnapshotData + + +class UICancelTaskMessage(BaseModel): + """RTVI ``ui-cancel-task`` message (client → server).""" + + label: MessageLiteral = MESSAGE_LABEL + type: Literal["ui-cancel-task"] = "ui-cancel-task" + id: str + data: UICancelTaskData + + +class UITaskMessage(BaseModel): + """RTVI ``ui-task`` message (server → client). + + The ``data`` field is one of the four task-lifecycle + discriminated by the ``kind`` field. + """ + + label: MessageLiteral = MESSAGE_LABEL + type: Literal["ui-task"] = "ui-task" + data: UITaskData + + +# -- UI command payloads -- +# +# These models describe commands that have matching default React +# handlers in ``@pipecat-ai/client-react``'s ``standardHandlers``. +# Apps can use them as-is, override the client handler to customize +# rendering, or ignore them entirely and define their own command +# names. +# +# Server-side helpers that send commands accept these models directly. +# ``BaseModel.model_dump()`` converts them to the plain-dict shape +# that travels over the wire. + + +class Toast(BaseModel): + """A transient notification surface shown on the client. + + Parameters: + title: Required headline. + subtitle: Optional second line beneath the title. + description: Optional body text. + image_url: Optional leading image. + duration_ms: Optional dismiss timer. Client default applies + when None. + """ + + title: str + subtitle: str | None = None + description: str | None = None + image_url: str | None = None + duration_ms: int | None = None + + +class Navigate(BaseModel): + """Client-side navigation to a named view. + + Parameters: + view: App-defined view name (route, screen id, tab key, etc.). + params: Optional view-specific parameters. + """ + + view: str + params: dict | None = None + + +class ScrollTo(BaseModel): + """Scroll a target element into view. + + The client resolves the target by ``ref`` first (a snapshot ref + like ``"e42"`` assigned by the a11y walker), then falls back to + ``target_id`` (``document.getElementById``). Supply whichever you + have; ``ref`` is the normal choice when acting on a node from + ````. + + Parameters: + ref: Snapshot ref from ````. + target_id: Element id registered on the client. + behavior: Optional scroll behavior hint. Typical values: + ``"smooth"`` or ``"instant"``. Clients may ignore. + """ + + ref: str | None = None + target_id: str | None = None + behavior: str | None = None + + +class Highlight(BaseModel): + """Briefly emphasize a target element (flash, glow, pulse). + + Parameters: + ref: Snapshot ref from ````. + target_id: Element id registered on the client. + duration_ms: Optional highlight duration. Client default + applies when None. + """ + + ref: str | None = None + target_id: str | None = None + duration_ms: int | None = None + + +class Focus(BaseModel): + """Move input focus to a target element. + + Parameters: + ref: Snapshot ref from ````. + target_id: Element id registered on the client. + """ + + ref: str | None = None + target_id: str | None = None + + +class Click(BaseModel): + """Click an element on the client. + + Closes the form-fill loop for non-text inputs (checkboxes, radios) + and exposes the rest of the action vocabulary (submit buttons, + links, app-specific clickable nodes). The standard handler + silently no-ops on ``disabled`` targets so the agent can't bypass + UI affordances the user is meant to control. + + For native ```` targets so the agent can't write + into fields the user can't. + + Parameters: + value: The text to write. + ref: Snapshot ref from ````. Typically the ref of + an ```` or ``