wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Inject <ui_state> via the LLM's on_before_process_frame hook

Browse Source 
Move <ui_state> snapshot injection out of respond_with_llm into a
cross-cutting on_before_process_frame handler on the UIWorker's LLM, so it
appends the current snapshot to the context the request is built from, just
before each inference. Injection is gated to the user-turn-initiating
inference so a tool-calling turn never stacks duplicate <ui_state> blocks;
respond_with_llm no longer injects manually.

Also drop the bridged parameter from UIWorker: there is no viable way to
bridge a UIWorker between workers — a shared, teed context would be polluted
by the injection, and per-worker turn detection off teed frames isn't
supported. Other workers keep their PipelineWorker bridging.
This commit is contained in:
Mark Backman
2026-05-21 22:42:12 -04:00
parent 950fc10f05
commit 1c94feaaff
3 changed files with 123 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
tests/test_ui_worker.py
View File

@@ -13,8 +13,14 @@ from unittest.mock import AsyncMock, MagicMock
from pipecat.bus.messages import BusJobCancelMessage, BusJobRequestMessage
from pipecat.bus.ui_messages import _UI_SNAPSHOT_BUS_EVENT_NAME, BusUIEventMessage
from pipecat.frames.frames import LLMMessagesAppendFrame, LLMMessagesUpdateFrame
from pipecat.frames.frames import (
LLMContextFrame,
LLMMessagesAppendFrame,
LLMMessagesUpdateFrame,
)
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.utils.asyncio.task_manager import TaskManager, TaskManagerParams
from pipecat.workers.ui import UI_STATE_PROMPT_GUIDE, UIWorker, on_ui_event
@@ -180,16 +186,6 @@ class TestUIWorkerDispatch(unittest.IsolatedAsyncioTestCase):
_Bad("ui", llm=MagicMock())
async def test_bridged_with_default_auto_inject_raises(self):
with self.assertRaises(ValueError) as ctx:
_PlainWorker("ui", llm=MagicMock(), bridged=())
self.assertIn("bridged", str(ctx.exception))
self.assertIn("auto_inject_ui_state", str(ctx.exception))
async def test_bridged_with_explicit_auto_inject_disabled_is_allowed(self):
worker = _PlainWorker("ui", llm=MagicMock(), bridged=(), auto_inject_ui_state=False)
self.assertFalse(worker._auto_inject_ui_state)
async def test_default_construction_unaffected(self):
worker = _PlainWorker("ui", llm=MagicMock())
self.assertTrue(worker._auto_inject_ui_state)
@@ -497,57 +493,71 @@ class TestUIWorkerSnapshot(unittest.IsolatedAsyncioTestCase):
self.assertIn("e5", refs)
class TestUIWorkerAutoInject(unittest.IsolatedAsyncioTestCase):
async def test_respond_auto_injects_latest_snapshot(self):
worker = await _make_worker()
class TestUIWorkerSnapshotInjection(unittest.IsolatedAsyncioTestCase):
"""The <ui_state> snapshot is injected just before inference via the LLM's
on_before_process_frame hook (e.g. during a respond job).
"""
def _worker(self, **kwargs) -> UIWorker:
# A real LLM service so the on_before_process_frame event actually fires.
llm = OpenAILLMService(api_key="sk-test")
return _PlainWorker("ui", llm=llm, active=False, **kwargs)
async def _fire(self, worker: UIWorker, context: LLMContext) -> None:
await worker.llm._call_event_handler(
"on_before_process_frame", LLMContextFrame(context=context)
)
def _developer_messages(self, context: LLMContext) -> list:
return [m for m in context.messages if isinstance(m, dict) and m.get("role") == "developer"]
async def test_injects_ui_state_on_user_turn(self):
worker = self._worker()
worker._latest_snapshot = _SAMPLE_SNAPSHOT
ctx = LLMContext([{"role": "user", "content": "hi"}])
await self._fire(worker, ctx)
devs = self._developer_messages(ctx)
self.assertEqual(len(devs), 1)
self.assertTrue(devs[0]["content"].startswith("<ui_state>"))
t = await _start(
worker,
BusJobRequestMessage(source="voice", target="ui", job_id="t1", payload={"query": "hi"}),
)
frames = _append_frames(worker)
self.assertEqual(len(frames), 2)
self.assertEqual(frames[0].messages[0]["role"], "developer")
self.assertTrue(frames[0].messages[0]["content"].startswith("<ui_state>"))
self.assertFalse(frames[0].run_llm)
self.assertEqual(frames[1].messages[0]["content"], "hi")
self.assertTrue(frames[1].run_llm)
await worker.respond_to_job()
await t
async def test_auto_inject_ui_state_false_suppresses_injection(self):
worker = await _make_worker(auto_inject_ui_state=False)
async def test_skips_tool_result_continuation(self):
# The follow-up inference after a tool result must not stack a second
# <ui_state> within the same turn.
worker = self._worker()
worker._latest_snapshot = _SAMPLE_SNAPSHOT
t = await _start(
worker,
BusJobRequestMessage(source="voice", target="ui", job_id="t1", payload={"query": "hi"}),
ctx = LLMContext(
[
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "ok"},
{"role": "tool", "content": "result"},
]
)
before = len(ctx.messages)
await self._fire(worker, ctx)
self.assertEqual(len(ctx.messages), before)
frames = _append_frames(worker)
self.assertEqual(len(frames), 1)
self.assertFalse(frames[0].messages[0]["content"].startswith("<ui_state>"))
async def test_no_injection_when_disabled(self):
worker = self._worker(auto_inject_ui_state=False)
worker._latest_snapshot = _SAMPLE_SNAPSHOT
ctx = LLMContext([{"role": "user", "content": "hi"}])
await self._fire(worker, ctx)
self.assertEqual(len(ctx.messages), 1)
await worker.respond_to_job()
await t
async def test_no_injection_without_snapshot(self):
worker = self._worker()
ctx = LLMContext([{"role": "user", "content": "hi"}])
await self._fire(worker, ctx)
self.assertEqual(len(ctx.messages), 1)
async def test_auto_inject_no_op_without_snapshot(self):
worker = await _make_worker()
t = await _start(
worker,
BusJobRequestMessage(source="voice", target="ui", job_id="t1", payload={"query": "hi"}),
)
frames = _append_frames(worker)
self.assertEqual(len(frames), 1)
self.assertFalse(frames[0].messages[0]["content"].startswith("<ui_state>"))
await worker.respond_to_job()
await t
async def test_injects_once_per_turn(self):
# After injecting, the tail is the developer message, so a re-fire on the
# same context is a no-op (no accumulation within a turn).
worker = self._worker()
worker._latest_snapshot = _SAMPLE_SNAPSHOT
ctx = LLMContext([{"role": "user", "content": "hi"}])
await self._fire(worker, ctx)
await self._fire(worker, ctx)
self.assertEqual(len(self._developer_messages(ctx)), 1)
class TestUIWorkerKeepHistory(unittest.IsolatedAsyncioTestCase):
@@ -594,8 +604,12 @@ class TestUIWorkerKeepHistory(unittest.IsolatedAsyncioTestCase):
BusJobRequestMessage(source="voice", target="ui", job_id="t1", payload={"query": "hi"}),
)
# Injection moved to the on_before_process_frame hook, so respond_with_llm
# itself queues only the query append.
self.assertEqual(_update_frames(worker), [])
self.assertEqual(len(_append_frames(worker)), 2)
appends = _append_frames(worker)
self.assertEqual(len(appends), 1)
self.assertEqual(appends[0].messages[0]["content"], "hi")
await worker.respond_to_job()
await t
@@ -739,12 +753,12 @@ class TestUIWorkerRespondJob(unittest.IsolatedAsyncioTestCase):
),
)
# The snapshot is injected by the hook at inference time, so the handler
# itself queues only the rendered query.
appends = _append_frames(worker)
self.assertEqual(len(appends), 2)
self.assertTrue(appends[0].messages[0]["content"].startswith("<ui_state>"))
self.assertFalse(appends[0].run_llm)
self.assertEqual(appends[1].messages[0]["content"], "hello")
self.assertTrue(appends[1].run_llm)
self.assertEqual(len(appends), 1)
self.assertEqual(appends[0].messages[0]["content"], "hello")
self.assertTrue(appends[0].run_llm)
self.assertEqual(worker.current_job.job_id, "t1")
await worker.respond_to_job(speak="done")