wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add session updates and message dialogs

Browse Source
This commit is contained in:
Xin Wang
2026-07-31 00:01:01 +08:00
parent 913435785e
commit c2f0f5eb04
14 changed files with 850 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
backend/services/brains/base.py
View File

@@ -44,6 +44,14 @@ class BrainSpec:
owns_context: bool
@dataclass(frozen=True)
class SessionVariableUpdate:
"""Result of one silent update to declared session variables."""
changed: list[str]
dynamic_variables: dict[str, str | int | float | bool]
class CallEndPort(Protocol):
"""Small call-lifecycle surface available to a brain."""
@@ -144,6 +152,13 @@ class BaseBrain:
async def on_client_ready(self) -> None:
"""Replay client-visible state after its app message channel is ready."""
async def on_session_update(
self,
dynamic_variables: dict[str, Any],
) -> SessionVariableUpdate:
"""Apply a silent client state update without starting an LLM turn."""
raise ValueError(f"助手类型 {self.spec.type} 不支持会话动态变量更新")
def record_user_message(self, content: str) -> None:
"""Observe a committed user message for brain-owned routing state."""
@@ -194,6 +209,11 @@ class Brain(Protocol):
async def on_client_ready(self) -> None: ...
async def on_session_update(
self,
dynamic_variables: dict[str, Any],
) -> SessionVariableUpdate: ...
def record_user_message(self, content: str) -> None: ...
async def on_user_turn_end(self, content: str) -> bool: ...

22
backend/services/brains/prompt_brain.py
View File

@@ -2,6 +2,7 @@
from __future__ import annotations
from typing import Any
from uuid import uuid4
from models import AssistantConfig
@@ -15,10 +16,13 @@ from pipecat.services.llm_service import (
)
from pipecat.utils.time import time_now_iso8601
from services.brains.base import BaseBrain, BrainRuntime, BrainSpec
from services.runtime_variables import (
DynamicVariableStore,
from services.brains.base import (
BaseBrain,
BrainRuntime,
BrainSpec,
SessionVariableUpdate,
)
from services.runtime_variables import DynamicVariableStore
from services.tool_executor import ToolExecutionError, ToolExecutor
from services.tool_policy import policy_for_tool
@@ -76,6 +80,18 @@ class PromptBrain(BaseBrain):
self._store.record("user", content)
self._refresh_prompt()
async def on_session_update(
self,
dynamic_variables: dict[str, Any],
) -> SessionVariableUpdate:
changed = self._store.assign_declared_many(dynamic_variables)
if changed:
self._refresh_prompt()
return SessionVariableUpdate(
changed=changed,
dynamic_variables=self._store.public_values(),
)
async def on_assistant_text_start(self, _turn_id: str) -> None:
if self._runtime is not None:
self._runtime.call_end.begin_response()

28
backend/services/brains/workflow_brain.py
View File

@@ -28,7 +28,12 @@ from pipecat.services.llm_service import (
FunctionCallParams,
FunctionCallResultProperties,
)
from services.brains.base import BaseBrain, BrainRuntime, BrainSpec
from services.brains.base import (
BaseBrain,
BrainRuntime,
BrainSpec,
SessionVariableUpdate,
)
from services.knowledge import search as search_knowledge
from services.runtime_variables import DynamicVariableStore
from services.tool_executor import ToolExecutionError, ToolExecutor
@@ -258,6 +263,27 @@ class WorkflowBrain(BaseBrain):
node_id=current_node,
)
async def on_session_update(
self,
dynamic_variables: dict[str, Any],
) -> SessionVariableUpdate:
if self._ended:
raise ValueError("工作流会话已经结束")
changed = self._store.assign_declared_many(dynamic_variables)
current = self._state.current_node_id
if changed and current and self._engine.node_type(current) == "agent":
await self._refresh_agent_prompt(current)
if changed:
await self._emit_variables(
reason="session_update",
node_id=current or None,
changed=changed,
)
return SessionVariableUpdate(
changed=changed,
dynamic_variables=self._store.public_values(),
)
def record_user_message(self, content: str) -> None:
if content and not self._ended:
self._store.record("user", content)

17
backend/services/pipecat/pipeline.py
View File

@@ -74,6 +74,7 @@ from services.pipecat.processors import (
PassthroughLLMAssistantAggregator,
RealtimeDynamicVariableProcessor,
RealtimeUserInputProcessor,
SessionUpdateProcessor,
UserInput,
UserInputProcessor,
UserTurnRoutingProcessor,
@@ -365,6 +366,10 @@ async def run_pipeline(
or tool_interruption_strategy.is_muted
)
)
session_update = SessionUpdateProcessor(
brain.on_session_update,
should_ignore_update=lambda: call_end.ending,
)
client_tools = ClientToolBroker()
vision_capture = VisionCaptureProcessor()
knowledge_retrieval = KnowledgeRetrievalProcessor(
@@ -578,6 +583,7 @@ async def run_pipeline(
[
transport.input(),
client_tools,
session_update,
vision_capture,
user_input,
stt_processor,
@@ -778,6 +784,16 @@ async def run_realtime_pipeline(
input_sample_rate, output_sample_rate = realtime_audio_sample_rates(cfg)
user_input = RealtimeUserInputProcessor()
dynamic_variables = RealtimeDynamicVariableProcessor(brain, cfg, realtime)
async def refresh_realtime_instructions() -> None:
update = getattr(realtime, "update_instructions", None)
if callable(update):
await update(brain.system_prompt(cfg))
session_update = SessionUpdateProcessor(
brain.on_session_update,
after_update=refresh_realtime_instructions,
)
greeting = await brain.greeting(cfg)
recorder = await ConversationRecorder.start(
@@ -790,6 +806,7 @@ async def run_realtime_pipeline(
pipeline = Pipeline(
[
transport.input(),
session_update,
user_input,
realtime,
dynamic_variables,

155
backend/services/pipecat/processors.py
View File

@@ -1,13 +1,14 @@
"""Reusable frame processors shared by cascade and realtime pipelines."""
import asyncio
from collections.abc import Callable
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from uuid import uuid4
from loguru import logger
from models import AssistantConfig
from services.brains import Brain
from services.brains.base import SessionVariableUpdate
from services.conversation_history import ConversationRecorder
from services.knowledge import search as search_knowledge
from db.session import SessionLocal
@@ -38,6 +39,7 @@ from pipecat.utils.time import time_now_iso8601
KNOWLEDGE_CONTEXT_MARKER = "<!-- knowledge-context -->"
MAX_SESSION_UPDATE_VARIABLES = 20
@dataclass(frozen=True)
@@ -69,6 +71,69 @@ class UserInputError(ValueError):
self.input_id = input_id
@dataclass(frozen=True)
class SessionUpdate:
"""Validated silent update to declared session variables."""
update_id: str
dynamic_variables: dict[str, str | int | float | bool]
class SessionUpdateError(ValueError):
def __init__(self, message: str, *, update_id: str = "") -> None:
super().__init__(message)
self.update_id = update_id
def parse_session_update(message) -> SessionUpdate | None:
"""Parse the silent session-update wire format."""
if not isinstance(message, dict) or message.get("type") != "session-update":
return None
update_id = str(message.get("update_id") or "").strip()
if not update_id or len(update_id) > 128:
raise SessionUpdateError(
"session-update 缺少有效的 update_id",
update_id=update_id,
)
if message.get("schema_version") != 1:
raise SessionUpdateError(
"不支持的 session-update schema_version",
update_id=update_id,
)
raw_variables = message.get("dynamic_variables")
if not isinstance(raw_variables, dict) or not raw_variables:
raise SessionUpdateError(
"session-update dynamic_variables 不能为空",
update_id=update_id,
)
if len(raw_variables) > MAX_SESSION_UPDATE_VARIABLES:
raise SessionUpdateError(
f"session-update 一次最多更新 {MAX_SESSION_UPDATE_VARIABLES} 个变量",
update_id=update_id,
)
dynamic_variables: dict[str, str | int | float | bool] = {}
for raw_name, value in raw_variables.items():
if not isinstance(raw_name, str):
raise SessionUpdateError(
"session-update 动态变量名必须是字符串",
update_id=update_id,
)
if not isinstance(value, (str, int, float, bool)) or value is None:
raise SessionUpdateError(
f"动态变量 {raw_name} 仅支持字符串、数字或布尔值",
update_id=update_id,
)
dynamic_variables[raw_name] = value
return SessionUpdate(
update_id=update_id,
dynamic_variables=dynamic_variables,
)
def parse_user_input(message) -> UserInput | None:
"""Parse the sole public user-input wire format."""
if not isinstance(message, dict):
@@ -198,6 +263,94 @@ class UserInputProcessor(FrameProcessor):
)
class SessionUpdateProcessor(FrameProcessor):
"""Apply silent dynamic-variable updates without creating a user turn."""
def __init__(
self,
apply_update: Callable[
[dict[str, str | int | float | bool]],
Awaitable[SessionVariableUpdate],
],
*,
after_update: Callable[[], Awaitable[None]] | None = None,
should_ignore_update: Callable[[], bool] | None = None,
):
super().__init__()
self._apply_update = apply_update
self._after_update = after_update
self._should_ignore_update = should_ignore_update or (lambda: False)
async def process_frame(self, frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if not isinstance(frame, InputTransportMessageFrame):
await self.push_frame(frame, direction)
return
try:
update = parse_session_update(frame.message)
except SessionUpdateError as exc:
await self._emit_result(exc.update_id, "error", message=str(exc))
return
if update is None:
await self.push_frame(frame, direction)
return
if self._should_ignore_update():
await self._emit_result(
update.update_id,
"error",
message="当前不能更新会话状态",
)
return
try:
result = await self._apply_update(update.dynamic_variables)
if result.changed and self._after_update is not None:
await self._after_update()
except Exception as exc: # noqa: BLE001 - protocol failures need a reply
logger.warning(f"会话状态更新失败: {exc}")
await self._emit_result(
update.update_id,
"error",
message=str(exc),
)
return
await self._emit_result(
update.update_id,
"accepted",
changed=result.changed,
dynamic_variables=result.dynamic_variables,
)
async def _emit_result(
self,
update_id: str,
status: str,
*,
message: str = "",
changed: list[str] | None = None,
dynamic_variables: dict[str, str | int | float | bool] | None = None,
) -> None:
await self.push_frame(
OutputTransportMessageUrgentFrame(
message={
"type": "session-update-result",
"update_id": update_id,
"status": status,
**({"message": message} if message else {}),
**({"changed": changed} if changed is not None else {}),
**(
{"dynamic_variables": dynamic_variables}
if dynamic_variables is not None
else {}
),
}
)
)
class CallEndingUserMuteStrategy(BaseUserMuteStrategy):
"""Keep user media muted after an end-call tool starts terminating a call."""

33
backend/services/runtime_variables.py
View File

@@ -210,6 +210,39 @@ class DynamicVariableStore:
raise DynamicVariableError(f"动态变量 {name} 类型应为 {expected}")
self.values[name] = primitive
def assign_declared_many(self, values: dict[str, Any]) -> list[str]:
"""Atomically update public variables declared by the assistant."""
staged: dict[str, Primitive] = {}
for name, value in values.items():
if (
name.startswith(("system__", "secret__"))
or not VARIABLE_NAME.fullmatch(name)
):
raise DynamicVariableError(f"会话不能更新保留变量: {name}")
if name not in self.variable_types:
raise DynamicVariableError(f"动态变量未声明: {name}")
primitive = _primitive(value, name)
expected = self.variable_types[name]
if not _type_matches(primitive, expected):
raise DynamicVariableError(f"动态变量 {name} 类型应为 {expected}")
staged[name] = primitive
changed = [
name
for name, value in staged.items()
if name not in self.values or self.values[name] != value
]
self.values.update(staged)
return changed
def public_values(self) -> dict[str, Primitive]:
"""Return the client-visible dynamic-variable snapshot."""
return {
name: value
for name, value in self.values.items()
if not name.startswith(("system__", "secret__"))
}
def prepare_dynamic_config(
cfg: AssistantConfig,

109
backend/tests/test_brains.py
View File

@@ -13,6 +13,7 @@ from pipecat.frames.frames import (
LLMMessagesUpdateFrame,
LLMRunFrame,
LLMTextFrame,
LLMUpdateSettingsFrame,
OutputTransportMessageUrgentFrame,
TTSSpeakFrame,
)
@@ -28,7 +29,7 @@ from services.brains.dify_llm import (
)
from services.brains.workflow_brain import WorkflowBrain
from services.runtime_variables import prepare_dynamic_config
from services.workflow.models import LLMRouteResult, RouteStatus
from services.workflow.models import LLMRouteResult, RouteStatus, WorkflowStatus
class FakeLLM:
@@ -267,6 +268,47 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
f"{GREETING_CONTEXT_MARKER}\n您好，王先生",
)
async def test_session_update_refreshes_prompt_without_running_llm(self):
cfg = prepare_dynamic_config(
AssistantConfig(
type="prompt",
runtimeMode="pipeline",
prompt="面板状态：{{panel_open}}",
dynamic_variable_definitions={
"panel_open": {"type": "boolean", "default": False}
},
),
{},
assistant_id="asst_session_update",
)
brain = build_brain(cfg)
prompts = []
queued_frames = []
async def queue_frame(frame):
queued_frames.append(frame)
await brain.setup(
cfg,
BrainRuntime(
context=LLMContext(
messages=[{"role": "system", "content": brain.system_prompt(cfg)}]
),
llm=FakeLLM(),
queue_frame=queue_frame,
set_system_prompt=prompts.append,
set_tools=lambda _tools: None,
call_end=FakeCallEnd(),
),
)
result = await brain.on_session_update({"panel_open": True})
self.assertEqual(result.changed, ["panel_open"])
self.assertEqual(result.dynamic_variables, {"panel_open": True})
self.assertEqual(prompts, ["面板状态：true"])
self.assertEqual(queued_frames, [])
async def test_end_call_tool_is_owned_by_prompt_brain(self):
brain = build_brain(
AssistantConfig(
@@ -472,6 +514,71 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
async def test_session_update_refreshes_current_agent_without_routing(self):
cfg = prepare_dynamic_config(
AssistantConfig(
type="workflow",
graph={
"specVersion": 3,
"settings": {"globalPrompt": "面板 {{panel_open}}"},
"nodes": [
{"id": "start", "type": "start", "data": {}},
{
"id": "agent",
"type": "agent",
"data": {"prompt": "等待用户操作"},
},
],
"edges": [
{
"id": "begin",
"source": "start",
"target": "agent",
"data": {"mode": "always", "priority": 0},
}
],
},
dynamic_variable_definitions={
"panel_open": {"type": "boolean", "default": False}
},
),
{},
assistant_id="asst_workflow_session_update",
)
brain = WorkflowBrain(cfg)
queued = []
async def queue_frame(frame):
queued.append(frame)
brain._runtime = BrainRuntime(
context=LLMContext(messages=[]),
llm=FakeLLM(),
queue_frame=queue_frame,
set_system_prompt=lambda _prompt: None,
set_tools=lambda _tools: None,
call_end=FakeCallEnd(),
)
brain._state.enter("agent", WorkflowStatus.WAITING_USER)
result = await brain.on_session_update({"panel_open": True})
self.assertEqual(result.changed, ["panel_open"])
prompt_updates = [
frame for frame in queued if isinstance(frame, LLMUpdateSettingsFrame)
]
self.assertEqual(len(prompt_updates), 1)
self.assertIn("面板 true", prompt_updates[0].delta.system_instruction)
self.assertFalse(any(isinstance(frame, LLMRunFrame) for frame in queued))
variable_event = next(
frame.message
for frame in queued
if isinstance(frame, OutputTransportMessageUrgentFrame)
and frame.message.get("type") == "workflow-variables"
)
self.assertEqual(variable_event["reason"], "session_update")
self.assertEqual(variable_event["changed"], ["panel_open"])
async def test_agent_vision_tool_is_scoped_to_effective_stage(self):
brain = WorkflowBrain(
{

28
backend/tests/test_runtime_variables.py
View File

@@ -146,6 +146,34 @@ class DynamicVariableTests(unittest.TestCase):
self.assertEqual(store.values["system__agent_turns"], 1)
self.assertIn("查订单", store.values["system__conversation_history"])
def test_session_assignment_is_declared_typed_and_atomic(self):
store = DynamicVariableStore(
{"panel_open": False, "selection": "none"},
variable_types={
"panel_open": "boolean",
"selection": "string",
},
)
changed = store.assign_declared_many(
{"panel_open": True, "selection": "confirmed"}
)
self.assertEqual(changed, ["panel_open", "selection"])
self.assertEqual(
store.public_values(),
{"panel_open": True, "selection": "confirmed"},
)
with self.assertRaisesRegex(DynamicVariableError, "类型应为 boolean"):
store.assign_declared_many(
{"selection": "cancelled", "panel_open": "yes"}
)
self.assertEqual(store.values["selection"], "confirmed")
self.assertIs(store.values["panel_open"], True)
with self.assertRaisesRegex(DynamicVariableError, "未声明"):
store.assign_declared_many({"unknown_state": True})
if __name__ == "__main__":
unittest.main()

138
backend/tests/test_session_update.py Normal file
View File

@@ -0,0 +1,138 @@
from __future__ import annotations
import unittest
from pipecat.frames.frames import (
InputTransportMessageFrame,
OutputTransportMessageUrgentFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from services.brains.base import SessionVariableUpdate
from services.pipecat.processors import (
SessionUpdateError,
SessionUpdateProcessor,
parse_session_update,
)
class SessionUpdateParserTests(unittest.TestCase):
def test_parses_declared_primitive_values(self):
value = parse_session_update(
{
"type": "session-update",
"schema_version": 1,
"update_id": "update_1",
"dynamic_variables": {
"panel_open": True,
"selected_option": "approve",
},
}
)
self.assertIsNotNone(value)
self.assertEqual(value.update_id, "update_1")
self.assertEqual(
value.dynamic_variables,
{"panel_open": True, "selected_option": "approve"},
)
def test_rejects_empty_or_nested_updates(self):
with self.assertRaisesRegex(SessionUpdateError, "不能为空"):
parse_session_update(
{
"type": "session-update",
"schema_version": 1,
"update_id": "update_empty",
"dynamic_variables": {},
}
)
with self.assertRaisesRegex(SessionUpdateError, "仅支持"):
parse_session_update(
{
"type": "session-update",
"schema_version": 1,
"update_id": "update_nested",
"dynamic_variables": {"dialog": {"open": True}},
}
)
class SessionUpdateProcessorTests(unittest.IsolatedAsyncioTestCase):
async def test_returns_snapshot_without_starting_a_turn(self):
calls = []
async def apply_update(dynamic_variables):
calls.append(dynamic_variables)
return SessionVariableUpdate(
changed=["panel_open"],
dynamic_variables={
"panel_open": True,
"selected_option": "none",
},
)
processor = SessionUpdateProcessor(apply_update)
outbound = []
async def push_frame(frame, direction=FrameDirection.DOWNSTREAM):
outbound.append((frame, direction))
processor.push_frame = push_frame
await processor.process_frame(
InputTransportMessageFrame(
message={
"type": "session-update",
"schema_version": 1,
"update_id": "update_accepted",
"dynamic_variables": {"panel_open": True},
}
),
FrameDirection.DOWNSTREAM,
)
self.assertEqual(calls, [{"panel_open": True}])
self.assertEqual(len(outbound), 1)
self.assertIsInstance(outbound[0][0], OutputTransportMessageUrgentFrame)
self.assertEqual(
outbound[0][0].message,
{
"type": "session-update-result",
"update_id": "update_accepted",
"status": "accepted",
"changed": ["panel_open"],
"dynamic_variables": {
"panel_open": True,
"selected_option": "none",
},
},
)
async def test_validation_failure_returns_error(self):
async def reject_update(_dynamic_variables):
raise ValueError("动态变量未声明: panel_open")
processor = SessionUpdateProcessor(reject_update)
outbound = []
async def push_frame(frame, direction=FrameDirection.DOWNSTREAM):
outbound.append((frame, direction))
processor.push_frame = push_frame
await processor.process_frame(
InputTransportMessageFrame(
message={
"type": "session-update",
"schema_version": 1,
"update_id": "update_rejected",
"dynamic_variables": {"panel_open": True},
}
),
FrameDirection.DOWNSTREAM,
)
self.assertEqual(outbound[0][0].message["status"], "error")
self.assertIn("未声明", outbound[0][0].message["message"])
if __name__ == "__main__":
unittest.main()