wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance greeting context management in Brain classes

Browse Source 
- Introduce greeting context handling in BaseBrain and WorkflowBrain to manage assistant greetings effectively.
- Implement prepare_greeting_context method to add greeting messages to the local context while preserving playback order.
- Update pipeline event handling to ensure greeting timestamps are maintained until the client is ready.
- Enhance tests to verify the correct behavior of greeting context management in various scenarios.
This commit is contained in:
Xin Wang
2026-07-14 13:26:47 +08:00
parent 35cbee4786
commit d069e5282e
6 changed files with 327 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
backend/services/brains/workflow_brain.py
View File

@@ -70,6 +70,9 @@ class WorkflowBrain(BaseBrain):
self._manager: FlowManager | None = None
self._router = WorkflowLLMRouter(cfg or AssistantConfig(type="workflow"))
self._ended = False
self._greeting_context_message: dict[str, str] | None = None
self._client_ready = False
self._pending_visible_speech_events: list[dict[str, Any]] = []
async def greeting(self, cfg: AssistantConfig) -> str:
return self._engine.greeting(self._store) or cfg.greeting
@@ -91,6 +94,9 @@ class WorkflowBrain(BaseBrain):
self._tools = ToolExecutor(self._store)
self._tool_by_id = {tool.id: tool for tool in cfg.tools}
self._router = WorkflowLLMRouter(cfg)
self._greeting_context_message = None
self._client_ready = False
self._pending_visible_speech_events = []
self._manager = FlowManager(
worker=runtime.worker,
llm=runtime.llm,
@@ -100,6 +106,15 @@ class WorkflowBrain(BaseBrain):
)
self._manager.state["variables"] = self._store.values
def prepare_greeting_context(
self,
greeting: str,
context: LLMContext,
) -> dict[str, str] | None:
message = super().prepare_greeting_context(greeting, context)
self._greeting_context_message = deepcopy(message) if message else None
return message
async def on_connected(self) -> None:
await self._emit_node_active(self._engine.start_id)
await self._emit_variables(
@@ -125,6 +140,13 @@ class WorkflowBrain(BaseBrain):
async def on_client_ready(self) -> None:
"""Replay state that may have been emitted before WebRTC data was ready."""
self._client_ready = True
pending_speech_events = self._pending_visible_speech_events
self._pending_visible_speech_events = []
for message in pending_speech_events:
await self._require_runtime().queue_frame(
OutputTransportMessageUrgentFrame(message=message)
)
current_node = (
str(self._manager.current_node)
if self._manager and self._manager.current_node
@@ -273,11 +295,15 @@ class WorkflowBrain(BaseBrain):
}
)
def _agent_config(self, node_id: str) -> NodeConfig:
def _agent_config(
self,
node_id: str,
leading_messages: list[dict[str, str]] | None = None,
) -> NodeConfig:
data = self._engine.data(node_id)
entry_mode = str(data.get("entryMode") or "wait_user")
entry_speech = self._store.render(str(data.get("entrySpeech") or ""))
fixed_reply_messages = (
fixed_reply_messages: list[dict[str, str]] = (
[{"role": "assistant", "content": entry_speech}]
if entry_mode == "fixed_speech" and entry_speech
else []
@@ -287,6 +313,16 @@ class WorkflowBrain(BaseBrain):
if data.get("contextPolicy") == "fresh"
else ContextStrategy.APPEND
)
greeting_messages = (
[deepcopy(self._greeting_context_message)]
if strategy == ContextStrategy.RESET and self._greeting_context_message
else []
)
task_messages = [
*greeting_messages,
*(leading_messages or []),
*fixed_reply_messages,
]
stage = self._engine.agent_stage_config(node_id)
functions: list[FlowsFunctionSchema] = []
for tool_id in stage.tool_ids:
@@ -302,7 +338,7 @@ class WorkflowBrain(BaseBrain):
# Flows writes task_messages into the Pipecat LLM context. The
# pre-action below is responsible only for display, persistence,
# dynamic conversation history, and TTS playback.
"task_messages": fixed_reply_messages,
"task_messages": task_messages,
"functions": functions,
"context_strategy": ContextStrategyConfig(strategy=strategy),
"respond_immediately": entry_mode == "generate",
@@ -339,26 +375,32 @@ class WorkflowBrain(BaseBrain):
return
self._store.record("agent", content)
runtime = self._require_runtime()
await runtime.queue_frame(
OutputTransportMessageUrgentFrame(
message={
"type": "transcript",
"role": "assistant",
"content": content,
"timestamp": time_now_iso8601(),
"source": source,
**({"nodeId": node_id} if node_id else {}),
}
transcript_message = {
"type": "transcript",
"role": "assistant",
"content": content,
"timestamp": time_now_iso8601(),
"source": source,
**({"nodeId": node_id} if node_id else {}),
}
if self._client_ready:
await runtime.queue_frame(
OutputTransportMessageUrgentFrame(message=transcript_message)
)
)
else:
self._pending_visible_speech_events.append(transcript_message)
await runtime.queue_frame(TTSSpeakFrame(content, append_to_context=False))
def _passive_node_config(self, node_id: str) -> NodeConfig:
def _passive_node_config(
self,
node_id: str,
task_messages: list[dict[str, str]] | None = None,
) -> NodeConfig:
"""Keep a non-conversational terminal node active without ending the call."""
return {
"name": node_id,
"role_message": self._store.render(self._engine.global_prompt()),
"task_messages": [],
"task_messages": list(task_messages or []),
"functions": [],
"context_strategy": ContextStrategyConfig(strategy=ContextStrategy.APPEND),
"respond_immediately": False,
@@ -440,20 +482,39 @@ class WorkflowBrain(BaseBrain):
)
async def _follow_edge(self, edge: dict) -> NodeConfig:
leading_messages: list[dict[str, str]] = []
speech = self._engine.edge_transition_speech(edge)
if speech:
await self._queue_visible_speech(self._store.render(speech))
return await self._resolve_path(str(edge.get("target") or ""))
content = self._store.render(speech).strip()
if content:
await self._queue_visible_speech(
content,
source="workflow-edge-transition",
node_id=str(edge.get("target") or "") or None,
)
leading_messages.append(
{"role": "assistant", "content": content}
)
return await self._resolve_path(
str(edge.get("target") or ""),
leading_messages=leading_messages,
)
async def _resolve_path(self, node_id: str) -> NodeConfig:
async def _resolve_path(
self,
node_id: str,
*,
leading_messages: list[dict[str, str]] | None = None,
) -> NodeConfig:
context_messages = list(leading_messages or [])
for _ in range(MAX_AUTOMATIC_HOPS):
node_type = self._engine.node_type(node_id)
if node_type == "agent":
await self._apply_agent_stage(node_id)
return self._agent_config(node_id)
return self._agent_config(node_id, context_messages)
if node_type == "end":
await self._enter_end(node_id)
return self._passive_node_config(node_id)
return self._passive_node_config(node_id, context_messages)
if node_type == "action":
await self._enter_action(node_id)
elif node_type == "handoff":
@@ -463,7 +524,7 @@ class WorkflowBrain(BaseBrain):
else:
raise RuntimeError(f"工作流指向未知节点:{node_id}")
if not self._engine.has_outgoing(node_id):
return self._passive_node_config(node_id)
return self._passive_node_config(node_id, context_messages)
edge = self._engine.deterministic_edge(
node_id,
self._store,
@@ -473,7 +534,17 @@ class WorkflowBrain(BaseBrain):
raise RuntimeError(f"自动节点 {node_id} 没有命中的表达式边或默认边")
speech = self._engine.edge_transition_speech(edge)
if speech:
await self._queue_visible_speech(self._store.render(speech))
content = self._store.render(speech).strip()
if content:
target_id = str(edge.get("target") or "")
await self._queue_visible_speech(
content,
source="workflow-edge-transition",
node_id=target_id or None,
)
context_messages.append(
{"role": "assistant", "content": content}
)
node_id = str(edge.get("target") or "")
raise RuntimeError("工作流连续自动跳转超过安全上限")