wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance conversation history and runtime variable management

Browse Source 
- Update ConversationRecorder to include source and nodeId metadata in transcripts for better context tracking.
- Introduce optional variable handling in DynamicVariableStore, allowing for unset variables to be rendered as empty without raising errors.
- Refactor WorkflowBrain to apply turn configurations and manage interaction policies dynamically, improving agent responsiveness.
- Implement tests to ensure proper handling of updated session variables and workflow metadata in various scenarios.
This commit is contained in:
Xin Wang
2026-07-14 11:08:11 +08:00
parent 665f727796
commit f74040adf3
18 changed files with 848 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/services/brains/base.py
View File

@@ -60,6 +60,9 @@ class BrainRuntime:
) = None
set_knowledge_scope: Callable[[dict[str, Any]], None] | None = None
set_input_enabled: Callable[[bool], None] | None = None
apply_turn_config: (
Callable[[bool, dict[str, Any]], Awaitable[None]] | None
) = None
flow_global_functions: list[Any] = field(default_factory=list)

100
backend/services/brains/workflow_brain.py
View File

@@ -2,6 +2,7 @@
from __future__ import annotations
from copy import deepcopy
from typing import Any
from loguru import logger
@@ -49,7 +50,13 @@ class WorkflowBrain(BaseBrain):
def __init__(self, cfg_or_graph: AssistantConfig | dict[str, Any]):
cfg = cfg_or_graph if isinstance(cfg_or_graph, AssistantConfig) else None
graph = cfg.graph if cfg is not None else cfg_or_graph
graph = deepcopy(cfg.graph if cfg is not None else cfg_or_graph)
if cfg is not None:
# Graph v3 owns Workflow defaults. Keep older saved graphs compatible
# by filling the new interaction settings from the assistant row.
settings = graph.setdefault("settings", {})
settings.setdefault("enableInterrupt", cfg.enableInterrupt)
settings.setdefault("turnConfig", deepcopy(cfg.turnConfig))
self._engine = WorkflowEngine(graph or {})
if not self._engine.has_graph() or not self._engine.start_id:
raise ValueError("WorkflowBrain 缺少有效的 Start 节点")
@@ -95,6 +102,10 @@ class WorkflowBrain(BaseBrain):
async def on_connected(self) -> None:
await self._emit_node_active(self._engine.start_id)
await self._emit_variables(
reason="initialized",
node_id=self._engine.start_id,
)
edge = self._engine.deterministic_edge(
self._engine.start_id,
self._store,
@@ -228,6 +239,11 @@ class WorkflowBrain(BaseBrain):
if self._runtime and self._runtime.set_input_enabled:
self._runtime.set_input_enabled(True)
runtime = self._require_runtime()
if runtime.apply_turn_config:
await runtime.apply_turn_config(
stage.enable_interrupt,
stage.turn_config,
)
if runtime.switch_services:
await runtime.switch_services(
stage.llm_resource_id or None,
@@ -248,6 +264,11 @@ class WorkflowBrain(BaseBrain):
data = self._engine.data(node_id)
entry_mode = str(data.get("entryMode") or "wait_user")
entry_speech = self._store.render(str(data.get("entrySpeech") or ""))
fixed_reply_messages = (
[{"role": "assistant", "content": entry_speech}]
if entry_mode == "fixed_speech" and entry_speech
else []
)
strategy = (
ContextStrategy.RESET
if data.get("contextPolicy") == "fresh"
@@ -265,11 +286,10 @@ class WorkflowBrain(BaseBrain):
config: NodeConfig = {
"name": node_id,
"role_message": self._agent_role_message(node_id),
"task_messages": (
[{"role": "assistant", "content": entry_speech}]
if entry_mode == "fixed_speech"
else []
),
# Flows writes task_messages into the Pipecat LLM context. The
# pre-action below is responsible only for display, persistence,
# dynamic conversation history, and TTS playback.
"task_messages": fixed_reply_messages,
"functions": functions,
"context_strategy": ContextStrategyConfig(strategy=strategy),
"respond_immediately": entry_mode == "generate",
@@ -279,6 +299,7 @@ class WorkflowBrain(BaseBrain):
{
"type": "workflow_fixed_speech",
"text": entry_speech,
"node_id": node_id,
"handler": self._play_fixed_speech,
}
]
@@ -286,9 +307,19 @@ class WorkflowBrain(BaseBrain):
async def _play_fixed_speech(self, action: dict, _flow_manager: FlowManager) -> None:
"""Play and persist Agent entry speech without creating an LLM turn."""
await self._queue_visible_speech(str(action.get("text") or ""))
await self._queue_visible_speech(
str(action.get("text") or ""),
source="workflow-fixed-reply",
node_id=str(action.get("node_id") or "") or None,
)
async def _queue_visible_speech(self, text: str) -> None:
async def _queue_visible_speech(
self,
text: str,
*,
source: str = "workflow-speech",
node_id: str | None = None,
) -> None:
"""Show and persist fixed workflow speech before sending it to TTS."""
content = text.strip()
if not content:
@@ -302,6 +333,8 @@ class WorkflowBrain(BaseBrain):
"role": "assistant",
"content": content,
"timestamp": time_now_iso8601(),
"source": source,
**({"nodeId": node_id} if node_id else {}),
}
)
)
@@ -327,7 +360,13 @@ class WorkflowBrain(BaseBrain):
result = await self._tools.execute(tool, dict(args or {}))
except ToolExecutionError as exc:
return {"status": "error", "message": str(exc)}
if result.get("updated_variables"):
updated_variables = list(result.get("updated_variables") or [])
if updated_variables:
await self._emit_variables(
reason="tool",
node_id=node_id,
changed=updated_variables,
)
await self._refresh_agent_prompt(node_id)
edge = self._engine.deterministic_edge(
node_id,
@@ -436,11 +475,18 @@ class WorkflowBrain(BaseBrain):
return
try:
arguments = self._store.render_data(data.get("arguments") or {})
await self._tools.execute(
result = await self._tools.execute(
tool,
arguments,
result_assignments=data.get("resultAssignments") or {},
)
updated_variables = list(result.get("updated_variables") or [])
if updated_variables:
await self._emit_variables(
reason="action",
node_id=node_id,
changed=updated_variables,
)
self._store.values["system__last_action_status"] = "ok"
self._store.values["system__last_action_error"] = ""
except (ToolExecutionError, ValueError) as exc:
@@ -501,6 +547,40 @@ class WorkflowBrain(BaseBrain):
)
)
def _public_variables(self) -> dict[str, str | int | float | bool]:
"""Return the browser-safe part of this session's variable state."""
return {
name: value
for name, value in self._store.values.items()
if not name.startswith(("system__", "secret__"))
and isinstance(value, (str, int, float, bool))
}
async def _emit_variables(
self,
*,
reason: str,
node_id: str | None,
changed: list[str] | None = None,
) -> None:
"""Publish a safe snapshot so Workflow debug mirrors runtime state."""
message: dict[str, Any] = {
"type": "workflow-variables",
"reason": reason,
"variables": self._public_variables(),
}
if node_id:
message["nodeId"] = node_id
if changed:
message["changed"] = [
name
for name in changed
if not name.startswith(("system__", "secret__"))
]
await self._require_runtime().queue_frame(
OutputTransportMessageUrgentFrame(message=message)
)
def _require_runtime(self) -> BrainRuntime:
if self._runtime is None:
raise RuntimeError("WorkflowBrain 尚未绑定 pipeline runtime")

4
backend/services/conversation_history.py
View File

@@ -77,6 +77,10 @@ class ConversationRecorder:
role = str(message.get("role") or "")
content = str(message.get("content") or "").strip()
event_key = f"transcript:{role}:{timestamp}:{content}"
if message.get("source"):
extra["source"] = str(message["source"])
if message.get("nodeId"):
extra["node_id"] = str(message["nodeId"])
elif event_type == "assistant-text-end":
role = "assistant"
content = str(message.get("content") or "").strip()

2
backend/services/node_specs.py
View File

@@ -162,6 +162,8 @@ def _normalize_settings(settings: dict[str, Any], *, global_prompt: str = "") ->
settings.setdefault("knowledgeMode", "automatic")
settings.setdefault("knowledgeTopN", 5)
settings.setdefault("knowledgeScoreThreshold", 0.0)
settings.setdefault("enableInterrupt", True)
settings.setdefault("turnConfig", {})
def normalize_graph(graph: dict[str, Any] | None) -> dict[str, Any]:

33
backend/services/pipecat/pipeline.py
View File

@@ -10,6 +10,7 @@ import asyncio
import base64
from collections.abc import Callable
from io import BytesIO
from typing import Any
from uuid import uuid4
from loguru import logger
@@ -50,6 +51,7 @@ from pipecat.frames.frames import (
TTSSpeakFrame,
UserImageRawFrame,
UserImageRequestFrame,
VADParamsUpdateFrame,
)
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.llm_switcher import LLMSwitcher
@@ -58,7 +60,6 @@ from pipecat.pipeline.worker import PipelineParams, PipelineWorker
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.aggregators.llm_response_universal import (
LLMAssistantAggregator,
LLMUserAggregator,
LLMUserAggregatorParams,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -72,8 +73,10 @@ from pipecat.turns.user_mute.function_call_user_mute_strategy import (
FunctionCallUserMuteStrategy,
)
from services.pipecat.turn_config import (
ConfigurableLLMUserAggregator,
create_user_turn_strategies,
create_vad_analyzer,
create_vad_params,
)
from pipecat.utils.time import time_now_iso8601
from pipecat.workers.runner import WorkerRunner
@@ -794,7 +797,7 @@ async def run_pipeline(
current_llm_service = llm
if cfg.type == "workflow":
llm, llm_services, current_llm_service = _workflow_llm_switcher(cfg, llm)
user_aggregator = LLMUserAggregator(
user_aggregator = ConfigurableLLMUserAggregator(
context,
params=LLMUserAggregatorParams(
vad_analyzer=create_vad_analyzer(cfg.turnConfig),
@@ -1063,6 +1066,31 @@ async def run_pipeline(
)
)
current_enable_interrupt = cfg.enableInterrupt
current_turn_config = dict(cfg.turnConfig)
async def apply_workflow_turn_config(
enable_interrupt: bool,
turn_config: dict[str, Any],
) -> None:
"""Apply one Agent's interaction policy before its next user turn."""
nonlocal current_enable_interrupt, current_turn_config
normalized = dict(turn_config or {})
if (
current_enable_interrupt == enable_interrupt
and current_turn_config == normalized
):
return
await user_aggregator.apply_turn_strategies(
normalized,
enable_interruptions=enable_interrupt,
)
await worker.queue_frame(
VADParamsUpdateFrame(params=create_vad_params(normalized))
)
current_enable_interrupt = enable_interrupt
current_turn_config = normalized
async def queue_transcript(role: str, content: str, timestamp: str) -> None:
if content:
await worker.queue_frame(
@@ -1107,6 +1135,7 @@ async def run_pipeline(
switch_services=switch_workflow_services,
set_knowledge_scope=knowledge_retrieval.set_scope,
set_input_enabled=lambda enabled: input_state.__setitem__("enabled", enabled),
apply_turn_config=apply_workflow_turn_config,
flow_global_functions=flow_global_functions,
),
)

55
backend/services/pipecat/turn_config.py
View File

@@ -7,6 +7,11 @@ from typing import Any
from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.aggregators.llm_response_universal import (
LLMUserAggregator,
LLMUserAggregatorParams,
)
from pipecat.turns.user_start import (
TranscriptionUserTurnStartStrategy,
VADUserTurnStartStrategy,
@@ -39,18 +44,21 @@ def _value(config: dict[str, Any], snake: str, camel: str, default: Any) -> Any:
return config.get(snake, config.get(camel, default))
def create_vad_analyzer(turn_config: dict[str, Any]) -> SileroVADAnalyzer:
def create_vad_params(turn_config: dict[str, Any]) -> VADParams:
"""Translate product settings into Pipecat's runtime VAD parameters."""
vad = _section(turn_config, "vad", "vad")
return SileroVADAnalyzer(
params=VADParams(
confidence=float(vad.get("confidence", DEFAULT_VAD["confidence"])),
start_secs=float(_value(vad, "start_secs", "startSecs", 0.2)),
stop_secs=float(_value(vad, "stop_secs", "stopSecs", 0.2)),
min_volume=float(_value(vad, "min_volume", "minVolume", 0.6)),
)
return VADParams(
confidence=float(vad.get("confidence", DEFAULT_VAD["confidence"])),
start_secs=float(_value(vad, "start_secs", "startSecs", 0.2)),
stop_secs=float(_value(vad, "stop_secs", "stopSecs", 0.2)),
min_volume=float(_value(vad, "min_volume", "minVolume", 0.6)),
)
def create_vad_analyzer(turn_config: dict[str, Any]) -> SileroVADAnalyzer:
return SileroVADAnalyzer(params=create_vad_params(turn_config))
def create_user_turn_strategies(
turn_config: dict[str, Any], *, enable_interruptions: bool
) -> UserTurnStrategies:
@@ -87,3 +95,34 @@ def create_user_turn_strategies(
)
]
return UserTurnStrategies(start=start, stop=stop)
class ConfigurableLLMUserAggregator(LLMUserAggregator):
"""LLM user aggregator with one stable project-level runtime update API.
Pipecat 1.5 exposes ``UserTurnController.update_strategies`` but does not
surface it on ``LLMUserAggregator``. Keeping that version-specific bridge
here prevents Workflow orchestration from depending on Pipecat internals.
VAD threshold updates still use Pipecat's public ``VADParamsUpdateFrame``.
"""
def __init__(
self,
context: LLMContext,
*,
params: LLMUserAggregatorParams | None = None,
**kwargs: Any,
) -> None:
super().__init__(context, params=params, **kwargs)
async def apply_turn_strategies(
self,
turn_config: dict[str, Any],
*,
enable_interruptions: bool,
) -> None:
strategies = create_user_turn_strategies(
turn_config,
enable_interruptions=enable_interruptions,
)
await self._user_turn_controller.update_strategies(strategies)

77
backend/services/runtime_variables.py
View File

@@ -1,4 +1,4 @@
"""Conversation-scoped dynamic variables for prompt pipeline assistants.
"""Conversation-scoped dynamic variables shared by Prompt and Workflow.
The renderer is deliberately small: it only understands ``{{ name }}``
placeholders and never evaluates expressions. A value is substituted once,
@@ -21,6 +21,7 @@ from models import AssistantConfig
Primitive = str | int | float | bool
VARIABLE_NAME = re.compile(r"^[A-Za-z][A-Za-z0-9_]{0,63}$")
PLACEHOLDER = re.compile(r"{{\s*([A-Za-z][A-Za-z0-9_]*)\s*}}")
FULL_PLACEHOLDER = re.compile(r"^{{\s*([A-Za-z][A-Za-z0-9_]*)\s*}}$")
MAX_VARIABLES = 50
MAX_VALUE_LENGTH = 2048
MAX_HISTORY_ENTRIES = 50
@@ -91,37 +92,71 @@ class DynamicVariableStore:
self,
values: dict[str, Primitive],
secrets: dict[str, str] | None = None,
*,
optional_names: set[str] | None = None,
variable_types: dict[str, str] | None = None,
):
self.values = dict(values)
self.secrets = dict(secrets or {})
self.optional_names = set(optional_names or set())
self.variable_types = dict(variable_types or {})
self.history: list[dict[str, str]] = []
@classmethod
def from_config(cls, cfg: AssistantConfig) -> "DynamicVariableStore":
return cls(cfg.dynamic_variables, cfg.secret_dynamic_variables)
definitions = cfg.dynamic_variable_definitions or {}
optional_names = {
name
for name, definition in definitions.items()
if not definition.get("required", False)
and definition.get("default") is None
}
variable_types = {
name: str(definition.get("type") or "string")
for name, definition in definitions.items()
}
return cls(
cfg.dynamic_variables,
cfg.secret_dynamic_variables,
optional_names=optional_names,
variable_types=variable_types,
)
def render(self, template: str, *, allow_secrets: bool = False) -> str:
if not template:
return template
def _refresh_time(self) -> None:
timezone = str(self.values.get("system__timezone") or "Asia/Shanghai")
try:
now = datetime.now(ZoneInfo(timezone))
self.values["system__time"] = now.strftime("%A, %H:%M %d %B %Y")
self.values["system__time_utc"] = now.astimezone(ZoneInfo("UTC")).isoformat()
self.values["system__time_utc"] = now.astimezone(
ZoneInfo("UTC")
).isoformat()
except ZoneInfoNotFoundError:
pass
def _resolve(self, name: str, *, allow_secrets: bool) -> Primitive:
if name.startswith("secret__"):
if not allow_secrets:
raise DynamicVariableError(f"密钥变量 {name} 只能用于 HTTP Header")
if name not in self.secrets:
raise DynamicVariableError(f"缺少密钥变量: {name}")
return self.secrets[name]
if name in self.values:
return self.values[name]
# Optional variables intentionally remain absent from ``values`` so an
# ``exists`` expression can distinguish unset from an explicit value.
# Text templates still render predictably instead of failing the call.
if name in self.optional_names:
return ""
raise DynamicVariableError(f"缺少动态变量: {name}")
def render(self, template: str, *, allow_secrets: bool = False) -> str:
if not template:
return template
self._refresh_time()
def replace(match: re.Match[str]) -> str:
name = match.group(1)
if name.startswith("secret__"):
if not allow_secrets:
raise DynamicVariableError(f"密钥变量 {name} 只能用于 HTTP Header")
if name not in self.secrets:
raise DynamicVariableError(f"缺少密钥变量: {name}")
return self.secrets[name]
if name not in self.values:
raise DynamicVariableError(f"缺少动态变量: {name}")
value = self.values[name]
value = self._resolve(name, allow_secrets=allow_secrets)
if isinstance(value, bool):
return "true" if value else "false"
return str(value)
@@ -130,6 +165,12 @@ class DynamicVariableStore:
def render_data(self, value: Any, *, allow_secrets: bool = False) -> Any:
if isinstance(value, str):
exact = FULL_PLACEHOLDER.fullmatch(value)
if exact:
self._refresh_time()
return deepcopy(
self._resolve(exact.group(1), allow_secrets=allow_secrets)
)
return self.render(value, allow_secrets=allow_secrets)
if isinstance(value, list):
return [self.render_data(item, allow_secrets=allow_secrets) for item in value]
@@ -163,7 +204,11 @@ class DynamicVariableStore:
def assign(self, name: str, value: Any) -> None:
if name.startswith(("system__", "secret__")) or not VARIABLE_NAME.fullmatch(name):
raise DynamicVariableError(f"工具不能更新保留变量: {name}")
self.values[name] = _primitive(value, name)
primitive = _primitive(value, name)
expected = self.variable_types.get(name)
if expected and not _type_matches(primitive, expected):
raise DynamicVariableError(f"动态变量 {name} 类型应为 {expected}")
self.values[name] = primitive
def prepare_dynamic_config(

15
backend/services/workflow_engine.py
View File

@@ -23,6 +23,8 @@ class AgentStageConfig:
knowledge_mode: str
knowledge_top_n: int
knowledge_score_threshold: float
enable_interrupt: bool
turn_config: dict[str, Any]
class WorkflowEngine:
@@ -106,6 +108,12 @@ class WorkflowEngine:
asr_key = "defaultAsrResourceId" if inherits_global else "asrResourceId"
tts_key = "defaultTtsResourceId" if inherits_global else "ttsResourceId"
knowledge_base_id = str(source.get("knowledgeBaseId") or "")
global_turn_config = self.settings.get("turnConfig")
if not isinstance(global_turn_config, dict):
global_turn_config = {}
turn_config = source.get("turnConfig", global_turn_config)
if not isinstance(turn_config, dict):
turn_config = global_turn_config
return AgentStageConfig(
inherits_global=inherits_global,
llm_resource_id=str(source.get(llm_key) or ""),
@@ -122,6 +130,13 @@ class WorkflowEngine:
knowledge_score_threshold=float(
source.get("knowledgeScoreThreshold") or 0.0
),
enable_interrupt=bool(
source.get(
"enableInterrupt",
self.settings.get("enableInterrupt", True),
)
),
turn_config=dict(turn_config),
)
def prompt_for(self, node_id: str, store: DynamicVariableStore) -> str: