wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add configurable client tools and photo input

Browse Source
This commit is contained in:
Xin Wang
2026-07-30 19:06:03 +08:00
parent 510a277b5a
commit 913435785e
24 changed files with 1802 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
backend/services/pipecat/pipeline.py
View File

@@ -30,6 +30,8 @@ from services.pipecat.service_factory import (
)
from db.session import SessionLocal
from services.knowledge import search as search_knowledge
from services.client_tools import ClientToolBroker
from services.tool_policy import policy_for_tool
from services.vision import (
VISION_ANALYSIS_SYSTEM_PROMPT,
VISION_SYSTEM_HINT,
@@ -44,6 +46,7 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
from pipecat.flows import FlowsFunctionSchema
from pipecat.frames.frames import (
EndFrame,
LLMMessagesAppendFrame,
OutputTransportMessageUrgentFrame,
UserImageRawFrame,
UserImageRequestFrame,
@@ -57,9 +60,6 @@ from pipecat.processors.aggregators.llm_response_universal import (
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.llm_service import FunctionCallParams
from pipecat.turns.user_mute.function_call_user_mute_strategy import (
FunctionCallUserMuteStrategy,
)
from services.pipecat.turn_config import (
ConfigurableLLMUserAggregator,
create_user_turn_strategies,
@@ -73,11 +73,13 @@ from services.pipecat.processors import (
KnowledgeRetrievalProcessor,
PassthroughLLMAssistantAggregator,
RealtimeDynamicVariableProcessor,
RealtimeTextInputProcessor,
TextInputProcessor,
RealtimeUserInputProcessor,
UserInput,
UserInputProcessor,
UserTurnRoutingProcessor,
VisionCaptureProcessor,
WorkflowAggregatorPair,
ToolInterruptionUserMuteStrategy,
)
from services.pipecat.workflow_services import (
WorkflowServiceController,
@@ -319,6 +321,14 @@ async def run_pipeline(
)
)
input_state = {"enabled": True}
tool_interruption_strategy = ToolInterruptionUserMuteStrategy(
{
tool.function_name: policy_for_tool(tool).execution_mode
for tool in cfg.tools
if tool.type in {"http", "mcp", "client"}
and not policy_for_tool(tool).allow_interruptions
}
)
# LLM 槽由大脑提供:本地模型或 Dify/FastGPT 外部托管适配器。
llm = brain.build_llm(
config_with_resource(cfg, default_llm_resource)
@@ -335,10 +345,10 @@ async def run_pipeline(
params=LLMUserAggregatorParams(
vad_analyzer=create_vad_analyzer(cfg.turnConfig),
user_mute_strategies=[
FunctionCallUserMuteStrategy(),
CallEndingUserMuteStrategy(
lambda: call_end.ending or not input_state["enabled"]
),
tool_interruption_strategy,
],
user_turn_strategies=create_user_turn_strategies(
cfg.turnConfig,
@@ -348,9 +358,14 @@ async def run_pipeline(
)
user_turn_router = UserTurnRoutingProcessor(brain)
assistant_aggregator = PassthroughLLMAssistantAggregator(context)
text_input = TextInputProcessor(
should_ignore_input=lambda: call_end.ending or not input_state["enabled"]
user_input = UserInputProcessor(
should_ignore_input=lambda: (
call_end.ending
or not input_state["enabled"]
or tool_interruption_strategy.is_muted
)
)
client_tools = ClientToolBroker()
vision_capture = VisionCaptureProcessor()
knowledge_retrieval = KnowledgeRetrievalProcessor(
automatic_knowledge_id,
@@ -471,6 +486,24 @@ async def run_pipeline(
flow_global_functions = []
workflow_vision_function = None
def active_vision_config() -> AssistantConfig:
if cfg.type != "workflow":
return cfg
if not workflow_vision_scope.get("enabled"):
raise ValueError("当前 Workflow Agent 节点未启用视觉能力")
vision_resource_id = str(
workflow_vision_scope.get("vision_model_resource_id") or ""
)
llm_resource_id = str(workflow_vision_scope.get("llm_resource_id") or "")
resource_id = vision_resource_id or llm_resource_id
resource = cfg.workflow_model_resources.get(resource_id)
if resource:
return config_with_vision_resource(cfg, resource)
if not vision_resource_id:
return config_with_main_llm_as_vision(cfg)
raise ValueError(f"视觉模型资源未加载:{vision_resource_id}")
if cfg.type == "workflow" and vision_enabled:
async def flow_fetch_user_image(args, _flow_manager):
if not workflow_vision_scope.get("enabled"):
@@ -492,23 +525,9 @@ async def run_pipeline(
function_name=VISION_TOOL_NAME,
)
try:
vision_resource_id = str(
workflow_vision_scope.get("vision_model_resource_id") or ""
)
llm_resource_id = str(
workflow_vision_scope.get("llm_resource_id") or ""
)
resource_id = vision_resource_id or llm_resource_id
resource = cfg.workflow_model_resources.get(resource_id)
if resource:
vision_cfg = config_with_vision_resource(cfg, resource)
elif not vision_resource_id:
vision_cfg = config_with_main_llm_as_vision(cfg)
else:
raise ValueError(f"视觉模型资源未加载:{vision_resource_id}")
frame = await vision_capture.request_image(llm, request)
observation = await _analyze_image_with_vision_model(
vision_cfg,
active_vision_config(),
frame,
question,
)
@@ -558,8 +577,9 @@ async def run_pipeline(
pipeline = Pipeline(
[
transport.input(),
client_tools,
vision_capture,
text_input,
user_input,
stt_processor,
user_aggregator,
user_turn_router,
@@ -638,6 +658,7 @@ async def run_pipeline(
set_system_prompt=set_system_prompt,
set_tools=set_visible_tools,
call_end=call_end,
client_tools=client_tools,
worker=worker,
context_aggregator=WorkflowAggregatorPair(
user_aggregator,
@@ -654,17 +675,78 @@ async def run_pipeline(
),
)
async def submit_user_input(value: UserInput) -> None:
if not value.has_camera_frame:
if not value.run_immediately:
brain.record_user_message(value.text)
await worker.queue_frame(
LLMMessagesAppendFrame(
messages=[{"role": "user", "content": value.text}],
run_llm=value.run_immediately,
)
)
return
if not value.run_immediately:
raise ValueError("P0 图片输入必须立即触发回复")
if not vision_enabled:
raise ValueError("当前助手未启用视觉能力")
user_id = vision_state.get("client_id")
if not user_id:
raise ValueError("当前没有可用的摄像头视频流")
analysis_cfg = None if vision_native_mode else active_vision_config()
request = UserImageRequestFrame(
user_id=user_id,
text=value.prompt_text,
append_to_context=False,
)
try:
image_frame = await vision_capture.request_image(llm, request)
except asyncio.TimeoutError as exc:
raise ValueError("等待摄像头视频帧超时") from exc
if vision_native_mode:
image_frame.text = value.prompt_text
image_frame.append_to_context = True
image_frame.request = None
await worker.queue_frame(image_frame)
return
try:
assert analysis_cfg is not None
observation = await _analyze_image_with_vision_model(
analysis_cfg,
image_frame,
value.prompt_text,
)
except Exception as exc:
logger.exception(f"用户图片视觉分析失败: {exc}")
raise ValueError("视觉理解暂时不可用") from exc
content = (
f"{value.prompt_text}\n\n"
"[视觉模型对用户刚提交图片的观察]\n"
f"{observation or '视觉模型没有返回有效观察结果。'}"
)
await worker.queue_frame(
LLMMessagesAppendFrame(
messages=[{"role": "user", "content": content}],
run_llm=True,
)
)
bind_cascade_pipeline_events(
transport=transport,
worker=worker,
brain=brain,
context=context,
text_input=text_input,
text_input=user_input,
user_aggregator=user_aggregator,
assistant_aggregator=assistant_aggregator,
greeting=greeting,
vision_enabled=vision_enabled,
vision_state=vision_state,
submit_user_input=submit_user_input,
)
runner = WorkerRunner(handle_sigint=False)
run_status = "completed"
@@ -694,7 +776,7 @@ async def run_realtime_pipeline(
instructions=brain.system_prompt(cfg),
)
input_sample_rate, output_sample_rate = realtime_audio_sample_rates(cfg)
text_input = RealtimeTextInputProcessor()
user_input = RealtimeUserInputProcessor()
dynamic_variables = RealtimeDynamicVariableProcessor(brain, cfg, realtime)
greeting = await brain.greeting(cfg)
@@ -708,7 +790,7 @@ async def run_realtime_pipeline(
pipeline = Pipeline(
[
transport.input(),
text_input,
user_input,
realtime,
dynamic_variables,
ConversationHistoryProcessor(recorder),
@@ -729,7 +811,7 @@ async def run_realtime_pipeline(
transport=transport,
worker=worker,
realtime=realtime,
text_input=text_input,
text_input=user_input,
greeting=greeting,
)
runner = WorkerRunner(handle_sigint=False)

89
backend/services/pipecat/pipeline_events.py
View File

@@ -1,12 +1,13 @@
"""Event registration for cascade and realtime conversation pipelines."""
from collections.abc import Awaitable, Callable
from loguru import logger
from pipecat.frames.frames import (
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
EndFrame,
LLMMessagesAppendFrame,
OutputTransportMessageUrgentFrame,
TTSSpeakFrame,
)
@@ -15,6 +16,7 @@ from pipecat.runner.utils import (
maybe_capture_participant_camera,
)
from pipecat.utils.time import time_now_iso8601
from services.pipecat.processors import UserInput
def bind_cascade_pipeline_events(
@@ -29,10 +31,11 @@ def bind_cascade_pipeline_events(
greeting: str,
vision_enabled: bool,
vision_state: dict[str, str | None],
submit_user_input: Callable[[UserInput], Awaitable[None]] | None = None,
) -> None:
"""Connect processors to transport events without owning pipeline assembly."""
pending_text_inputs: list[str] = []
pending_user_inputs: list[UserInput] = []
greeting_transcript_sent = False
greeting_timestamp = ""
greeting_playback_pending = False
@@ -72,14 +75,33 @@ def bind_cascade_pipeline_events(
)
)
async def append_user_text_to_context(text: str, *, run_llm: bool) -> None:
async def queue_input_result(
user_input: UserInput,
status: str,
message: str = "",
) -> None:
await worker.queue_frame(
LLMMessagesAppendFrame(
messages=[{"role": "user", "content": text}],
run_llm=run_llm,
OutputTransportMessageUrgentFrame(
message={
"type": "user-input-result",
"input_id": user_input.input_id,
"status": status,
**({"message": message} if message else {}),
}
)
)
async def finish_user_input(user_input: UserInput) -> None:
try:
if submit_user_input is None:
raise RuntimeError("用户输入提交器尚未配置")
await submit_user_input(user_input)
except Exception as exc: # noqa: BLE001 - input errors must reach the client
logger.warning(f"用户输入处理失败: {exc}")
await queue_input_result(user_input, "error", str(exc))
return
await queue_input_result(user_input, "accepted")
@user_aggregator.event_handler("on_user_turn_stopped")
async def on_user_turn_stopped(_aggregator, _strategy, message):
await queue_transcript("user", message.content, message.timestamp)
@@ -123,24 +145,23 @@ def bind_cascade_pipeline_events(
)
await brain.on_assistant_text_end(turn_id, content, interrupted)
@text_input.event_handler("on_text_input")
async def on_text_input(_processor, text):
pending_text_inputs.append(text)
# The transcript must be queued before the interruption is broadcast.
await queue_transcript("user", text, time_now_iso8601())
@text_input.event_handler("on_user_input")
async def on_user_input(_processor, user_input: UserInput):
await queue_transcript(
"user",
user_input.transcript_text,
time_now_iso8601(),
)
if user_input.run_immediately and user_input.interrupt:
pending_user_inputs.append(user_input)
return
await finish_user_input(user_input)
@assistant_aggregator.event_handler("on_interruption_processed")
async def on_interruption_processed(_aggregator):
if not pending_text_inputs:
if not pending_user_inputs:
return
text = pending_text_inputs.pop(0)
await append_user_text_to_context(text, run_llm=True)
@text_input.event_handler("on_text_append")
async def on_text_append(_processor, text):
brain.record_user_message(text)
await queue_transcript("user", text, time_now_iso8601())
await append_user_text_to_context(text, run_llm=False)
await finish_user_input(pending_user_inputs.pop(0))
@text_input.event_handler("on_client_ready")
async def on_client_ready(_processor):
@@ -218,16 +239,24 @@ def bind_realtime_pipeline_events(
)
)
@text_input.event_handler("on_text_input")
async def on_text_input(_processor, text):
await queue_transcript("user", text)
await realtime.interrupt()
await realtime.send_text(text, run_immediately=True)
@text_input.event_handler("on_text_append")
async def on_text_append(_processor, text):
await queue_transcript("user", text)
await realtime.send_text(text, run_immediately=False)
@text_input.event_handler("on_user_input")
async def on_user_input(_processor, user_input: UserInput):
await queue_transcript("user", user_input.text)
if user_input.run_immediately and user_input.interrupt:
await realtime.interrupt()
await realtime.send_text(
user_input.text,
run_immediately=user_input.run_immediately,
)
await worker.queue_frame(
OutputTransportMessageUrgentFrame(
message={
"type": "user-input-result",
"input_id": user_input.input_id,
"status": "accepted",
}
)
)
@transport.event_handler("on_client_connected")
async def on_client_connected(_transport, _client):

292
backend/services/pipecat/processors.py
View File

@@ -2,6 +2,7 @@
import asyncio
from collections.abc import Callable
from dataclasses import dataclass
from uuid import uuid4
from loguru import logger
@@ -12,6 +13,11 @@ from services.knowledge import search as search_knowledge
from db.session import SessionLocal
from pipecat.frames.frames import (
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
FunctionCallCancelFrame,
FunctionCallResultFrame,
FunctionCallsStartedFrame,
InputTransportMessageFrame,
InterruptionFrame,
LLMContextFrame,
@@ -34,42 +40,120 @@ from pipecat.utils.time import time_now_iso8601
KNOWLEDGE_CONTEXT_MARKER = "<!-- knowledge-context -->"
def _text_input(message) -> tuple[str, bool] | None:
"""解析现有 user-text 与 RTVI send-text 两种前端文字消息。"""
@dataclass(frozen=True)
class UserInput:
"""Validated application input submitted as one user turn."""
input_id: str
text: str
has_camera_frame: bool
run_immediately: bool
interrupt: bool
@property
def prompt_text(self) -> str:
if self.text:
return self.text
return "请根据用户刚提交的图片进行回复。"
@property
def transcript_text(self) -> str:
if not self.has_camera_frame:
return self.text
return f"{self.text}\n已发送一张图片".strip()
class UserInputError(ValueError):
def __init__(self, message: str, *, input_id: str = "") -> None:
super().__init__(message)
self.input_id = input_id
def parse_user_input(message) -> UserInput | None:
"""Parse the sole public user-input wire format."""
if not isinstance(message, dict):
return None
if message.get("type") == "user-text":
text = str(message.get("text") or "").strip()
return (text, True) if text else None
if message.get("type") == "send-text":
data = message.get("data")
if not isinstance(data, dict):
return None
text = str(data.get("content") or "").strip()
options = data.get("options")
run_immediately = not isinstance(options, dict) or options.get(
"run_immediately", True
)
return (text, bool(run_immediately)) if text else None
return None
if message.get("type") != "user-input":
return None
input_id = str(message.get("input_id") or "").strip()
if not input_id or len(input_id) > 128:
raise UserInputError("user-input 缺少有效的 input_id", input_id=input_id)
if message.get("schema_version") != 1:
raise UserInputError("不支持的 user-input schema_version", input_id=input_id)
parts = message.get("parts")
if not isinstance(parts, list) or not parts:
raise UserInputError("user-input parts 不能为空", input_id=input_id)
text = ""
has_camera_frame = False
for part in parts:
if not isinstance(part, dict):
raise UserInputError("user-input part 格式不正确", input_id=input_id)
part_type = part.get("type")
if part_type == "input_text":
if text:
raise UserInputError("P0 只支持一个 input_text", input_id=input_id)
text = str(part.get("text") or "").strip()
if not text:
raise UserInputError("input_text 不能为空", input_id=input_id)
elif part_type == "input_image":
if has_camera_frame:
raise UserInputError("P0 只支持一张图片", input_id=input_id)
source = part.get("source")
if (
not isinstance(source, dict)
or source.get("type") != "camera_frame"
or source.get("frame") != "current"
):
raise UserInputError(
"P0 只支持当前摄像头画面",
input_id=input_id,
)
has_camera_frame = True
else:
raise UserInputError(f"不支持的输入类型: {part_type}", input_id=input_id)
if not text and not has_camera_frame:
raise UserInputError("user-input 没有有效内容", input_id=input_id)
options = message.get("options")
options = options if isinstance(options, dict) else {}
run_immediately = bool(options.get("run_immediately", True))
interrupt = bool(options.get("interrupt", True))
return UserInput(
input_id=input_id,
text=text,
has_camera_frame=has_camera_frame,
run_immediately=run_immediately,
interrupt=interrupt,
)
class TextInputProcessor(FrameProcessor):
"""把 transport 文字消息转换成 LLM 可消费的帧。
def message_text(message: dict) -> str:
"""Extract textual content from plain or multimodal LLM messages."""
content = message.get("content")
if isinstance(content, str):
return content.strip()
if not isinstance(content, list):
return ""
return "\n".join(
str(part.get("text") or "").strip()
for part in content
if isinstance(part, dict)
and part.get("type") in {"text", "input_text"}
and str(part.get("text") or "").strip()
).strip()
run_immediately(默认/打断):先通过 on_text_input 事件把用户文字交给
run_pipeline 登记,再用 broadcast_interruption() 打断当前播报。新的 LLM
回复由 assistant aggregator 确认处理完 interruption 后触发。
run_immediately=False(RTVI send-text 静默追加):仅把文字写进上下文,
不打断、不触发推理。
"""
class UserInputProcessor(FrameProcessor):
"""Validate app inputs and coordinate their interruption boundary."""
def __init__(self, should_ignore_input: Callable[[], bool] | None = None):
super().__init__()
self._should_ignore_input = should_ignore_input or (lambda: False)
# 立即触发的文字(含打断语义)走 on_text_input;静默追加另走一条事件
self._register_event_handler("on_text_input")
self._register_event_handler("on_text_append")
self._register_event_handler("on_user_input")
self._register_event_handler("on_client_ready")
async def process_frame(self, frame, direction: FrameDirection):
@@ -83,23 +167,35 @@ class TextInputProcessor(FrameProcessor):
await self._call_event_handler("on_client_ready")
return
parsed = _text_input(frame.message)
if not parsed:
try:
user_input = parse_user_input(frame.message)
except UserInputError as exc:
await self._emit_result(exc.input_id, "error", str(exc))
return
if user_input is None:
await self.push_frame(frame, direction)
return
if self._should_ignore_input():
logger.debug("通话正在结束,忽略后续文字输入")
await self._emit_result(user_input.input_id, "error", "当前不能接收新的用户输入")
return
text, run_immediately = parsed
if run_immediately:
# 先登记文字再打断。下一轮 LLM 由 assistant aggregator 在真正处理完
# InterruptionFrame 后触发,避免新回复被这次 interruption 一起取消。
await self._call_event_handler("on_text_input", text)
await self._call_event_handler("on_user_input", user_input)
if user_input.run_immediately and user_input.interrupt:
await self.broadcast_interruption()
else:
await self._call_event_handler("on_text_append", text)
async def _emit_result(self, input_id: str, status: str, message: str = "") -> None:
await self.push_frame(
OutputTransportMessageUrgentFrame(
message={
"type": "user-input-result",
"input_id": input_id,
"status": status,
**({"message": message} if message else {}),
}
)
)
class CallEndingUserMuteStrategy(BaseUserMuteStrategy):
@@ -114,6 +210,84 @@ class CallEndingUserMuteStrategy(BaseUserMuteStrategy):
return self._is_call_ending()
@dataclass
class _BlockedToolCall:
tool_finished: bool = False
response_started: bool = False
response_finished: bool = False
class ToolInterruptionUserMuteStrategy(BaseUserMuteStrategy):
"""Mute user media for selected tools and their associated bot response.
Pipecat's ``cancel_on_interruption`` also controls whether a tool is
synchronous or asynchronous. This strategy keeps the independent product
setting ("allow user interruptions") at the user-input boundary.
"""
def __init__(
self,
blocked_tools: dict[str, str] | set[str],
):
super().__init__()
self._blocked_tools = (
dict(blocked_tools)
if isinstance(blocked_tools, dict)
else {name: "immediate" for name in blocked_tools}
)
self._calls: dict[str, _BlockedToolCall] = {}
self._bot_speaking = False
@property
def is_muted(self) -> bool:
return bool(self._calls)
async def process_frame(self, frame) -> bool:
await super().process_frame(frame)
if isinstance(frame, BotStartedSpeakingFrame):
self._bot_speaking = True
for call in self._calls.values():
call.response_started = True
elif isinstance(frame, BotStoppedSpeakingFrame):
self._bot_speaking = False
for call in self._calls.values():
if call.response_started:
call.response_finished = True
self._release_completed()
elif isinstance(frame, FunctionCallsStartedFrame):
for call in frame.function_calls:
execution_mode = self._blocked_tools.get(call.function_name)
if execution_mode:
self._calls[call.tool_call_id] = _BlockedToolCall(
# An async tool may continue within the current model
# turn. An immediate tool's current speech is a preamble;
# its protected follow-up starts only after the result.
response_started=(
self._bot_speaking and execution_mode == "async"
),
)
elif isinstance(frame, FunctionCallResultFrame):
call = self._calls.get(frame.tool_call_id)
if call is not None:
call.tool_finished = True
self._release_completed()
elif isinstance(frame, FunctionCallCancelFrame):
# A canceled call cannot reliably produce a follow-up response.
self._calls.pop(frame.tool_call_id, None)
return self.is_muted
def _release_completed(self) -> None:
completed = [
call_id
for call_id, state in self._calls.items()
if state.tool_finished and state.response_finished
]
for call_id in completed:
self._calls.pop(call_id, None)
class VisionCaptureProcessor(FrameProcessor):
"""Capture one requested video frame for auxiliary vision-model analysis."""
@@ -193,13 +367,12 @@ class RealtimeDynamicVariableProcessor(FrameProcessor):
await self.push_frame(frame, direction)
class RealtimeTextInputProcessor(FrameProcessor):
"""Route text input directly to a realtime service without cascade semantics."""
class RealtimeUserInputProcessor(FrameProcessor):
"""Route text-only user-input messages to a realtime service."""
def __init__(self):
super().__init__()
self._register_event_handler("on_text_input")
self._register_event_handler("on_text_append")
self._register_event_handler("on_user_input")
async def process_frame(self, frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -208,15 +381,32 @@ class RealtimeTextInputProcessor(FrameProcessor):
await self.push_frame(frame, direction)
return
parsed = _text_input(frame.message)
if not parsed:
try:
user_input = parse_user_input(frame.message)
except UserInputError as exc:
await self._emit_error(exc.input_id, str(exc))
return
if user_input is None:
await self.push_frame(frame, direction)
return
if user_input.has_camera_frame:
await self._emit_error(
user_input.input_id,
"Realtime 模式暂不支持图片输入",
)
return
await self._call_event_handler("on_user_input", user_input)
text, run_immediately = parsed
await self._call_event_handler(
"on_text_input" if run_immediately else "on_text_append",
text,
async def _emit_error(self, input_id: str, message: str) -> None:
await self.push_frame(
OutputTransportMessageUrgentFrame(
message={
"type": "user-input-result",
"input_id": input_id,
"status": "error",
"message": message,
}
)
)
@@ -304,7 +494,7 @@ class KnowledgeRetrievalProcessor(FrameProcessor):
if not user_messages:
await self.push_frame(frame, direction)
return
query = str(user_messages[-1].get("content") or "").strip()
query = message_text(user_messages[-1])
signature = f"{len(user_messages)}:{query}"
if not query or signature == self._last_signature:
await self.push_frame(frame, direction)
@@ -354,8 +544,7 @@ class UserTurnRoutingProcessor(FrameProcessor):
message
for message in reversed(frame.context.get_messages())
if message.get("role") == "user"
and isinstance(message.get("content"), str)
and str(message.get("content") or "").strip()
and message_text(message)
),
None,
)
@@ -370,7 +559,7 @@ class UserTurnRoutingProcessor(FrameProcessor):
return
self._last_user_message = user_message
content = str(user_message.get("content") or "").strip()
content = message_text(user_message)
handled = await self._brain.on_user_turn_end(content)
if not handled:
await self.push_frame(frame, direction)
@@ -447,6 +636,3 @@ class WorkflowAggregatorPair:
def assistant(self):
return self._assistant