wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance workflow routing and agent configuration management

Browse Source 
- Introduce WorkflowLLMRouter for pre-response LLM routing, allowing agents to determine the appropriate function to call based on user input.
- Implement UserTurnRoutingProcessor to manage user turns before reaching the LLM, ensuring proper routing and handling of user messages.
- Refactor WorkflowBrain to integrate new routing logic and enhance agent stage configuration, including entry modes and resource management.
- Update service factory to support dynamic LLM resource configuration based on workflow settings.
- Add tests for new routing functionality and ensure proper handling of user messages in various scenarios.
This commit is contained in:
Xin Wang
2026-07-14 09:36:28 +08:00
parent 32aef14ddb
commit 72856bf3a7
19 changed files with 2611 additions and 552 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
backend/services/pipecat/pipeline.py
View File

@@ -24,6 +24,7 @@ from services.pipecat.call_lifecycle import (
)
from services.pipecat.service_factory import (
config_with_resource,
create_llm,
create_realtime_service,
create_stt,
create_tts,
@@ -51,6 +52,7 @@ from pipecat.frames.frames import (
UserImageRequestFrame,
)
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.llm_switcher import LLMSwitcher
from pipecat.pipeline.service_switcher import ServiceSwitcher
from pipecat.pipeline.worker import PipelineParams, PipelineWorker
from pipecat.processors.aggregators.llm_context import LLMContext
@@ -487,6 +489,49 @@ class KnowledgeRetrievalProcessor(FrameProcessor):
await self.push_frame(frame, direction)
class UserTurnRoutingProcessor(FrameProcessor):
"""Give a brain first right of refusal before a new user turn reaches the LLM."""
def __init__(self, brain: Brain):
super().__init__()
self._brain = brain
self._last_user_message: dict | None = None
async def process_frame(self, frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if direction != FrameDirection.DOWNSTREAM or not isinstance(
frame, LLMContextFrame
):
await self.push_frame(frame, direction)
return
user_message = next(
(
message
for message in reversed(frame.context.get_messages())
if message.get("role") == "user"
and isinstance(message.get("content"), str)
and str(message.get("content") or "").strip()
),
None,
)
if user_message is None:
await self.push_frame(frame, direction)
return
if user_message is self._last_user_message:
# Programmatic LLMRunFrame after a node transition reuses the same
# user message. It is a response run, not another routing event.
await self.push_frame(frame, direction)
return
self._last_user_message = user_message
content = str(user_message.get("content") or "").strip()
handled = await self._brain.on_user_turn_end(content)
if not handled:
await self.push_frame(frame, direction)
class PassthroughLLMAssistantAggregator(LLMAssistantAggregator):
"""聚合 LLM 回复进上下文,同时继续把回复帧交给下游 TTS。"""
@@ -589,6 +634,29 @@ def _workflow_service_switcher(
return ServiceSwitcher(services=services), services_by_id, primary
def _workflow_llm_switcher(cfg: AssistantConfig, base_service):
"""Build an LLM switcher for the global model and Agent overrides."""
settings = cfg.graph.get("settings") or {}
default_id = str(settings.get("defaultLlmResourceId") or "")
services_by_id = {}
for resource_id, resource in cfg.workflow_model_resources.items():
if resource.capability != "LLM":
continue
services_by_id[resource_id] = (
base_service
if resource_id == default_id
else create_llm(config_with_resource(cfg, resource))
)
primary = services_by_id.get(default_id, base_service)
services = [primary]
services.extend(
service for service in services_by_id.values() if service is not primary
)
if base_service is not primary:
services.append(base_service)
return LLMSwitcher(llms=services), services_by_id, primary
async def run_pipeline(
transport,
cfg: AssistantConfig,
@@ -630,6 +698,9 @@ async def run_pipeline(
return
graph_settings = cfg.graph.get("settings") or {}
default_llm_resource = cfg.workflow_model_resources.get(
str(graph_settings.get("defaultLlmResourceId") or "")
)
default_asr_resource = cfg.workflow_model_resources.get(
str(graph_settings.get("defaultAsrResourceId") or "")
)
@@ -713,7 +784,16 @@ async def run_pipeline(
)
input_state = {"enabled": True}
# LLM 槽由大脑提供:本地模型或 Dify/FastGPT 外部托管适配器。
llm = brain.build_llm(cfg, context)
llm = brain.build_llm(
config_with_resource(cfg, default_llm_resource)
if cfg.type == "workflow" and default_llm_resource
else cfg,
context,
)
llm_services: dict[str, FrameProcessor] = {}
current_llm_service = llm
if cfg.type == "workflow":
llm, llm_services, current_llm_service = _workflow_llm_switcher(cfg, llm)
user_aggregator = LLMUserAggregator(
context,
params=LLMUserAggregatorParams(
@@ -730,6 +810,7 @@ async def run_pipeline(
),
),
)
user_turn_router = UserTurnRoutingProcessor(brain)
assistant_aggregator = PassthroughLLMAssistantAggregator(context)
text_input = TextInputProcessor(
should_ignore_input=lambda: call_end.ending or not input_state["enabled"]
@@ -880,6 +961,7 @@ async def run_pipeline(
properties=vision_schema.properties,
required=vision_schema.required,
handler=flow_fetch_user_image,
cancel_on_interruption=True,
)
)
@@ -913,6 +995,7 @@ async def run_pipeline(
text_input,
stt_processor,
user_aggregator,
user_turn_router,
knowledge_retrieval,
llm,
# Aggregate the streamed LLM text before TTS. On interruption,
@@ -934,24 +1017,42 @@ async def run_pipeline(
enable_rtvi=False,
)
worker_holder["worker"] = worker
default_voice_services = dict(current_voice_services)
default_workflow_services = {
"llm": current_llm_service,
**current_voice_services,
}
async def switch_workflow_services(
llm_resource_id: str | None,
asr_resource_id: str | None,
tts_resource_id: str | None,
) -> None:
nonlocal current_llm_service
requested = (
("llm", llm_services, llm_resource_id),
("asr", stt_services, asr_resource_id),
("tts", tts_services, tts_resource_id),
)
for kind, services, resource_id in requested:
target = services.get(resource_id) if resource_id else default_voice_services[kind]
target = (
services.get(resource_id)
if resource_id
else default_workflow_services[kind]
)
if target is None:
raise ValueError(f"Workflow {kind.upper()} 资源未加载:{resource_id}")
if current_voice_services[kind] is target:
current = (
current_llm_service
if kind == "llm"
else current_voice_services[kind]
)
if current is target:
continue
await worker.queue_frame(ManuallySwitchServiceFrame(service=target))
current_voice_services[kind] = target
if kind == "llm":
current_llm_service = target
else:
current_voice_services[kind] = target
await worker.queue_frame(
OutputTransportMessageUrgentFrame(
message={
@@ -1020,8 +1121,6 @@ async def run_pipeline(
@user_aggregator.event_handler("on_user_turn_stopped")
async def on_user_turn_stopped(_aggregator, _strategy, message):
if message.content:
brain.record_user_message(message.content)
await queue_transcript("user", message.content, message.timestamp)
@assistant_aggregator.event_handler("on_assistant_text_start")
@@ -1066,7 +1165,6 @@ async def run_pipeline(
@text_input.event_handler("on_text_input")
async def on_text_input(_processor, text):
pending_text_inputs.append(text)
brain.record_user_message(text)
# 前端显示不依赖 interruption 后续事件,必须在打断前先排入发送队列。
await queue_transcript("user", text, time_now_iso8601())

13
backend/services/pipecat/service_factory.py
View File

@@ -29,11 +29,18 @@ TTS_STOP_FRAME_TIMEOUT_S = 1.0
def config_with_resource(
cfg: AssistantConfig, resource: RuntimeModelResource
) -> AssistantConfig:
"""Return a call-local config view for one workflow ASR/TTS resource."""
"""Return a call-local config view for one workflow model resource."""
result = cfg.model_copy(deep=True)
values = resource.values or {}
secrets = resource.secrets or {}
if resource.capability == "ASR":
if resource.capability == "LLM":
result.model = str(values.get("modelId") or "")
result.llm_interface_type = resource.interface_type
result.llm_values = values
result.llm_secrets = secrets
result.llm_api_key = str(secrets.get("apiKey") or "")
result.llm_base_url = str(values.get("apiUrl") or "")
elif resource.capability == "ASR":
result.asr = str(values.get("modelId") or "")
result.stt_language = str(values.get("language") or "")
result.stt_interface_type = resource.interface_type
@@ -51,7 +58,7 @@ def config_with_resource(
result.tts_api_key = str(secrets.get("apiKey") or "")
result.tts_base_url = str(values.get("apiUrl") or "")
else:
raise ValueError(f"工作流语音资源能力无效:{resource.capability}")
raise ValueError(f"工作流模型资源能力无效:{resource.capability}")
return result