wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add workflow support and enhance runtime configuration in models and services

Browse Source 
- Introduce RuntimeModelResource and RuntimeKnowledgeBase classes to manage workflow resources.
- Update AssistantConfig to include workflow_model_resources and workflow_knowledge_bases for better integration.
- Refactor validation and processing logic in routes and services to accommodate workflow types.
- Implement dynamic variable support for workflow assistants and enhance graph normalization.
- Add ToolExecutor for reusable tool execution across different assistant types.
- Update various services to ensure compatibility with new workflow features and improve error handling.
This commit is contained in:
Xin Wang
2026-07-13 16:13:27 +08:00
parent 6108b00007
commit 32aef14ddb
27 changed files with 2563 additions and 910 deletions
Download Patch File Download Diff File Expand all files Collapse all files

233
backend/services/pipecat/pipeline.py
View File

@@ -23,6 +23,7 @@ from services.pipecat.call_lifecycle import (
EndCallAfterSpeechProcessor,
)
from services.pipecat.service_factory import (
config_with_resource,
create_realtime_service,
create_stt,
create_tts,
@@ -32,6 +33,7 @@ from services.knowledge import search as search_knowledge
from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema
from pipecat.flows import FlowsFunctionSchema
from pipecat.frames.frames import (
EndFrame,
InputTransportMessageFrame,
@@ -40,6 +42,7 @@ from pipecat.frames.frames import (
LLMFullResponseStartFrame,
LLMContextFrame,
LLMTextFrame,
ManuallySwitchServiceFrame,
LLMMessagesAppendFrame,
OutputTransportMessageUrgentFrame,
TextFrame,
@@ -48,6 +51,7 @@ from pipecat.frames.frames import (
UserImageRequestFrame,
)
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.service_switcher import ServiceSwitcher
from pipecat.pipeline.worker import PipelineParams, PipelineWorker
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.aggregators.llm_response_universal import (
@@ -397,15 +401,59 @@ class KnowledgeRetrievalProcessor(FrameProcessor):
self._knowledge_base_id = knowledge_base_id
self._top_n = top_n
self._score_threshold = score_threshold
self._mode = "automatic" if knowledge_base_id else "disabled"
self._last_signature = ""
def set_scope(self, scope: dict) -> None:
self._knowledge_base_id = scope.get("knowledge_base_id") or None
self._mode = str(scope.get("mode") or "disabled")
self._top_n = int(scope.get("top_n") or 5)
self._score_threshold = float(scope.get("score_threshold") or 0.0)
self._last_signature = ""
def _clear_context(self, messages: list[dict]) -> None:
# Remove the legacy Workflow knowledge message so an in-flight context
# created before this compatibility fix cannot keep sending that role.
messages[:] = [
message
for message in messages
if not (
message.get("role") == "developer"
and KNOWLEDGE_CONTEXT_MARKER in str(message.get("content") or "")
)
]
system_message = next(
(message for message in messages if message.get("role") == "system"),
None,
)
if system_message is not None:
content = str(system_message.get("content") or "")
system_message["content"] = content.split(KNOWLEDGE_CONTEXT_MARKER, 1)[0].rstrip()
def _set_context(self, messages: list[dict], block: str) -> None:
"""Store retrieved knowledge in a provider-compatible system message."""
self._clear_context(messages)
system_message = next(
(message for message in messages if message.get("role") == "system"),
None,
)
if system_message is None:
messages.insert(0, {"role": "system", "content": block})
return
content = str(system_message.get("content") or "").rstrip()
system_message["content"] = f"{content}\n\n{block}" if content else block
async def process_frame(self, frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if not self._knowledge_base_id or not isinstance(frame, LLMContextFrame):
if not isinstance(frame, LLMContextFrame):
await self.push_frame(frame, direction)
return
messages = frame.context.get_messages()
if self._mode != "automatic" or not self._knowledge_base_id:
self._clear_context(messages)
await self.push_frame(frame, direction)
return
user_messages = [message for message in messages if message.get("role") == "user"]
if not user_messages:
await self.push_frame(frame, direction)
@@ -435,13 +483,7 @@ class KnowledgeRetrievalProcessor(FrameProcessor):
for index, item in enumerate(results)
) or "未检索到相关资料。"
block = f"{KNOWLEDGE_CONTEXT_MARKER}\n当前问题的知识库检索结果：\n{sources}"
system_message = next((message for message in messages if message.get("role") == "system"), None)
if system_message is None:
messages.insert(0, {"role": "system", "content": block})
else:
content = str(system_message.get("content") or "")
base = content.split(KNOWLEDGE_CONTEXT_MARKER, 1)[0].rstrip()
system_message["content"] = f"{base}\n\n{block}" if base else block
self._set_context(messages, block)
await self.push_frame(frame, direction)
@@ -457,7 +499,6 @@ class PassthroughLLMAssistantAggregator(LLMAssistantAggregator):
self._stream_turn_id: str | None = None
self._stream_timestamp = ""
self._stream_text = ""
async def process_frame(self, frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -505,6 +546,49 @@ class PassthroughLLMAssistantAggregator(LLMAssistantAggregator):
self._stream_text = ""
class WorkflowAggregatorPair:
"""Small public-shape adapter required by Pipecat FlowManager."""
def __init__(self, user_aggregator, assistant_aggregator):
self._user = user_aggregator
self._assistant = assistant_aggregator
def user(self):
return self._user
def assistant(self):
return self._assistant
def _workflow_service_switcher(
cfg: AssistantConfig, capability: str, base_service: FrameProcessor
):
"""Build one switcher and an ID lookup for every referenced voice resource."""
create = create_stt if capability == "ASR" else create_tts
settings = cfg.graph.get("settings") or {}
default_key = (
"defaultAsrResourceId" if capability == "ASR" else "defaultTtsResourceId"
)
default_id = str(settings.get(default_key) or "")
services_by_id = {}
for resource_id, resource in cfg.workflow_model_resources.items():
if resource.capability != capability:
continue
services_by_id[resource_id] = (
base_service
if resource_id == default_id
else create(config_with_resource(cfg, resource))
)
primary = services_by_id.get(default_id, base_service)
services = [primary]
services.extend(
service for service in services_by_id.values() if service is not primary
)
if base_service is not primary:
services.append(base_service)
return ServiceSwitcher(services=services), services_by_id, primary
async def run_pipeline(
transport,
cfg: AssistantConfig,
@@ -545,8 +629,35 @@ async def run_pipeline(
)
return
stt = create_stt(cfg)
tts = create_tts(cfg)
graph_settings = cfg.graph.get("settings") or {}
default_asr_resource = cfg.workflow_model_resources.get(
str(graph_settings.get("defaultAsrResourceId") or "")
)
default_tts_resource = cfg.workflow_model_resources.get(
str(graph_settings.get("defaultTtsResourceId") or "")
)
stt = create_stt(
config_with_resource(cfg, default_asr_resource)
if cfg.type == "workflow" and default_asr_resource
else cfg
)
tts = create_tts(
config_with_resource(cfg, default_tts_resource)
if cfg.type == "workflow" and default_tts_resource
else cfg
)
stt_processor = stt
tts_processor = tts
stt_services: dict[str, FrameProcessor] = {}
tts_services: dict[str, FrameProcessor] = {}
current_voice_services: dict[str, FrameProcessor] = {"asr": stt, "tts": tts}
if cfg.type == "workflow":
stt_processor, stt_services, current_voice_services["asr"] = (
_workflow_service_switcher(cfg, "ASR", stt)
)
tts_processor, tts_services, current_voice_services["tts"] = (
_workflow_service_switcher(cfg, "TTS", tts)
)
greeting = await brain.greeting(cfg)
system_content = brain.system_prompt(cfg)
@@ -594,8 +705,13 @@ async def run_pipeline(
return "\n\n".join(part for part in [text, *hints] if part)
context = LLMContext(
messages=[{"role": "system", "content": with_vision_hint(system_content)}]
messages=(
[]
if cfg.type == "workflow"
else [{"role": "system", "content": with_vision_hint(system_content)}]
)
)
input_state = {"enabled": True}
# LLM 槽由大脑提供:本地模型或 Dify/FastGPT 外部托管适配器。
llm = brain.build_llm(cfg, context)
user_aggregator = LLMUserAggregator(
@@ -604,7 +720,9 @@ async def run_pipeline(
vad_analyzer=create_vad_analyzer(cfg.turnConfig),
user_mute_strategies=[
FunctionCallUserMuteStrategy(),
CallEndingUserMuteStrategy(lambda: call_end.ending),
CallEndingUserMuteStrategy(
lambda: call_end.ending or not input_state["enabled"]
),
],
user_turn_strategies=create_user_turn_strategies(
cfg.turnConfig,
@@ -613,7 +731,9 @@ async def run_pipeline(
),
)
assistant_aggregator = PassthroughLLMAssistantAggregator(context)
text_input = TextInputProcessor(should_ignore_input=lambda: call_end.ending)
text_input = TextInputProcessor(
should_ignore_input=lambda: call_end.ending or not input_state["enabled"]
)
vision_capture = VisionCaptureProcessor()
knowledge_retrieval = KnowledgeRetrievalProcessor(
automatic_knowledge_id,
@@ -723,14 +843,54 @@ async def run_pipeline(
}
)
if vision_enabled:
flow_global_functions = []
if cfg.type == "workflow" and vision_enabled:
async def flow_fetch_user_image(args, _flow_manager):
question = str((args or {}).get("question") or "请描述当前画面。")
user_id = vision_state.get("client_id")
if not user_id:
return {
"status": "no_video_client",
"message": "当前还没有可用的摄像头视频流。",
}
request = UserImageRequestFrame(
user_id=user_id,
text=question,
append_to_context=False,
function_name=VISION_TOOL_NAME,
)
try:
frame = await vision_capture.request_image(llm, request)
observation = await _analyze_image_with_vision_model(cfg, frame, question)
return {
"status": "ok",
"question": question,
"observation": observation or "视觉模型没有返回有效观察结果。",
}
except asyncio.TimeoutError:
return {"status": "timeout", "message": "等待摄像头视频帧超时。"}
except Exception as exc: # noqa: BLE001 - return tool errors to the LLM
logger.warning(f"Workflow 视觉理解失败:{exc}")
return {"status": "error", "message": "视觉理解暂时不可用。"}
flow_global_functions.append(
FlowsFunctionSchema(
name=VISION_TOOL_NAME,
description=vision_schema.description,
properties=vision_schema.properties,
required=vision_schema.required,
handler=flow_fetch_user_image,
)
)
if vision_enabled and cfg.type != "workflow":
llm.register_function(VISION_TOOL_NAME, fetch_user_image)
if cfg.knowledge_base_id and knowledge_mode == "on_demand":
llm.register_function(KNOWLEDGE_TOOL_NAME, search_bound_knowledge)
def set_visible_tools(schemas: list[FunctionSchema] | None = None) -> None:
tools = list(schemas or [])
if vision_enabled:
if vision_enabled and cfg.type != "workflow":
tools.append(vision_schema)
if cfg.knowledge_base_id and knowledge_mode == "on_demand":
tools.append(knowledge_schema)
@@ -751,7 +911,7 @@ async def run_pipeline(
transport.input(),
vision_capture,
text_input,
stt,
stt_processor,
user_aggregator,
knowledge_retrieval,
llm,
@@ -759,7 +919,7 @@ async def run_pipeline(
# Pipecat commits the generated prefix immediately instead of
# waiting for a TTS provider to emit spoken-text/timestamp frames.
assistant_aggregator,
tts,
tts_processor,
EndCallAfterSpeechProcessor(call_end),
ConversationHistoryProcessor(recorder),
transport.output(),
@@ -774,6 +934,33 @@ async def run_pipeline(
enable_rtvi=False,
)
worker_holder["worker"] = worker
default_voice_services = dict(current_voice_services)
async def switch_workflow_services(
asr_resource_id: str | None,
tts_resource_id: str | None,
) -> None:
requested = (
("asr", stt_services, asr_resource_id),
("tts", tts_services, tts_resource_id),
)
for kind, services, resource_id in requested:
target = services.get(resource_id) if resource_id else default_voice_services[kind]
if target is None:
raise ValueError(f"Workflow {kind.upper()} 资源未加载:{resource_id}")
if current_voice_services[kind] is target:
continue
await worker.queue_frame(ManuallySwitchServiceFrame(service=target))
current_voice_services[kind] = target
await worker.queue_frame(
OutputTransportMessageUrgentFrame(
message={
"type": "service-switched",
"capability": kind.upper(),
"resourceId": resource_id,
}
)
)
async def queue_transcript(role: str, content: str, timestamp: str) -> None:
if content:
@@ -810,6 +997,16 @@ async def run_pipeline(
set_system_prompt=set_system_prompt,
set_tools=set_visible_tools,
call_end=call_end,
worker=worker,
context_aggregator=WorkflowAggregatorPair(
user_aggregator,
assistant_aggregator,
),
transport=transport,
switch_services=switch_workflow_services,
set_knowledge_scope=knowledge_retrieval.set_scope,
set_input_enabled=lambda enabled: input_state.__setitem__("enabled", enabled),
flow_global_functions=flow_global_functions,
),
)

31
backend/services/pipecat/service_factory.py
View File

@@ -5,7 +5,7 @@
"""
from loguru import logger
from models import AssistantConfig
from models import AssistantConfig, RuntimeModelResource
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.services.openai.stt import OpenAISTTService
@@ -26,6 +26,35 @@ from services.pipecat.xfyun_tts import DEFAULT_XFYUN_TTS_URL, XfyunTTSService
TTS_STOP_FRAME_TIMEOUT_S = 1.0
def config_with_resource(
cfg: AssistantConfig, resource: RuntimeModelResource
) -> AssistantConfig:
"""Return a call-local config view for one workflow ASR/TTS resource."""
result = cfg.model_copy(deep=True)
values = resource.values or {}
secrets = resource.secrets or {}
if resource.capability == "ASR":
result.asr = str(values.get("modelId") or "")
result.stt_language = str(values.get("language") or "")
result.stt_interface_type = resource.interface_type
result.stt_values = values
result.stt_secrets = secrets
result.stt_api_key = str(secrets.get("apiKey") or "")
result.stt_base_url = str(values.get("apiUrl") or "")
elif resource.capability == "TTS":
result.tts_model = str(values.get("modelId") or "")
result.voice = str(values.get("voice") or "")
result.tts_speed = float(values.get("speed") or 1.0)
result.tts_interface_type = resource.interface_type
result.tts_values = values
result.tts_secrets = secrets
result.tts_api_key = str(secrets.get("apiKey") or "")
result.tts_base_url = str(values.get("apiUrl") or "")
else:
raise ValueError(f"工作流语音资源能力无效:{resource.capability}")
return result
def _require(value: str, label: str) -> str:
if value:
return value