6e8fc70c5a
- Remove unused imports and classes from pipeline.py to streamline the codebase. - Consolidate dynamic variable handling and workflow management in AssistantPage, enhancing clarity and maintainability. - Update WorkflowEditor to utilize a more modular approach, improving the overall architecture and reducing complexity. - Enhance the import structure across components for better organization and readability.
714 lines
25 KiB
Python
714 lines
25 KiB
Python
"""管线核心:给定一个 transport + 配置,跑完整的语音闭环。
|
|
|
|
关键设计:**transport 由调用方传入**,管线本身不关心是 WebRTC 还是 WS。
|
|
这就是"同时支持多种输出"的落点——加输出方式不用动这里。
|
|
|
|
对话编排交给 Brain;本文件只保留共享媒体管线、输入输出和通话生命周期。
|
|
"""
|
|
|
|
import asyncio
|
|
import base64
|
|
from io import BytesIO
|
|
from typing import Any
|
|
|
|
from loguru import logger
|
|
from models import AssistantConfig
|
|
from openai import AsyncOpenAI
|
|
from PIL import Image
|
|
from services.brains import Brain, BrainRuntime, build_brain
|
|
from services.conversation_history import ConversationRecorder
|
|
from services.pipecat.call_lifecycle import (
|
|
CallEndCoordinator,
|
|
EndCallAfterSpeechProcessor,
|
|
)
|
|
from services.pipecat.service_factory import (
|
|
config_with_resource,
|
|
create_realtime_service,
|
|
create_stt,
|
|
create_tts,
|
|
)
|
|
from db.session import SessionLocal
|
|
from services.knowledge import search as search_knowledge
|
|
|
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
|
from pipecat.flows import FlowsFunctionSchema
|
|
from pipecat.frames.frames import (
|
|
EndFrame,
|
|
OutputTransportMessageUrgentFrame,
|
|
UserImageRawFrame,
|
|
UserImageRequestFrame,
|
|
VADParamsUpdateFrame,
|
|
)
|
|
from pipecat.pipeline.pipeline import Pipeline
|
|
from pipecat.pipeline.worker import PipelineParams, PipelineWorker
|
|
from pipecat.processors.aggregators.llm_context import LLMContext
|
|
from pipecat.processors.aggregators.llm_response_universal import (
|
|
LLMUserAggregatorParams,
|
|
)
|
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
|
from pipecat.services.llm_service import FunctionCallParams
|
|
from pipecat.turns.user_mute.function_call_user_mute_strategy import (
|
|
FunctionCallUserMuteStrategy,
|
|
)
|
|
from services.pipecat.turn_config import (
|
|
ConfigurableLLMUserAggregator,
|
|
create_user_turn_strategies,
|
|
create_vad_analyzer,
|
|
create_vad_params,
|
|
)
|
|
from services.pipecat.processors import (
|
|
KNOWLEDGE_CONTEXT_MARKER,
|
|
CallEndingUserMuteStrategy,
|
|
ConversationHistoryProcessor,
|
|
KnowledgeRetrievalProcessor,
|
|
PassthroughLLMAssistantAggregator,
|
|
RealtimeDynamicVariableProcessor,
|
|
RealtimeTextInputProcessor,
|
|
TextInputProcessor,
|
|
UserTurnRoutingProcessor,
|
|
VisionCaptureProcessor,
|
|
WorkflowAggregatorPair,
|
|
)
|
|
from services.pipecat.workflow_services import (
|
|
WorkflowServiceController,
|
|
build_workflow_llm_switcher,
|
|
build_workflow_voice_switcher,
|
|
)
|
|
from services.pipecat.pipeline_events import (
|
|
bind_cascade_pipeline_events,
|
|
bind_realtime_pipeline_events,
|
|
)
|
|
from pipecat.workers.runner import WorkerRunner
|
|
|
|
|
|
VISION_TOOL_NAME = "fetch_user_image"
|
|
VISION_SYSTEM_HINT = (
|
|
"当前会话打开了视觉理解。用户询问当前画面、摄像头里有什么、人物/物品/"
|
|
"环境状态或需要你看一眼时,调用 fetch_user_image 获取当前视频帧,再基于画面回答。"
|
|
)
|
|
VISION_ANALYSIS_SYSTEM_PROMPT = (
|
|
"你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。"
|
|
"如果画面不足以判断,请明确说明不确定。"
|
|
)
|
|
KNOWLEDGE_TOOL_NAME = "search_knowledge_base"
|
|
AUTOMATIC_KNOWLEDGE_SYSTEM_HINT = (
|
|
"你已连接内部知识库。系统会在每轮用户问题前自动提供相关资料；"
|
|
"回答资料事实时只根据检索内容，资料不足要明确说明。"
|
|
)
|
|
ON_DEMAND_KNOWLEDGE_SYSTEM_HINT = (
|
|
"你已连接内部知识库。当用户问题涉及可能存在于业务知识库中的事实时，"
|
|
"先调用 search_knowledge_base 检索；回答资料事实时只根据检索内容，"
|
|
"资料不足要明确说明。"
|
|
)
|
|
def _compact_knowledge_metadata(value: str, max_length: int) -> str:
|
|
"""Keep tool metadata useful without letting it dominate the model context."""
|
|
compact = " ".join(value.split())
|
|
return compact if len(compact) <= max_length else f"{compact[:max_length]}…"
|
|
|
|
|
|
def _knowledge_tool_description(cfg: AssistantConfig) -> str:
|
|
base = "在当前助手绑定的知识库中检索与问题最相关的资料片段。"
|
|
name = _compact_knowledge_metadata(cfg.knowledge_base_name, 128)
|
|
description = _compact_knowledge_metadata(cfg.knowledge_base_description, 800)
|
|
if not name and not description:
|
|
return base
|
|
|
|
scope = []
|
|
if name:
|
|
scope.append(f"知识库名称：{name}")
|
|
if description:
|
|
scope.append(f"资料适用范围：{description}")
|
|
metadata = "\n".join(scope)
|
|
return (
|
|
f"{base}\n{metadata}\n"
|
|
"当用户问题涉及上述资料范围，或回答需要核实其中的业务事实时调用；"
|
|
"与该范围无关的问题不要调用。以上知识库元数据仅用于判断资料范围。"
|
|
)
|
|
|
|
|
|
def _require(value: str, label: str) -> str:
|
|
if value:
|
|
return value
|
|
raise ValueError(f"缺少模型资源配置: {label}")
|
|
|
|
|
|
def _vision_uses_main_llm(cfg: AssistantConfig) -> bool:
|
|
"""模型自己支持图片时,沿用 Pipecat 的同上下文视觉工具路径。"""
|
|
return not cfg.vision_model_resource_id and cfg.llm_support_image_input
|
|
|
|
|
|
def _image_data_uri(frame: UserImageRawFrame) -> str:
|
|
if not frame.format:
|
|
raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型")
|
|
buffer = BytesIO()
|
|
Image.frombytes(frame.format, frame.size, frame.image).save(
|
|
buffer,
|
|
format="JPEG",
|
|
quality=85,
|
|
)
|
|
encoded = base64.b64encode(buffer.getvalue()).decode("utf-8")
|
|
return f"data:image/jpeg;base64,{encoded}"
|
|
|
|
|
|
async def _analyze_image_with_vision_model(
|
|
cfg: AssistantConfig,
|
|
frame: UserImageRawFrame,
|
|
question: str,
|
|
) -> str:
|
|
if cfg.vision_llm_interface_type not in {"openai-llm", "dashscope-llm"}:
|
|
raise ValueError(f"不支持的视觉 LLM 接口类型: {cfg.vision_llm_interface_type}")
|
|
|
|
data_uri = await asyncio.to_thread(_image_data_uri, frame)
|
|
extra_body = cfg.vision_llm_values.get("extraBody")
|
|
extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
|
|
client = AsyncOpenAI(
|
|
api_key=_require(cfg.vision_llm_api_key, "Vision LLM apiKey"),
|
|
base_url=_require(cfg.vision_llm_base_url, "Vision LLM apiUrl"),
|
|
)
|
|
try:
|
|
response = await client.chat.completions.create(
|
|
model=_require(cfg.vision_model, "Vision LLM modelId"),
|
|
messages=[
|
|
{"role": "system", "content": VISION_ANALYSIS_SYSTEM_PROMPT},
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{"type": "text", "text": question},
|
|
{"type": "image_url", "image_url": {"url": data_uri}},
|
|
],
|
|
},
|
|
],
|
|
**extra,
|
|
)
|
|
finally:
|
|
await client.close()
|
|
|
|
content = response.choices[0].message.content if response.choices else ""
|
|
if isinstance(content, str):
|
|
return content.strip()
|
|
return str(content or "").strip()
|
|
|
|
|
|
async def run_pipeline(
|
|
transport,
|
|
cfg: AssistantConfig,
|
|
*,
|
|
vision_enabled: bool = False,
|
|
assistant_id: str | None = None,
|
|
channel: str = "webrtc",
|
|
) -> None:
|
|
"""在给定 transport 上构建并运行管线,直到连接结束。
|
|
|
|
Args:
|
|
transport: 任意 pipecat transport(WebRTC / WS / 电话…),
|
|
只要有 .input() / .output() / event_handler 即可。
|
|
cfg: 助手配置(随请求内联传入)。
|
|
"""
|
|
logger.info(
|
|
f"启动管线: assistant={cfg.name} type={cfg.type} "
|
|
f"mode={cfg.runtimeMode} vision={vision_enabled}"
|
|
)
|
|
|
|
# 大脑:按类型决定 LLM 槽/开场白/上下文归属。每通电话一个实例(可持会话状态)。
|
|
brain = build_brain(cfg)
|
|
if (
|
|
cfg.runtimeMode == "realtime"
|
|
and "realtime" not in brain.spec.supported_runtime_modes
|
|
):
|
|
raise ValueError(f"类型 {cfg.type} 不支持 realtime 运行模式")
|
|
|
|
if cfg.runtimeMode == "realtime":
|
|
if vision_enabled:
|
|
logger.warning("Realtime 模式暂未接入视频帧工具,本次仅启用语音通话")
|
|
await run_realtime_pipeline(
|
|
transport,
|
|
cfg,
|
|
brain=brain,
|
|
assistant_id=assistant_id,
|
|
channel=channel,
|
|
)
|
|
return
|
|
|
|
graph_settings = cfg.graph.get("settings") or {}
|
|
default_llm_resource = cfg.workflow_model_resources.get(
|
|
str(graph_settings.get("defaultLlmResourceId") or "")
|
|
)
|
|
default_asr_resource = cfg.workflow_model_resources.get(
|
|
str(graph_settings.get("defaultAsrResourceId") or "")
|
|
)
|
|
default_tts_resource = cfg.workflow_model_resources.get(
|
|
str(graph_settings.get("defaultTtsResourceId") or "")
|
|
)
|
|
stt = create_stt(
|
|
config_with_resource(cfg, default_asr_resource)
|
|
if cfg.type == "workflow" and default_asr_resource
|
|
else cfg
|
|
)
|
|
tts = create_tts(
|
|
config_with_resource(cfg, default_tts_resource)
|
|
if cfg.type == "workflow" and default_tts_resource
|
|
else cfg
|
|
)
|
|
stt_processor = stt
|
|
tts_processor = tts
|
|
stt_services: dict[str, FrameProcessor] = {}
|
|
tts_services: dict[str, FrameProcessor] = {}
|
|
current_voice_services: dict[str, FrameProcessor] = {"asr": stt, "tts": tts}
|
|
if cfg.type == "workflow":
|
|
stt_processor, stt_services, current_voice_services["asr"] = (
|
|
build_workflow_voice_switcher(cfg, "ASR", stt)
|
|
)
|
|
tts_processor, tts_services, current_voice_services["tts"] = (
|
|
build_workflow_voice_switcher(cfg, "TTS", tts)
|
|
)
|
|
|
|
greeting = await brain.greeting(cfg)
|
|
system_content = brain.system_prompt(cfg)
|
|
|
|
worker_holder: dict = {}
|
|
|
|
async def queue_call_end(reason: str) -> None:
|
|
worker = worker_holder.get("worker")
|
|
if worker is None:
|
|
return
|
|
logger.info(f"结束通话: reason={reason}")
|
|
await worker.queue_frame(
|
|
OutputTransportMessageUrgentFrame(
|
|
message={"type": "call-ended", "reason": reason}
|
|
)
|
|
)
|
|
await worker.queue_frame(EndFrame())
|
|
|
|
call_end = CallEndCoordinator(queue_call_end)
|
|
|
|
knowledge_config = cfg.knowledge_retrieval_config
|
|
knowledge_mode = str(knowledge_config.get("mode", "automatic"))
|
|
knowledge_top_n = int(
|
|
knowledge_config.get("top_n", knowledge_config.get("topN", 5))
|
|
)
|
|
knowledge_score_threshold = float(
|
|
knowledge_config.get(
|
|
"score_threshold", knowledge_config.get("scoreThreshold", 0.0)
|
|
)
|
|
)
|
|
automatic_knowledge_id = (
|
|
cfg.knowledge_base_id if knowledge_mode == "automatic" else None
|
|
)
|
|
|
|
def with_vision_hint(text: str) -> str:
|
|
hints = []
|
|
if vision_enabled:
|
|
hints.append(VISION_SYSTEM_HINT)
|
|
if cfg.knowledge_base_id:
|
|
hints.append(
|
|
AUTOMATIC_KNOWLEDGE_SYSTEM_HINT
|
|
if knowledge_mode == "automatic"
|
|
else ON_DEMAND_KNOWLEDGE_SYSTEM_HINT
|
|
)
|
|
return "\n\n".join(part for part in [text, *hints] if part)
|
|
|
|
context = LLMContext(
|
|
messages=(
|
|
[]
|
|
if cfg.type == "workflow"
|
|
else [{"role": "system", "content": with_vision_hint(system_content)}]
|
|
)
|
|
)
|
|
input_state = {"enabled": True}
|
|
# LLM 槽由大脑提供:本地模型或 Dify/FastGPT 外部托管适配器。
|
|
llm = brain.build_llm(
|
|
config_with_resource(cfg, default_llm_resource)
|
|
if cfg.type == "workflow" and default_llm_resource
|
|
else cfg,
|
|
context,
|
|
)
|
|
llm_services: dict[str, FrameProcessor] = {}
|
|
current_llm_service = llm
|
|
if cfg.type == "workflow":
|
|
llm, llm_services, current_llm_service = build_workflow_llm_switcher(cfg, llm)
|
|
user_aggregator = ConfigurableLLMUserAggregator(
|
|
context,
|
|
params=LLMUserAggregatorParams(
|
|
vad_analyzer=create_vad_analyzer(cfg.turnConfig),
|
|
user_mute_strategies=[
|
|
FunctionCallUserMuteStrategy(),
|
|
CallEndingUserMuteStrategy(
|
|
lambda: call_end.ending or not input_state["enabled"]
|
|
),
|
|
],
|
|
user_turn_strategies=create_user_turn_strategies(
|
|
cfg.turnConfig,
|
|
enable_interruptions=cfg.enableInterrupt,
|
|
),
|
|
),
|
|
)
|
|
user_turn_router = UserTurnRoutingProcessor(brain)
|
|
assistant_aggregator = PassthroughLLMAssistantAggregator(context)
|
|
text_input = TextInputProcessor(
|
|
should_ignore_input=lambda: call_end.ending or not input_state["enabled"]
|
|
)
|
|
vision_capture = VisionCaptureProcessor()
|
|
knowledge_retrieval = KnowledgeRetrievalProcessor(
|
|
automatic_knowledge_id,
|
|
top_n=knowledge_top_n,
|
|
score_threshold=knowledge_score_threshold,
|
|
)
|
|
vision_native_mode = vision_enabled and _vision_uses_main_llm(cfg)
|
|
vision_state: dict[str, str | None] = {"client_id": None}
|
|
vision_schema = FunctionSchema(
|
|
name=VISION_TOOL_NAME,
|
|
description=(
|
|
"获取用户当前摄像头画面。当用户询问当前画面、看到了什么、"
|
|
"人/物品/环境状态或需要视觉判断时调用。"
|
|
),
|
|
properties={
|
|
"question": {
|
|
"type": "string",
|
|
"description": "用户关于当前视频画面的具体问题。",
|
|
}
|
|
},
|
|
required=["question"],
|
|
)
|
|
knowledge_schema = FunctionSchema(
|
|
name=KNOWLEDGE_TOOL_NAME,
|
|
description=_knowledge_tool_description(cfg),
|
|
properties={
|
|
"query": {"type": "string", "description": "用于检索的完整问题或关键词"}
|
|
},
|
|
required=["query"],
|
|
)
|
|
|
|
async def search_bound_knowledge(params: FunctionCallParams):
|
|
query = str(params.arguments.get("query") or "").strip()
|
|
if not query or not cfg.knowledge_base_id:
|
|
await params.result_callback({"status": "error", "message": "检索问题为空或未绑定知识库"})
|
|
return
|
|
try:
|
|
async with SessionLocal() as session:
|
|
results = await search_knowledge(
|
|
session,
|
|
cfg.knowledge_base_id,
|
|
query,
|
|
top_k=knowledge_top_n,
|
|
score_threshold=knowledge_score_threshold,
|
|
)
|
|
await params.result_callback({"status": "ok", "results": results})
|
|
except Exception as exc:
|
|
logger.exception(f"知识库检索失败: {exc}")
|
|
await params.result_callback({"status": "error", "message": "知识库检索暂时不可用"})
|
|
|
|
async def fetch_user_image(params: FunctionCallParams):
|
|
question = str(params.arguments.get("question") or "请描述当前画面。")
|
|
user_id = vision_state.get("client_id")
|
|
if not user_id:
|
|
await params.result_callback(
|
|
{
|
|
"status": "no_video_client",
|
|
"message": "当前还没有可用的摄像头视频流。",
|
|
}
|
|
)
|
|
return
|
|
|
|
request = UserImageRequestFrame(
|
|
user_id=user_id,
|
|
text=question,
|
|
append_to_context=vision_native_mode,
|
|
function_name=params.function_name,
|
|
tool_call_id=params.tool_call_id,
|
|
result_callback=params.result_callback if vision_native_mode else None,
|
|
)
|
|
if vision_native_mode:
|
|
logger.debug(
|
|
f"请求当前视频帧进入主 LLM 上下文: user_id={user_id}, question={question}"
|
|
)
|
|
await params.llm.push_frame(request, FrameDirection.UPSTREAM)
|
|
return
|
|
|
|
logger.debug(
|
|
f"请求当前视频帧给单独视觉模型分析: user_id={user_id}, question={question}"
|
|
)
|
|
try:
|
|
frame = await vision_capture.request_image(params.llm, request)
|
|
observation = await _analyze_image_with_vision_model(cfg, frame, question)
|
|
except asyncio.TimeoutError:
|
|
await params.result_callback(
|
|
{
|
|
"status": "timeout",
|
|
"message": "等待摄像头视频帧超时。",
|
|
}
|
|
)
|
|
return
|
|
except Exception as e:
|
|
logger.exception(f"视觉模型分析失败: {e}")
|
|
await params.result_callback(
|
|
{
|
|
"status": "error",
|
|
"message": f"视觉理解失败: {type(e).__name__}",
|
|
}
|
|
)
|
|
return
|
|
|
|
await params.result_callback(
|
|
{
|
|
"status": "ok",
|
|
"question": question,
|
|
"observation": observation or "视觉模型没有返回有效观察结果。",
|
|
}
|
|
)
|
|
|
|
flow_global_functions = []
|
|
if cfg.type == "workflow" and vision_enabled:
|
|
async def flow_fetch_user_image(args, _flow_manager):
|
|
question = str((args or {}).get("question") or "请描述当前画面。")
|
|
user_id = vision_state.get("client_id")
|
|
if not user_id:
|
|
return {
|
|
"status": "no_video_client",
|
|
"message": "当前还没有可用的摄像头视频流。",
|
|
}
|
|
request = UserImageRequestFrame(
|
|
user_id=user_id,
|
|
text=question,
|
|
append_to_context=False,
|
|
function_name=VISION_TOOL_NAME,
|
|
)
|
|
try:
|
|
frame = await vision_capture.request_image(llm, request)
|
|
observation = await _analyze_image_with_vision_model(cfg, frame, question)
|
|
return {
|
|
"status": "ok",
|
|
"question": question,
|
|
"observation": observation or "视觉模型没有返回有效观察结果。",
|
|
}
|
|
except asyncio.TimeoutError:
|
|
return {"status": "timeout", "message": "等待摄像头视频帧超时。"}
|
|
except Exception as exc: # noqa: BLE001 - return tool errors to the LLM
|
|
logger.warning(f"Workflow 视觉理解失败:{exc}")
|
|
return {"status": "error", "message": "视觉理解暂时不可用。"}
|
|
|
|
flow_global_functions.append(
|
|
FlowsFunctionSchema(
|
|
name=VISION_TOOL_NAME,
|
|
description=vision_schema.description,
|
|
properties=vision_schema.properties,
|
|
required=vision_schema.required,
|
|
handler=flow_fetch_user_image,
|
|
cancel_on_interruption=True,
|
|
)
|
|
)
|
|
|
|
if vision_enabled and cfg.type != "workflow":
|
|
llm.register_function(VISION_TOOL_NAME, fetch_user_image)
|
|
if cfg.knowledge_base_id and knowledge_mode == "on_demand":
|
|
llm.register_function(KNOWLEDGE_TOOL_NAME, search_bound_knowledge)
|
|
|
|
def set_visible_tools(schemas: list[FunctionSchema] | None = None) -> None:
|
|
tools = list(schemas or [])
|
|
if vision_enabled and cfg.type != "workflow":
|
|
tools.append(vision_schema)
|
|
if cfg.knowledge_base_id and knowledge_mode == "on_demand":
|
|
tools.append(knowledge_schema)
|
|
if tools:
|
|
context.set_tools(ToolsSchema(standard_tools=tools))
|
|
else:
|
|
context.set_tools()
|
|
|
|
recorder = await ConversationRecorder.start(
|
|
assistant_id=assistant_id,
|
|
assistant_name=cfg.name,
|
|
channel=channel,
|
|
runtime_mode=cfg.runtimeMode,
|
|
session_id=cfg.conversation_id or None,
|
|
)
|
|
pipeline = Pipeline(
|
|
[
|
|
transport.input(),
|
|
vision_capture,
|
|
text_input,
|
|
stt_processor,
|
|
user_aggregator,
|
|
user_turn_router,
|
|
knowledge_retrieval,
|
|
llm,
|
|
# Aggregate the streamed LLM text before TTS. On interruption,
|
|
# Pipecat commits the generated prefix immediately instead of
|
|
# waiting for a TTS provider to emit spoken-text/timestamp frames.
|
|
assistant_aggregator,
|
|
tts_processor,
|
|
EndCallAfterSpeechProcessor(call_end),
|
|
ConversationHistoryProcessor(recorder),
|
|
transport.output(),
|
|
]
|
|
)
|
|
|
|
worker = PipelineWorker(
|
|
pipeline,
|
|
params=PipelineParams(
|
|
enable_metrics=False,
|
|
),
|
|
enable_rtvi=False,
|
|
)
|
|
worker_holder["worker"] = worker
|
|
service_controller = WorkflowServiceController(
|
|
worker=worker,
|
|
llm_services=llm_services,
|
|
voice_services={"asr": stt_services, "tts": tts_services},
|
|
current_services={
|
|
"llm": current_llm_service,
|
|
**current_voice_services,
|
|
},
|
|
)
|
|
current_enable_interrupt = cfg.enableInterrupt
|
|
current_turn_config = dict(cfg.turnConfig)
|
|
|
|
async def apply_workflow_turn_config(
|
|
enable_interrupt: bool,
|
|
turn_config: dict[str, Any],
|
|
) -> None:
|
|
"""Apply one Agent's interaction policy before its next user turn."""
|
|
nonlocal current_enable_interrupt, current_turn_config
|
|
normalized = dict(turn_config or {})
|
|
if (
|
|
current_enable_interrupt == enable_interrupt
|
|
and current_turn_config == normalized
|
|
):
|
|
return
|
|
await user_aggregator.apply_turn_strategies(
|
|
normalized,
|
|
enable_interruptions=enable_interrupt,
|
|
)
|
|
await worker.queue_frame(
|
|
VADParamsUpdateFrame(params=create_vad_params(normalized))
|
|
)
|
|
current_enable_interrupt = enable_interrupt
|
|
current_turn_config = normalized
|
|
|
|
|
|
def set_system_prompt(text: str) -> None:
|
|
"""替换上下文里的系统提示(节点切换时整体替换,而非追加)。"""
|
|
messages = context.get_messages()
|
|
content = with_vision_hint(text)
|
|
if messages and messages[0].get("role") == "system":
|
|
messages[0] = {"role": "system", "content": content}
|
|
else:
|
|
messages.insert(0, {"role": "system", "content": content})
|
|
|
|
set_visible_tools([])
|
|
await brain.setup(
|
|
cfg,
|
|
BrainRuntime(
|
|
context=context,
|
|
llm=llm,
|
|
queue_frame=worker.queue_frame,
|
|
set_system_prompt=set_system_prompt,
|
|
set_tools=set_visible_tools,
|
|
call_end=call_end,
|
|
worker=worker,
|
|
context_aggregator=WorkflowAggregatorPair(
|
|
user_aggregator,
|
|
assistant_aggregator,
|
|
),
|
|
transport=transport,
|
|
switch_services=service_controller.switch,
|
|
set_knowledge_scope=knowledge_retrieval.set_scope,
|
|
set_input_enabled=lambda enabled: input_state.__setitem__("enabled", enabled),
|
|
apply_turn_config=apply_workflow_turn_config,
|
|
flow_global_functions=flow_global_functions,
|
|
),
|
|
)
|
|
|
|
bind_cascade_pipeline_events(
|
|
transport=transport,
|
|
worker=worker,
|
|
brain=brain,
|
|
context=context,
|
|
text_input=text_input,
|
|
user_aggregator=user_aggregator,
|
|
assistant_aggregator=assistant_aggregator,
|
|
greeting=greeting,
|
|
vision_enabled=vision_enabled,
|
|
vision_state=vision_state,
|
|
)
|
|
runner = WorkerRunner(handle_sigint=False)
|
|
run_status = "completed"
|
|
try:
|
|
await runner.add_workers(worker)
|
|
await runner.run()
|
|
except Exception:
|
|
run_status = "failed"
|
|
raise
|
|
finally:
|
|
if recorder:
|
|
await recorder.finish(status=run_status)
|
|
logger.info("管线已结束")
|
|
|
|
|
|
async def run_realtime_pipeline(
|
|
transport,
|
|
cfg: AssistantConfig,
|
|
*,
|
|
brain: Brain,
|
|
assistant_id: str | None = None,
|
|
channel: str = "webrtc",
|
|
) -> None:
|
|
"""Run a speech-to-speech model that owns ASR, reasoning, and synthesis."""
|
|
realtime = create_realtime_service(
|
|
cfg,
|
|
instructions=brain.system_prompt(cfg),
|
|
)
|
|
text_input = RealtimeTextInputProcessor()
|
|
dynamic_variables = RealtimeDynamicVariableProcessor(brain, cfg, realtime)
|
|
greeting = await brain.greeting(cfg)
|
|
|
|
recorder = await ConversationRecorder.start(
|
|
assistant_id=assistant_id,
|
|
assistant_name=cfg.name,
|
|
channel=channel,
|
|
runtime_mode=cfg.runtimeMode,
|
|
session_id=cfg.conversation_id or None,
|
|
)
|
|
pipeline = Pipeline(
|
|
[
|
|
transport.input(),
|
|
text_input,
|
|
realtime,
|
|
dynamic_variables,
|
|
ConversationHistoryProcessor(recorder),
|
|
transport.output(),
|
|
]
|
|
)
|
|
worker = PipelineWorker(
|
|
pipeline,
|
|
params=PipelineParams(
|
|
enable_metrics=False,
|
|
audio_in_sample_rate=int(
|
|
cfg.realtime_values.get("inputSampleRate") or 24000
|
|
),
|
|
audio_out_sample_rate=int(
|
|
cfg.realtime_values.get("outputSampleRate") or 24000
|
|
),
|
|
),
|
|
enable_rtvi=False,
|
|
)
|
|
|
|
bind_realtime_pipeline_events(
|
|
transport=transport,
|
|
worker=worker,
|
|
realtime=realtime,
|
|
text_input=text_input,
|
|
greeting=greeting,
|
|
)
|
|
runner = WorkerRunner(handle_sigint=False)
|
|
run_status = "completed"
|
|
try:
|
|
await runner.add_workers(worker)
|
|
await runner.run()
|
|
except Exception:
|
|
run_status = "failed"
|
|
raise
|
|
finally:
|
|
if recorder:
|
|
await recorder.finish(status=run_status)
|
|
logger.info("Realtime 管线已结束")
|