"""管线核心:给定一个 transport + 配置,跑完整的语音闭环。 关键设计:**transport 由调用方传入**,管线本身不关心是 WebRTC 还是 WS。 这就是"同时支持多种输出"的落点——加输出方式不用动这里。 对话编排交给 Brain;本文件只保留共享媒体管线、输入输出和通话生命周期。 """ import asyncio import base64 from io import BytesIO from typing import Any from loguru import logger from models import AssistantConfig from openai import AsyncOpenAI from PIL import Image from services.brains import Brain, BrainRuntime, build_brain from services.conversation_history import ConversationRecorder from services.pipecat.call_lifecycle import ( CallEndCoordinator, EndCallAfterSpeechProcessor, ) from services.pipecat.service_factory import ( config_with_resource, create_realtime_service, create_stt, create_tts, ) from db.session import SessionLocal from services.knowledge import search as search_knowledge from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.flows import FlowsFunctionSchema from pipecat.frames.frames import ( EndFrame, OutputTransportMessageUrgentFrame, UserImageRawFrame, UserImageRequestFrame, VADParamsUpdateFrame, ) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.worker import PipelineParams, PipelineWorker from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import ( LLMUserAggregatorParams, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.llm_service import FunctionCallParams from pipecat.turns.user_mute.function_call_user_mute_strategy import ( FunctionCallUserMuteStrategy, ) from services.pipecat.turn_config import ( ConfigurableLLMUserAggregator, create_user_turn_strategies, create_vad_analyzer, create_vad_params, ) from services.pipecat.processors import ( KNOWLEDGE_CONTEXT_MARKER, CallEndingUserMuteStrategy, ConversationHistoryProcessor, KnowledgeRetrievalProcessor, PassthroughLLMAssistantAggregator, RealtimeDynamicVariableProcessor, RealtimeTextInputProcessor, TextInputProcessor, UserTurnRoutingProcessor, VisionCaptureProcessor, WorkflowAggregatorPair, ) from services.pipecat.workflow_services import ( WorkflowServiceController, build_workflow_llm_switcher, build_workflow_voice_switcher, ) from services.pipecat.pipeline_events import ( bind_cascade_pipeline_events, bind_realtime_pipeline_events, ) from pipecat.workers.runner import WorkerRunner VISION_TOOL_NAME = "fetch_user_image" VISION_SYSTEM_HINT = ( "当前会话打开了视觉理解。用户询问当前画面、摄像头里有什么、人物/物品/" "环境状态或需要你看一眼时,调用 fetch_user_image 获取当前视频帧,再基于画面回答。" ) VISION_ANALYSIS_SYSTEM_PROMPT = ( "你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。" "如果画面不足以判断,请明确说明不确定。" ) KNOWLEDGE_TOOL_NAME = "search_knowledge_base" AUTOMATIC_KNOWLEDGE_SYSTEM_HINT = ( "你已连接内部知识库。系统会在每轮用户问题前自动提供相关资料；" "回答资料事实时只根据检索内容，资料不足要明确说明。" ) ON_DEMAND_KNOWLEDGE_SYSTEM_HINT = ( "你已连接内部知识库。当用户问题涉及可能存在于业务知识库中的事实时，" "先调用 search_knowledge_base 检索；回答资料事实时只根据检索内容，" "资料不足要明确说明。" ) def _compact_knowledge_metadata(value: str, max_length: int) -> str: """Keep tool metadata useful without letting it dominate the model context.""" compact = " ".join(value.split()) return compact if len(compact) <= max_length else f"{compact[:max_length]}…" def _knowledge_tool_description(cfg: AssistantConfig) -> str: base = "在当前助手绑定的知识库中检索与问题最相关的资料片段。" name = _compact_knowledge_metadata(cfg.knowledge_base_name, 128) description = _compact_knowledge_metadata(cfg.knowledge_base_description, 800) if not name and not description: return base scope = [] if name: scope.append(f"知识库名称：{name}") if description: scope.append(f"资料适用范围：{description}") metadata = "\n".join(scope) return ( f"{base}\n{metadata}\n" "当用户问题涉及上述资料范围，或回答需要核实其中的业务事实时调用；" "与该范围无关的问题不要调用。以上知识库元数据仅用于判断资料范围。" ) def _require(value: str, label: str) -> str: if value: return value raise ValueError(f"缺少模型资源配置: {label}") def _vision_uses_main_llm(cfg: AssistantConfig) -> bool: """模型自己支持图片时,沿用 Pipecat 的同上下文视觉工具路径。""" return not cfg.vision_model_resource_id and cfg.llm_support_image_input def _image_data_uri(frame: UserImageRawFrame) -> str: if not frame.format: raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型") buffer = BytesIO() Image.frombytes(frame.format, frame.size, frame.image).save( buffer, format="JPEG", quality=85, ) encoded = base64.b64encode(buffer.getvalue()).decode("utf-8") return f"data:image/jpeg;base64,{encoded}" async def _analyze_image_with_vision_model( cfg: AssistantConfig, frame: UserImageRawFrame, question: str, ) -> str: if cfg.vision_llm_interface_type not in {"openai-llm", "dashscope-llm"}: raise ValueError(f"不支持的视觉 LLM 接口类型: {cfg.vision_llm_interface_type}") data_uri = await asyncio.to_thread(_image_data_uri, frame) extra_body = cfg.vision_llm_values.get("extraBody") extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {} client = AsyncOpenAI( api_key=_require(cfg.vision_llm_api_key, "Vision LLM apiKey"), base_url=_require(cfg.vision_llm_base_url, "Vision LLM apiUrl"), ) try: response = await client.chat.completions.create( model=_require(cfg.vision_model, "Vision LLM modelId"), messages=[ {"role": "system", "content": VISION_ANALYSIS_SYSTEM_PROMPT}, { "role": "user", "content": [ {"type": "text", "text": question}, {"type": "image_url", "image_url": {"url": data_uri}}, ], }, ], **extra, ) finally: await client.close() content = response.choices[0].message.content if response.choices else "" if isinstance(content, str): return content.strip() return str(content or "").strip() async def run_pipeline( transport, cfg: AssistantConfig, *, vision_enabled: bool = False, assistant_id: str | None = None, channel: str = "webrtc", ) -> None: """在给定 transport 上构建并运行管线,直到连接结束。 Args: transport: 任意 pipecat transport(WebRTC / WS / 电话…), 只要有 .input() / .output() / event_handler 即可。 cfg: 助手配置(随请求内联传入)。 """ logger.info( f"启动管线: assistant={cfg.name} type={cfg.type} " f"mode={cfg.runtimeMode} vision={vision_enabled}" ) # 大脑:按类型决定 LLM 槽/开场白/上下文归属。每通电话一个实例(可持会话状态)。 brain = build_brain(cfg) if ( cfg.runtimeMode == "realtime" and "realtime" not in brain.spec.supported_runtime_modes ): raise ValueError(f"类型 {cfg.type} 不支持 realtime 运行模式") if cfg.runtimeMode == "realtime": if vision_enabled: logger.warning("Realtime 模式暂未接入视频帧工具,本次仅启用语音通话") await run_realtime_pipeline( transport, cfg, brain=brain, assistant_id=assistant_id, channel=channel, ) return graph_settings = cfg.graph.get("settings") or {} default_llm_resource = cfg.workflow_model_resources.get( str(graph_settings.get("defaultLlmResourceId") or "") ) default_asr_resource = cfg.workflow_model_resources.get( str(graph_settings.get("defaultAsrResourceId") or "") ) default_tts_resource = cfg.workflow_model_resources.get( str(graph_settings.get("defaultTtsResourceId") or "") ) stt = create_stt( config_with_resource(cfg, default_asr_resource) if cfg.type == "workflow" and default_asr_resource else cfg ) tts = create_tts( config_with_resource(cfg, default_tts_resource) if cfg.type == "workflow" and default_tts_resource else cfg ) stt_processor = stt tts_processor = tts stt_services: dict[str, FrameProcessor] = {} tts_services: dict[str, FrameProcessor] = {} current_voice_services: dict[str, FrameProcessor] = {"asr": stt, "tts": tts} if cfg.type == "workflow": stt_processor, stt_services, current_voice_services["asr"] = ( build_workflow_voice_switcher(cfg, "ASR", stt) ) tts_processor, tts_services, current_voice_services["tts"] = ( build_workflow_voice_switcher(cfg, "TTS", tts) ) greeting = await brain.greeting(cfg) system_content = brain.system_prompt(cfg) worker_holder: dict = {} async def queue_call_end(reason: str) -> None: worker = worker_holder.get("worker") if worker is None: return logger.info(f"结束通话: reason={reason}") await worker.queue_frame( OutputTransportMessageUrgentFrame( message={"type": "call-ended", "reason": reason} ) ) await worker.queue_frame(EndFrame()) call_end = CallEndCoordinator(queue_call_end) knowledge_config = cfg.knowledge_retrieval_config knowledge_mode = str(knowledge_config.get("mode", "automatic")) knowledge_top_n = int( knowledge_config.get("top_n", knowledge_config.get("topN", 5)) ) knowledge_score_threshold = float( knowledge_config.get( "score_threshold", knowledge_config.get("scoreThreshold", 0.0) ) ) automatic_knowledge_id = ( cfg.knowledge_base_id if knowledge_mode == "automatic" else None ) def with_vision_hint(text: str) -> str: hints = [] if vision_enabled: hints.append(VISION_SYSTEM_HINT) if cfg.knowledge_base_id: hints.append( AUTOMATIC_KNOWLEDGE_SYSTEM_HINT if knowledge_mode == "automatic" else ON_DEMAND_KNOWLEDGE_SYSTEM_HINT ) return "\n\n".join(part for part in [text, *hints] if part) context = LLMContext( messages=( [] if cfg.type == "workflow" else [{"role": "system", "content": with_vision_hint(system_content)}] ) ) input_state = {"enabled": True} # LLM 槽由大脑提供:本地模型或 Dify/FastGPT 外部托管适配器。 llm = brain.build_llm( config_with_resource(cfg, default_llm_resource) if cfg.type == "workflow" and default_llm_resource else cfg, context, ) llm_services: dict[str, FrameProcessor] = {} current_llm_service = llm if cfg.type == "workflow": llm, llm_services, current_llm_service = build_workflow_llm_switcher(cfg, llm) user_aggregator = ConfigurableLLMUserAggregator( context, params=LLMUserAggregatorParams( vad_analyzer=create_vad_analyzer(cfg.turnConfig), user_mute_strategies=[ FunctionCallUserMuteStrategy(), CallEndingUserMuteStrategy( lambda: call_end.ending or not input_state["enabled"] ), ], user_turn_strategies=create_user_turn_strategies( cfg.turnConfig, enable_interruptions=cfg.enableInterrupt, ), ), ) user_turn_router = UserTurnRoutingProcessor(brain) assistant_aggregator = PassthroughLLMAssistantAggregator(context) text_input = TextInputProcessor( should_ignore_input=lambda: call_end.ending or not input_state["enabled"] ) vision_capture = VisionCaptureProcessor() knowledge_retrieval = KnowledgeRetrievalProcessor( automatic_knowledge_id, top_n=knowledge_top_n, score_threshold=knowledge_score_threshold, ) vision_native_mode = vision_enabled and _vision_uses_main_llm(cfg) vision_state: dict[str, str | None] = {"client_id": None} vision_schema = FunctionSchema( name=VISION_TOOL_NAME, description=( "获取用户当前摄像头画面。当用户询问当前画面、看到了什么、" "人/物品/环境状态或需要视觉判断时调用。" ), properties={ "question": { "type": "string", "description": "用户关于当前视频画面的具体问题。", } }, required=["question"], ) knowledge_schema = FunctionSchema( name=KNOWLEDGE_TOOL_NAME, description=_knowledge_tool_description(cfg), properties={ "query": {"type": "string", "description": "用于检索的完整问题或关键词"} }, required=["query"], ) async def search_bound_knowledge(params: FunctionCallParams): query = str(params.arguments.get("query") or "").strip() if not query or not cfg.knowledge_base_id: await params.result_callback({"status": "error", "message": "检索问题为空或未绑定知识库"}) return try: async with SessionLocal() as session: results = await search_knowledge( session, cfg.knowledge_base_id, query, top_k=knowledge_top_n, score_threshold=knowledge_score_threshold, ) await params.result_callback({"status": "ok", "results": results}) except Exception as exc: logger.exception(f"知识库检索失败: {exc}") await params.result_callback({"status": "error", "message": "知识库检索暂时不可用"}) async def fetch_user_image(params: FunctionCallParams): question = str(params.arguments.get("question") or "请描述当前画面。") user_id = vision_state.get("client_id") if not user_id: await params.result_callback( { "status": "no_video_client", "message": "当前还没有可用的摄像头视频流。", } ) return request = UserImageRequestFrame( user_id=user_id, text=question, append_to_context=vision_native_mode, function_name=params.function_name, tool_call_id=params.tool_call_id, result_callback=params.result_callback if vision_native_mode else None, ) if vision_native_mode: logger.debug( f"请求当前视频帧进入主 LLM 上下文: user_id={user_id}, question={question}" ) await params.llm.push_frame(request, FrameDirection.UPSTREAM) return logger.debug( f"请求当前视频帧给单独视觉模型分析: user_id={user_id}, question={question}" ) try: frame = await vision_capture.request_image(params.llm, request) observation = await _analyze_image_with_vision_model(cfg, frame, question) except asyncio.TimeoutError: await params.result_callback( { "status": "timeout", "message": "等待摄像头视频帧超时。", } ) return except Exception as e: logger.exception(f"视觉模型分析失败: {e}") await params.result_callback( { "status": "error", "message": f"视觉理解失败: {type(e).__name__}", } ) return await params.result_callback( { "status": "ok", "question": question, "observation": observation or "视觉模型没有返回有效观察结果。", } ) flow_global_functions = [] if cfg.type == "workflow" and vision_enabled: async def flow_fetch_user_image(args, _flow_manager): question = str((args or {}).get("question") or "请描述当前画面。") user_id = vision_state.get("client_id") if not user_id: return { "status": "no_video_client", "message": "当前还没有可用的摄像头视频流。", } request = UserImageRequestFrame( user_id=user_id, text=question, append_to_context=False, function_name=VISION_TOOL_NAME, ) try: frame = await vision_capture.request_image(llm, request) observation = await _analyze_image_with_vision_model(cfg, frame, question) return { "status": "ok", "question": question, "observation": observation or "视觉模型没有返回有效观察结果。", } except asyncio.TimeoutError: return {"status": "timeout", "message": "等待摄像头视频帧超时。"} except Exception as exc: # noqa: BLE001 - return tool errors to the LLM logger.warning(f"Workflow 视觉理解失败:{exc}") return {"status": "error", "message": "视觉理解暂时不可用。"} flow_global_functions.append( FlowsFunctionSchema( name=VISION_TOOL_NAME, description=vision_schema.description, properties=vision_schema.properties, required=vision_schema.required, handler=flow_fetch_user_image, cancel_on_interruption=True, ) ) if vision_enabled and cfg.type != "workflow": llm.register_function(VISION_TOOL_NAME, fetch_user_image) if cfg.knowledge_base_id and knowledge_mode == "on_demand": llm.register_function(KNOWLEDGE_TOOL_NAME, search_bound_knowledge) def set_visible_tools(schemas: list[FunctionSchema] | None = None) -> None: tools = list(schemas or []) if vision_enabled and cfg.type != "workflow": tools.append(vision_schema) if cfg.knowledge_base_id and knowledge_mode == "on_demand": tools.append(knowledge_schema) if tools: context.set_tools(ToolsSchema(standard_tools=tools)) else: context.set_tools() recorder = await ConversationRecorder.start( assistant_id=assistant_id, assistant_name=cfg.name, channel=channel, runtime_mode=cfg.runtimeMode, session_id=cfg.conversation_id or None, ) pipeline = Pipeline( [ transport.input(), vision_capture, text_input, stt_processor, user_aggregator, user_turn_router, knowledge_retrieval, llm, # Aggregate the streamed LLM text before TTS. On interruption, # Pipecat commits the generated prefix immediately instead of # waiting for a TTS provider to emit spoken-text/timestamp frames. assistant_aggregator, tts_processor, EndCallAfterSpeechProcessor(call_end), ConversationHistoryProcessor(recorder), transport.output(), ] ) worker = PipelineWorker( pipeline, params=PipelineParams( enable_metrics=False, ), enable_rtvi=False, ) worker_holder["worker"] = worker service_controller = WorkflowServiceController( worker=worker, llm_services=llm_services, voice_services={"asr": stt_services, "tts": tts_services}, current_services={ "llm": current_llm_service, **current_voice_services, }, ) current_enable_interrupt = cfg.enableInterrupt current_turn_config = dict(cfg.turnConfig) async def apply_workflow_turn_config( enable_interrupt: bool, turn_config: dict[str, Any], ) -> None: """Apply one Agent's interaction policy before its next user turn.""" nonlocal current_enable_interrupt, current_turn_config normalized = dict(turn_config or {}) if ( current_enable_interrupt == enable_interrupt and current_turn_config == normalized ): return await user_aggregator.apply_turn_strategies( normalized, enable_interruptions=enable_interrupt, ) await worker.queue_frame( VADParamsUpdateFrame(params=create_vad_params(normalized)) ) current_enable_interrupt = enable_interrupt current_turn_config = normalized def set_system_prompt(text: str) -> None: """替换上下文里的系统提示(节点切换时整体替换,而非追加)。""" messages = context.get_messages() content = with_vision_hint(text) if messages and messages[0].get("role") == "system": messages[0] = {"role": "system", "content": content} else: messages.insert(0, {"role": "system", "content": content}) set_visible_tools([]) await brain.setup( cfg, BrainRuntime( context=context, llm=llm, queue_frame=worker.queue_frame, set_system_prompt=set_system_prompt, set_tools=set_visible_tools, call_end=call_end, worker=worker, context_aggregator=WorkflowAggregatorPair( user_aggregator, assistant_aggregator, ), transport=transport, switch_services=service_controller.switch, set_knowledge_scope=knowledge_retrieval.set_scope, set_input_enabled=lambda enabled: input_state.__setitem__("enabled", enabled), apply_turn_config=apply_workflow_turn_config, flow_global_functions=flow_global_functions, ), ) bind_cascade_pipeline_events( transport=transport, worker=worker, brain=brain, context=context, text_input=text_input, user_aggregator=user_aggregator, assistant_aggregator=assistant_aggregator, greeting=greeting, vision_enabled=vision_enabled, vision_state=vision_state, ) runner = WorkerRunner(handle_sigint=False) run_status = "completed" try: await runner.add_workers(worker) await runner.run() except Exception: run_status = "failed" raise finally: if recorder: await recorder.finish(status=run_status) logger.info("管线已结束") async def run_realtime_pipeline( transport, cfg: AssistantConfig, *, brain: Brain, assistant_id: str | None = None, channel: str = "webrtc", ) -> None: """Run a speech-to-speech model that owns ASR, reasoning, and synthesis.""" realtime = create_realtime_service( cfg, instructions=brain.system_prompt(cfg), ) text_input = RealtimeTextInputProcessor() dynamic_variables = RealtimeDynamicVariableProcessor(brain, cfg, realtime) greeting = await brain.greeting(cfg) recorder = await ConversationRecorder.start( assistant_id=assistant_id, assistant_name=cfg.name, channel=channel, runtime_mode=cfg.runtimeMode, session_id=cfg.conversation_id or None, ) pipeline = Pipeline( [ transport.input(), text_input, realtime, dynamic_variables, ConversationHistoryProcessor(recorder), transport.output(), ] ) worker = PipelineWorker( pipeline, params=PipelineParams( enable_metrics=False, audio_in_sample_rate=int( cfg.realtime_values.get("inputSampleRate") or 24000 ), audio_out_sample_rate=int( cfg.realtime_values.get("outputSampleRate") or 24000 ), ), enable_rtvi=False, ) bind_realtime_pipeline_events( transport=transport, worker=worker, realtime=realtime, text_input=text_input, greeting=greeting, ) runner = WorkerRunner(handle_sigint=False) run_status = "completed" try: await runner.add_workers(worker) await runner.run() except Exception: run_status = "failed" raise finally: if recorder: await recorder.finish(status=run_status) logger.info("Realtime 管线已结束")