"""管线核心:给定一个 transport + 配置,跑完整的语音闭环。 关键设计:**transport 由调用方传入**,管线本身不关心是 WebRTC 还是 WS。 这就是"同时支持多种输出"的落点——加输出方式不用动这里。 对话编排交给 Brain;本文件只保留共享媒体管线、输入输出和通话生命周期。 """ import asyncio import base64 from collections.abc import Callable from io import BytesIO from typing import Any from uuid import uuid4 from loguru import logger from models import AssistantConfig from openai import AsyncOpenAI from PIL import Image from services.brains import Brain, BrainRuntime, build_brain from services.conversation_history import ConversationRecorder from services.pipecat.call_lifecycle import ( CallEndCoordinator, EndCallAfterSpeechProcessor, ) from services.pipecat.service_factory import ( config_with_resource, create_llm, create_realtime_service, create_stt, create_tts, ) from db.session import SessionLocal from services.knowledge import search as search_knowledge from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.flows import FlowsFunctionSchema from pipecat.frames.frames import ( EndFrame, InputTransportMessageFrame, InterruptionFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMContextFrame, LLMTextFrame, ManuallySwitchServiceFrame, LLMMessagesAppendFrame, OutputTransportMessageUrgentFrame, TextFrame, TTSSpeakFrame, UserImageRawFrame, UserImageRequestFrame, VADParamsUpdateFrame, ) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.llm_switcher import LLMSwitcher from pipecat.pipeline.service_switcher import ServiceSwitcher from pipecat.pipeline.worker import PipelineParams, PipelineWorker from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import ( LLMAssistantAggregator, LLMUserAggregatorParams, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.utils import ( get_transport_client_id, maybe_capture_participant_camera, ) from pipecat.services.llm_service import FunctionCallParams from pipecat.turns.user_mute.base_user_mute_strategy import BaseUserMuteStrategy from pipecat.turns.user_mute.function_call_user_mute_strategy import ( FunctionCallUserMuteStrategy, ) from services.pipecat.turn_config import ( ConfigurableLLMUserAggregator, create_user_turn_strategies, create_vad_analyzer, create_vad_params, ) from pipecat.utils.time import time_now_iso8601 from pipecat.workers.runner import WorkerRunner VISION_TOOL_NAME = "fetch_user_image" VISION_SYSTEM_HINT = ( "当前会话打开了视觉理解。用户询问当前画面、摄像头里有什么、人物/物品/" "环境状态或需要你看一眼时,调用 fetch_user_image 获取当前视频帧,再基于画面回答。" ) VISION_ANALYSIS_SYSTEM_PROMPT = ( "你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。" "如果画面不足以判断,请明确说明不确定。" ) KNOWLEDGE_TOOL_NAME = "search_knowledge_base" AUTOMATIC_KNOWLEDGE_SYSTEM_HINT = ( "你已连接内部知识库。系统会在每轮用户问题前自动提供相关资料；" "回答资料事实时只根据检索内容，资料不足要明确说明。" ) ON_DEMAND_KNOWLEDGE_SYSTEM_HINT = ( "你已连接内部知识库。当用户问题涉及可能存在于业务知识库中的事实时，" "先调用 search_knowledge_base 检索；回答资料事实时只根据检索内容，" "资料不足要明确说明。" ) KNOWLEDGE_CONTEXT_MARKER = "" def _compact_knowledge_metadata(value: str, max_length: int) -> str: """Keep tool metadata useful without letting it dominate the model context.""" compact = " ".join(value.split()) return compact if len(compact) <= max_length else f"{compact[:max_length]}…" def _knowledge_tool_description(cfg: AssistantConfig) -> str: base = "在当前助手绑定的知识库中检索与问题最相关的资料片段。" name = _compact_knowledge_metadata(cfg.knowledge_base_name, 128) description = _compact_knowledge_metadata(cfg.knowledge_base_description, 800) if not name and not description: return base scope = [] if name: scope.append(f"知识库名称：{name}") if description: scope.append(f"资料适用范围：{description}") metadata = "\n".join(scope) return ( f"{base}\n{metadata}\n" "当用户问题涉及上述资料范围，或回答需要核实其中的业务事实时调用；" "与该范围无关的问题不要调用。以上知识库元数据仅用于判断资料范围。" ) def _require(value: str, label: str) -> str: if value: return value raise ValueError(f"缺少模型资源配置: {label}") def _vision_uses_main_llm(cfg: AssistantConfig) -> bool: """模型自己支持图片时,沿用 Pipecat 的同上下文视觉工具路径。""" return not cfg.vision_model_resource_id and cfg.llm_support_image_input def _image_data_uri(frame: UserImageRawFrame) -> str: if not frame.format: raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型") buffer = BytesIO() Image.frombytes(frame.format, frame.size, frame.image).save( buffer, format="JPEG", quality=85, ) encoded = base64.b64encode(buffer.getvalue()).decode("utf-8") return f"data:image/jpeg;base64,{encoded}" async def _analyze_image_with_vision_model( cfg: AssistantConfig, frame: UserImageRawFrame, question: str, ) -> str: if cfg.vision_llm_interface_type not in {"openai-llm", "dashscope-llm"}: raise ValueError(f"不支持的视觉 LLM 接口类型: {cfg.vision_llm_interface_type}") data_uri = await asyncio.to_thread(_image_data_uri, frame) extra_body = cfg.vision_llm_values.get("extraBody") extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {} client = AsyncOpenAI( api_key=_require(cfg.vision_llm_api_key, "Vision LLM apiKey"), base_url=_require(cfg.vision_llm_base_url, "Vision LLM apiUrl"), ) try: response = await client.chat.completions.create( model=_require(cfg.vision_model, "Vision LLM modelId"), messages=[ {"role": "system", "content": VISION_ANALYSIS_SYSTEM_PROMPT}, { "role": "user", "content": [ {"type": "text", "text": question}, {"type": "image_url", "image_url": {"url": data_uri}}, ], }, ], **extra, ) finally: await client.close() content = response.choices[0].message.content if response.choices else "" if isinstance(content, str): return content.strip() return str(content or "").strip() def _text_input(message) -> tuple[str, bool] | None: """解析现有 user-text 与 RTVI send-text 两种前端文字消息。""" if not isinstance(message, dict): return None if message.get("type") == "user-text": text = str(message.get("text") or "").strip() return (text, True) if text else None if message.get("type") == "send-text": data = message.get("data") if not isinstance(data, dict): return None text = str(data.get("content") or "").strip() options = data.get("options") run_immediately = not isinstance(options, dict) or options.get( "run_immediately", True ) return (text, bool(run_immediately)) if text else None return None class TextInputProcessor(FrameProcessor): """把 transport 文字消息转换成 LLM 可消费的帧。 run_immediately(默认/打断):先通过 on_text_input 事件把用户文字交给 run_pipeline 登记,再用 broadcast_interruption() 打断当前播报。新的 LLM 回复由 assistant aggregator 确认处理完 interruption 后触发。 run_immediately=False(RTVI send-text 静默追加):仅把文字写进上下文, 不打断、不触发推理。 """ def __init__(self, should_ignore_input: Callable[[], bool] | None = None): super().__init__() self._should_ignore_input = should_ignore_input or (lambda: False) # 立即触发的文字(含打断语义)走 on_text_input;静默追加另走一条事件 self._register_event_handler("on_text_input") self._register_event_handler("on_text_append") self._register_event_handler("on_client_ready") async def process_frame(self, frame, direction: FrameDirection): await super().process_frame(frame, direction) if not isinstance(frame, InputTransportMessageFrame): await self.push_frame(frame, direction) return if isinstance(frame.message, dict) and frame.message.get("type") == "client-ready": await self._call_event_handler("on_client_ready") return parsed = _text_input(frame.message) if not parsed: await self.push_frame(frame, direction) return if self._should_ignore_input(): logger.debug("通话正在结束,忽略后续文字输入") return text, run_immediately = parsed if run_immediately: # 先登记文字再打断。下一轮 LLM 由 assistant aggregator 在真正处理完 # InterruptionFrame 后触发,避免新回复被这次 interruption 一起取消。 await self._call_event_handler("on_text_input", text) await self.broadcast_interruption() else: await self._call_event_handler("on_text_append", text) class CallEndingUserMuteStrategy(BaseUserMuteStrategy): """Keep user media muted after an end-call tool starts terminating a call.""" def __init__(self, is_call_ending: Callable[[], bool]): super().__init__() self._is_call_ending = is_call_ending async def process_frame(self, frame) -> bool: await super().process_frame(frame) return self._is_call_ending() class VisionCaptureProcessor(FrameProcessor): """Capture one requested video frame for auxiliary vision-model analysis.""" def __init__(self, timeout_s: float = 3.0): super().__init__() self._timeout_s = timeout_s self._pending: dict[str, asyncio.Future[UserImageRawFrame]] = {} async def request_image( self, requester: FrameProcessor, request: UserImageRequestFrame, ) -> UserImageRawFrame: key = request.tool_call_id or str(uuid4()) request.tool_call_id = key request.append_to_context = False request.result_callback = None loop = asyncio.get_running_loop() future: asyncio.Future[UserImageRawFrame] = loop.create_future() self._pending[key] = future await requester.push_frame(request, FrameDirection.UPSTREAM) try: return await asyncio.wait_for(future, timeout=self._timeout_s) finally: self._pending.pop(key, None) async def process_frame(self, frame, direction: FrameDirection): await super().process_frame(frame, direction) if ( isinstance(frame, UserImageRawFrame) and frame.request and frame.request.tool_call_id and frame.request.tool_call_id in self._pending ): future = self._pending[frame.request.tool_call_id] if not future.done(): future.set_result(frame) return await self.push_frame(frame, direction) class RealtimeDynamicVariableProcessor(FrameProcessor): """Keep realtime system turn/history variables current between responses.""" def __init__(self, brain: Brain, cfg: AssistantConfig, realtime): super().__init__() self._brain = brain self._cfg = cfg self._realtime = realtime async def _refresh_instructions(self) -> None: update = getattr(self._realtime, "update_instructions", None) if callable(update): await update(self._brain.system_prompt(self._cfg)) async def process_frame(self, frame, direction: FrameDirection): await super().process_frame(frame, direction) if isinstance(frame, OutputTransportMessageUrgentFrame): message = frame.message if isinstance(message, dict): event_type = message.get("type") if event_type == "transcript" and message.get("role") == "user": content = str(message.get("content") or "").strip() if content: self._brain.record_user_message(content) await self._refresh_instructions() elif event_type == "assistant-text-end": await self._brain.on_assistant_text_end( str(message.get("turn_id") or ""), str(message.get("content") or ""), bool(message.get("interrupted", False)), ) await self._refresh_instructions() await self.push_frame(frame, direction) class RealtimeTextInputProcessor(FrameProcessor): """Route text input directly to a realtime service without cascade semantics.""" def __init__(self): super().__init__() self._register_event_handler("on_text_input") self._register_event_handler("on_text_append") async def process_frame(self, frame, direction: FrameDirection): await super().process_frame(frame, direction) if not isinstance(frame, InputTransportMessageFrame): await self.push_frame(frame, direction) return parsed = _text_input(frame.message) if not parsed: await self.push_frame(frame, direction) return text, run_immediately = parsed await self._call_event_handler( "on_text_input" if run_immediately else "on_text_append", text, ) class ConversationHistoryProcessor(FrameProcessor): """从最终客户端事件旁路保存历史，不改变 Pipecat 的上下文与帧语义。""" def __init__(self, recorder: ConversationRecorder | None): super().__init__() self._recorder = recorder async def process_frame(self, frame, direction: FrameDirection): await super().process_frame(frame, direction) await self.push_frame(frame, direction) if self._recorder and isinstance(frame, OutputTransportMessageUrgentFrame): await self._recorder.record_transport_message(frame.message) class KnowledgeRetrievalProcessor(FrameProcessor): """Retrieve before local LLM inference without changing Pipecat internals.""" def __init__( self, knowledge_base_id: str | None, top_n: int = 5, score_threshold: float = 0.0, ): super().__init__() self._knowledge_base_id = knowledge_base_id self._top_n = top_n self._score_threshold = score_threshold self._mode = "automatic" if knowledge_base_id else "disabled" self._last_signature = "" def set_scope(self, scope: dict) -> None: self._knowledge_base_id = scope.get("knowledge_base_id") or None self._mode = str(scope.get("mode") or "disabled") self._top_n = int(scope.get("top_n") or 5) self._score_threshold = float(scope.get("score_threshold") or 0.0) self._last_signature = "" def _clear_context(self, messages: list[dict]) -> None: # Remove the legacy Workflow knowledge message so an in-flight context # created before this compatibility fix cannot keep sending that role. messages[:] = [ message for message in messages if not ( message.get("role") == "developer" and KNOWLEDGE_CONTEXT_MARKER in str(message.get("content") or "") ) ] system_message = next( (message for message in messages if message.get("role") == "system"), None, ) if system_message is not None: content = str(system_message.get("content") or "") system_message["content"] = content.split(KNOWLEDGE_CONTEXT_MARKER, 1)[0].rstrip() def _set_context(self, messages: list[dict], block: str) -> None: """Store retrieved knowledge in a provider-compatible system message.""" self._clear_context(messages) system_message = next( (message for message in messages if message.get("role") == "system"), None, ) if system_message is None: messages.insert(0, {"role": "system", "content": block}) return content = str(system_message.get("content") or "").rstrip() system_message["content"] = f"{content}\n\n{block}" if content else block async def process_frame(self, frame, direction: FrameDirection): await super().process_frame(frame, direction) if not isinstance(frame, LLMContextFrame): await self.push_frame(frame, direction) return messages = frame.context.get_messages() if self._mode != "automatic" or not self._knowledge_base_id: self._clear_context(messages) await self.push_frame(frame, direction) return user_messages = [message for message in messages if message.get("role") == "user"] if not user_messages: await self.push_frame(frame, direction) return query = str(user_messages[-1].get("content") or "").strip() signature = f"{len(user_messages)}:{query}" if not query or signature == self._last_signature: await self.push_frame(frame, direction) return self._last_signature = signature try: async with SessionLocal() as session: results = await search_knowledge( session, self._knowledge_base_id, query, top_k=self._top_n, score_threshold=self._score_threshold, ) except Exception as exc: logger.warning(f"自动知识库检索失败: {exc}") results = [] sources = "\n\n".join( f"[{index + 1}] 来源：{item['document']}（相关度 {item['score']}）\n{item['content']}" for index, item in enumerate(results) ) or "未检索到相关资料。" block = f"{KNOWLEDGE_CONTEXT_MARKER}\n当前问题的知识库检索结果：\n{sources}" self._set_context(messages, block) await self.push_frame(frame, direction) class UserTurnRoutingProcessor(FrameProcessor): """Give a brain first right of refusal before a new user turn reaches the LLM.""" def __init__(self, brain: Brain): super().__init__() self._brain = brain self._last_user_message: dict | None = None async def process_frame(self, frame, direction: FrameDirection): await super().process_frame(frame, direction) if direction != FrameDirection.DOWNSTREAM or not isinstance( frame, LLMContextFrame ): await self.push_frame(frame, direction) return user_message = next( ( message for message in reversed(frame.context.get_messages()) if message.get("role") == "user" and isinstance(message.get("content"), str) and str(message.get("content") or "").strip() ), None, ) if user_message is None: await self.push_frame(frame, direction) return if user_message is self._last_user_message: # Programmatic LLMRunFrame after a node transition reuses the same # user message. It is a response run, not another routing event. await self.push_frame(frame, direction) return self._last_user_message = user_message content = str(user_message.get("content") or "").strip() handled = await self._brain.on_user_turn_end(content) if not handled: await self.push_frame(frame, direction) class PassthroughLLMAssistantAggregator(LLMAssistantAggregator): """聚合 LLM 回复进上下文,同时继续把回复帧交给下游 TTS。""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._register_event_handler("on_interruption_processed") self._register_event_handler("on_assistant_text_start") self._register_event_handler("on_assistant_text_delta") self._register_event_handler("on_assistant_text_end") self._stream_turn_id: str | None = None self._stream_timestamp = "" self._stream_text = "" async def process_frame(self, frame, direction: FrameDirection): await super().process_frame(frame, direction) if isinstance(frame, LLMFullResponseStartFrame): self._stream_turn_id = uuid4().hex self._stream_timestamp = time_now_iso8601() self._stream_text = "" await self._call_event_handler( "on_assistant_text_start", self._stream_turn_id, self._stream_timestamp, ) elif isinstance(frame, LLMTextFrame) and self._stream_turn_id: self._stream_text += frame.text await self._call_event_handler( "on_assistant_text_delta", self._stream_turn_id, frame.text, ) elif isinstance(frame, LLMFullResponseEndFrame): await self._finish_text_stream(interrupted=False) # LLMAssistantAggregator 默认会消费这些帧。放在 TTS 前用于中断时保存 # 已生成前缀时,必须显式透传,否则 TTS 收不到任何 LLM 回复。 if isinstance( frame, (LLMFullResponseStartFrame, LLMFullResponseEndFrame, TextFrame), ): await self.push_frame(frame, direction) elif isinstance(frame, InterruptionFrame): await self._finish_text_stream(interrupted=True) await self._call_event_handler("on_interruption_processed") async def _finish_text_stream(self, *, interrupted: bool): if not self._stream_turn_id: return await self._call_event_handler( "on_assistant_text_end", self._stream_turn_id, self._stream_text, interrupted, ) self._stream_turn_id = None self._stream_timestamp = "" self._stream_text = "" class WorkflowAggregatorPair: """Small public-shape adapter required by Pipecat FlowManager.""" def __init__(self, user_aggregator, assistant_aggregator): self._user = user_aggregator self._assistant = assistant_aggregator def user(self): return self._user def assistant(self): return self._assistant def _workflow_service_switcher( cfg: AssistantConfig, capability: str, base_service: FrameProcessor ): """Build one switcher and an ID lookup for every referenced voice resource.""" create = create_stt if capability == "ASR" else create_tts settings = cfg.graph.get("settings") or {} default_key = ( "defaultAsrResourceId" if capability == "ASR" else "defaultTtsResourceId" ) default_id = str(settings.get(default_key) or "") services_by_id = {} for resource_id, resource in cfg.workflow_model_resources.items(): if resource.capability != capability: continue services_by_id[resource_id] = ( base_service if resource_id == default_id else create(config_with_resource(cfg, resource)) ) primary = services_by_id.get(default_id, base_service) services = [primary] services.extend( service for service in services_by_id.values() if service is not primary ) if base_service is not primary: services.append(base_service) return ServiceSwitcher(services=services), services_by_id, primary def _workflow_llm_switcher(cfg: AssistantConfig, base_service): """Build an LLM switcher for the global model and Agent overrides.""" settings = cfg.graph.get("settings") or {} default_id = str(settings.get("defaultLlmResourceId") or "") services_by_id = {} for resource_id, resource in cfg.workflow_model_resources.items(): if resource.capability != "LLM": continue services_by_id[resource_id] = ( base_service if resource_id == default_id else create_llm(config_with_resource(cfg, resource)) ) primary = services_by_id.get(default_id, base_service) services = [primary] services.extend( service for service in services_by_id.values() if service is not primary ) if base_service is not primary: services.append(base_service) return LLMSwitcher(llms=services), services_by_id, primary async def run_pipeline( transport, cfg: AssistantConfig, *, vision_enabled: bool = False, assistant_id: str | None = None, channel: str = "webrtc", ) -> None: """在给定 transport 上构建并运行管线,直到连接结束。 Args: transport: 任意 pipecat transport(WebRTC / WS / 电话…), 只要有 .input() / .output() / event_handler 即可。 cfg: 助手配置(随请求内联传入)。 """ logger.info( f"启动管线: assistant={cfg.name} type={cfg.type} " f"mode={cfg.runtimeMode} vision={vision_enabled}" ) # 大脑:按类型决定 LLM 槽/开场白/上下文归属。每通电话一个实例(可持会话状态)。 brain = build_brain(cfg) if ( cfg.runtimeMode == "realtime" and "realtime" not in brain.spec.supported_runtime_modes ): raise ValueError(f"类型 {cfg.type} 不支持 realtime 运行模式") if cfg.runtimeMode == "realtime": if vision_enabled: logger.warning("Realtime 模式暂未接入视频帧工具,本次仅启用语音通话") await run_realtime_pipeline( transport, cfg, brain=brain, assistant_id=assistant_id, channel=channel, ) return graph_settings = cfg.graph.get("settings") or {} default_llm_resource = cfg.workflow_model_resources.get( str(graph_settings.get("defaultLlmResourceId") or "") ) default_asr_resource = cfg.workflow_model_resources.get( str(graph_settings.get("defaultAsrResourceId") or "") ) default_tts_resource = cfg.workflow_model_resources.get( str(graph_settings.get("defaultTtsResourceId") or "") ) stt = create_stt( config_with_resource(cfg, default_asr_resource) if cfg.type == "workflow" and default_asr_resource else cfg ) tts = create_tts( config_with_resource(cfg, default_tts_resource) if cfg.type == "workflow" and default_tts_resource else cfg ) stt_processor = stt tts_processor = tts stt_services: dict[str, FrameProcessor] = {} tts_services: dict[str, FrameProcessor] = {} current_voice_services: dict[str, FrameProcessor] = {"asr": stt, "tts": tts} if cfg.type == "workflow": stt_processor, stt_services, current_voice_services["asr"] = ( _workflow_service_switcher(cfg, "ASR", stt) ) tts_processor, tts_services, current_voice_services["tts"] = ( _workflow_service_switcher(cfg, "TTS", tts) ) greeting = await brain.greeting(cfg) system_content = brain.system_prompt(cfg) worker_holder: dict = {} async def queue_call_end(reason: str) -> None: worker = worker_holder.get("worker") if worker is None: return logger.info(f"结束通话: reason={reason}") await worker.queue_frame( OutputTransportMessageUrgentFrame( message={"type": "call-ended", "reason": reason} ) ) await worker.queue_frame(EndFrame()) call_end = CallEndCoordinator(queue_call_end) knowledge_config = cfg.knowledge_retrieval_config knowledge_mode = str(knowledge_config.get("mode", "automatic")) knowledge_top_n = int( knowledge_config.get("top_n", knowledge_config.get("topN", 5)) ) knowledge_score_threshold = float( knowledge_config.get( "score_threshold", knowledge_config.get("scoreThreshold", 0.0) ) ) automatic_knowledge_id = ( cfg.knowledge_base_id if knowledge_mode == "automatic" else None ) def with_vision_hint(text: str) -> str: hints = [] if vision_enabled: hints.append(VISION_SYSTEM_HINT) if cfg.knowledge_base_id: hints.append( AUTOMATIC_KNOWLEDGE_SYSTEM_HINT if knowledge_mode == "automatic" else ON_DEMAND_KNOWLEDGE_SYSTEM_HINT ) return "\n\n".join(part for part in [text, *hints] if part) context = LLMContext( messages=( [] if cfg.type == "workflow" else [{"role": "system", "content": with_vision_hint(system_content)}] ) ) input_state = {"enabled": True} # LLM 槽由大脑提供:本地模型或 Dify/FastGPT 外部托管适配器。 llm = brain.build_llm( config_with_resource(cfg, default_llm_resource) if cfg.type == "workflow" and default_llm_resource else cfg, context, ) llm_services: dict[str, FrameProcessor] = {} current_llm_service = llm if cfg.type == "workflow": llm, llm_services, current_llm_service = _workflow_llm_switcher(cfg, llm) user_aggregator = ConfigurableLLMUserAggregator( context, params=LLMUserAggregatorParams( vad_analyzer=create_vad_analyzer(cfg.turnConfig), user_mute_strategies=[ FunctionCallUserMuteStrategy(), CallEndingUserMuteStrategy( lambda: call_end.ending or not input_state["enabled"] ), ], user_turn_strategies=create_user_turn_strategies( cfg.turnConfig, enable_interruptions=cfg.enableInterrupt, ), ), ) user_turn_router = UserTurnRoutingProcessor(brain) assistant_aggregator = PassthroughLLMAssistantAggregator(context) text_input = TextInputProcessor( should_ignore_input=lambda: call_end.ending or not input_state["enabled"] ) vision_capture = VisionCaptureProcessor() knowledge_retrieval = KnowledgeRetrievalProcessor( automatic_knowledge_id, top_n=knowledge_top_n, score_threshold=knowledge_score_threshold, ) vision_native_mode = vision_enabled and _vision_uses_main_llm(cfg) vision_state: dict[str, str | None] = {"client_id": None} vision_schema = FunctionSchema( name=VISION_TOOL_NAME, description=( "获取用户当前摄像头画面。当用户询问当前画面、看到了什么、" "人/物品/环境状态或需要视觉判断时调用。" ), properties={ "question": { "type": "string", "description": "用户关于当前视频画面的具体问题。", } }, required=["question"], ) knowledge_schema = FunctionSchema( name=KNOWLEDGE_TOOL_NAME, description=_knowledge_tool_description(cfg), properties={ "query": {"type": "string", "description": "用于检索的完整问题或关键词"} }, required=["query"], ) async def search_bound_knowledge(params: FunctionCallParams): query = str(params.arguments.get("query") or "").strip() if not query or not cfg.knowledge_base_id: await params.result_callback({"status": "error", "message": "检索问题为空或未绑定知识库"}) return try: async with SessionLocal() as session: results = await search_knowledge( session, cfg.knowledge_base_id, query, top_k=knowledge_top_n, score_threshold=knowledge_score_threshold, ) await params.result_callback({"status": "ok", "results": results}) except Exception as exc: logger.exception(f"知识库检索失败: {exc}") await params.result_callback({"status": "error", "message": "知识库检索暂时不可用"}) async def fetch_user_image(params: FunctionCallParams): question = str(params.arguments.get("question") or "请描述当前画面。") user_id = vision_state.get("client_id") if not user_id: await params.result_callback( { "status": "no_video_client", "message": "当前还没有可用的摄像头视频流。", } ) return request = UserImageRequestFrame( user_id=user_id, text=question, append_to_context=vision_native_mode, function_name=params.function_name, tool_call_id=params.tool_call_id, result_callback=params.result_callback if vision_native_mode else None, ) if vision_native_mode: logger.debug( f"请求当前视频帧进入主 LLM 上下文: user_id={user_id}, question={question}" ) await params.llm.push_frame(request, FrameDirection.UPSTREAM) return logger.debug( f"请求当前视频帧给单独视觉模型分析: user_id={user_id}, question={question}" ) try: frame = await vision_capture.request_image(params.llm, request) observation = await _analyze_image_with_vision_model(cfg, frame, question) except asyncio.TimeoutError: await params.result_callback( { "status": "timeout", "message": "等待摄像头视频帧超时。", } ) return except Exception as e: logger.exception(f"视觉模型分析失败: {e}") await params.result_callback( { "status": "error", "message": f"视觉理解失败: {type(e).__name__}", } ) return await params.result_callback( { "status": "ok", "question": question, "observation": observation or "视觉模型没有返回有效观察结果。", } ) flow_global_functions = [] if cfg.type == "workflow" and vision_enabled: async def flow_fetch_user_image(args, _flow_manager): question = str((args or {}).get("question") or "请描述当前画面。") user_id = vision_state.get("client_id") if not user_id: return { "status": "no_video_client", "message": "当前还没有可用的摄像头视频流。", } request = UserImageRequestFrame( user_id=user_id, text=question, append_to_context=False, function_name=VISION_TOOL_NAME, ) try: frame = await vision_capture.request_image(llm, request) observation = await _analyze_image_with_vision_model(cfg, frame, question) return { "status": "ok", "question": question, "observation": observation or "视觉模型没有返回有效观察结果。", } except asyncio.TimeoutError: return {"status": "timeout", "message": "等待摄像头视频帧超时。"} except Exception as exc: # noqa: BLE001 - return tool errors to the LLM logger.warning(f"Workflow 视觉理解失败:{exc}") return {"status": "error", "message": "视觉理解暂时不可用。"} flow_global_functions.append( FlowsFunctionSchema( name=VISION_TOOL_NAME, description=vision_schema.description, properties=vision_schema.properties, required=vision_schema.required, handler=flow_fetch_user_image, cancel_on_interruption=True, ) ) if vision_enabled and cfg.type != "workflow": llm.register_function(VISION_TOOL_NAME, fetch_user_image) if cfg.knowledge_base_id and knowledge_mode == "on_demand": llm.register_function(KNOWLEDGE_TOOL_NAME, search_bound_knowledge) def set_visible_tools(schemas: list[FunctionSchema] | None = None) -> None: tools = list(schemas or []) if vision_enabled and cfg.type != "workflow": tools.append(vision_schema) if cfg.knowledge_base_id and knowledge_mode == "on_demand": tools.append(knowledge_schema) if tools: context.set_tools(ToolsSchema(standard_tools=tools)) else: context.set_tools() recorder = await ConversationRecorder.start( assistant_id=assistant_id, assistant_name=cfg.name, channel=channel, runtime_mode=cfg.runtimeMode, session_id=cfg.conversation_id or None, ) pipeline = Pipeline( [ transport.input(), vision_capture, text_input, stt_processor, user_aggregator, user_turn_router, knowledge_retrieval, llm, # Aggregate the streamed LLM text before TTS. On interruption, # Pipecat commits the generated prefix immediately instead of # waiting for a TTS provider to emit spoken-text/timestamp frames. assistant_aggregator, tts_processor, EndCallAfterSpeechProcessor(call_end), ConversationHistoryProcessor(recorder), transport.output(), ] ) worker = PipelineWorker( pipeline, params=PipelineParams( enable_metrics=False, ), enable_rtvi=False, ) worker_holder["worker"] = worker default_workflow_services = { "llm": current_llm_service, **current_voice_services, } async def switch_workflow_services( llm_resource_id: str | None, asr_resource_id: str | None, tts_resource_id: str | None, ) -> None: nonlocal current_llm_service requested = ( ("llm", llm_services, llm_resource_id), ("asr", stt_services, asr_resource_id), ("tts", tts_services, tts_resource_id), ) for kind, services, resource_id in requested: target = ( services.get(resource_id) if resource_id else default_workflow_services[kind] ) if target is None: raise ValueError(f"Workflow {kind.upper()} 资源未加载:{resource_id}") current = ( current_llm_service if kind == "llm" else current_voice_services[kind] ) if current is target: continue await worker.queue_frame(ManuallySwitchServiceFrame(service=target)) if kind == "llm": current_llm_service = target else: current_voice_services[kind] = target await worker.queue_frame( OutputTransportMessageUrgentFrame( message={ "type": "service-switched", "capability": kind.upper(), "resourceId": resource_id, } ) ) current_enable_interrupt = cfg.enableInterrupt current_turn_config = dict(cfg.turnConfig) async def apply_workflow_turn_config( enable_interrupt: bool, turn_config: dict[str, Any], ) -> None: """Apply one Agent's interaction policy before its next user turn.""" nonlocal current_enable_interrupt, current_turn_config normalized = dict(turn_config or {}) if ( current_enable_interrupt == enable_interrupt and current_turn_config == normalized ): return await user_aggregator.apply_turn_strategies( normalized, enable_interruptions=enable_interrupt, ) await worker.queue_frame( VADParamsUpdateFrame(params=create_vad_params(normalized)) ) current_enable_interrupt = enable_interrupt current_turn_config = normalized async def queue_transcript(role: str, content: str, timestamp: str) -> None: if content: await worker.queue_frame( OutputTransportMessageUrgentFrame( message={ "type": "transcript", "role": role, "content": content, "timestamp": timestamp, }, ) ) greeting_transcript_sent = False pending_text_inputs: list[str] = [] def set_system_prompt(text: str) -> None: """替换上下文里的系统提示(节点切换时整体替换,而非追加)。""" messages = context.get_messages() content = with_vision_hint(text) if messages and messages[0].get("role") == "system": messages[0] = {"role": "system", "content": content} else: messages.insert(0, {"role": "system", "content": content}) set_visible_tools([]) await brain.setup( cfg, BrainRuntime( context=context, llm=llm, queue_frame=worker.queue_frame, set_system_prompt=set_system_prompt, set_tools=set_visible_tools, call_end=call_end, worker=worker, context_aggregator=WorkflowAggregatorPair( user_aggregator, assistant_aggregator, ), transport=transport, switch_services=switch_workflow_services, set_knowledge_scope=knowledge_retrieval.set_scope, set_input_enabled=lambda enabled: input_state.__setitem__("enabled", enabled), apply_turn_config=apply_workflow_turn_config, flow_global_functions=flow_global_functions, ), ) async def append_user_text_to_context(text: str, *, run_llm: bool) -> None: await worker.queue_frame( LLMMessagesAppendFrame( messages=[{"role": "user", "content": text}], run_llm=run_llm, ) ) @user_aggregator.event_handler("on_user_turn_stopped") async def on_user_turn_stopped(_aggregator, _strategy, message): await queue_transcript("user", message.content, message.timestamp) @assistant_aggregator.event_handler("on_assistant_text_start") async def on_assistant_text_start(_aggregator, turn_id, timestamp): await brain.on_assistant_text_start(turn_id) await worker.queue_frame( OutputTransportMessageUrgentFrame( message={ "type": "assistant-text-start", "turn_id": turn_id, "timestamp": timestamp, } ) ) @assistant_aggregator.event_handler("on_assistant_text_delta") async def on_assistant_text_delta(_aggregator, turn_id, delta): await worker.queue_frame( OutputTransportMessageUrgentFrame( message={ "type": "assistant-text-delta", "turn_id": turn_id, "delta": delta, } ) ) @assistant_aggregator.event_handler("on_assistant_text_end") async def on_assistant_text_end(_aggregator, turn_id, content, interrupted): await worker.queue_frame( OutputTransportMessageUrgentFrame( message={ "type": "assistant-text-end", "turn_id": turn_id, "content": content, "interrupted": interrupted, } ) ) await brain.on_assistant_text_end(turn_id, content, interrupted) @text_input.event_handler("on_text_input") async def on_text_input(_processor, text): pending_text_inputs.append(text) # 前端显示不依赖 interruption 后续事件,必须在打断前先排入发送队列。 await queue_transcript("user", text, time_now_iso8601()) @assistant_aggregator.event_handler("on_interruption_processed") async def on_interruption_processed(_aggregator): if not pending_text_inputs: return text = pending_text_inputs.pop(0) # assistant aggregator 已处理完 interruption,现在再启动下一轮 LLM。 await append_user_text_to_context(text, run_llm=True) @text_input.event_handler("on_text_append") async def on_text_append(_processor, text): # 静默追加:写进上下文但不打断、不触发推理;transcript 照常上报 brain.record_user_message(text) await queue_transcript("user", text, time_now_iso8601()) await append_user_text_to_context(text, run_llm=False) @text_input.event_handler("on_client_ready") async def on_client_ready(_processor): nonlocal greeting_transcript_sent if greeting and not greeting_transcript_sent: greeting_transcript_sent = True await queue_transcript("assistant", greeting, time_now_iso8601()) @transport.event_handler("on_client_connected") async def on_client_connected(_transport, _client): if vision_enabled: try: vision_state["client_id"] = get_transport_client_id( _transport, _client, ) await maybe_capture_participant_camera(_transport, _client) logger.info(f"视觉理解已接入视频客户端: {vision_state['client_id']}") except Exception as e: logger.warning(f"视觉理解摄像头捕获初始化失败: {e}") if greeting: # 外部托管类型的上下文由对方服务端维护,开场白不写入本地 context if brain.spec.owns_context: context.add_message({"role": "assistant", "content": greeting}) await worker.queue_frame(TTSSpeakFrame(greeting, append_to_context=False)) await brain.on_connected() @transport.event_handler("on_client_disconnected") async def on_client_disconnected(_transport, _client): logger.info("对端断开,结束管线") await worker.queue_frame(EndFrame()) runner = WorkerRunner(handle_sigint=False) run_status = "completed" try: await runner.add_workers(worker) await runner.run() except Exception: run_status = "failed" raise finally: if recorder: await recorder.finish(status=run_status) logger.info("管线已结束") async def run_realtime_pipeline( transport, cfg: AssistantConfig, *, brain: Brain, assistant_id: str | None = None, channel: str = "webrtc", ) -> None: """Run a speech-to-speech model that owns ASR, reasoning, and synthesis.""" realtime = create_realtime_service( cfg, instructions=brain.system_prompt(cfg), ) text_input = RealtimeTextInputProcessor() dynamic_variables = RealtimeDynamicVariableProcessor(brain, cfg, realtime) greeting = await brain.greeting(cfg) recorder = await ConversationRecorder.start( assistant_id=assistant_id, assistant_name=cfg.name, channel=channel, runtime_mode=cfg.runtimeMode, session_id=cfg.conversation_id or None, ) pipeline = Pipeline( [ transport.input(), text_input, realtime, dynamic_variables, ConversationHistoryProcessor(recorder), transport.output(), ] ) worker = PipelineWorker( pipeline, params=PipelineParams( enable_metrics=False, audio_in_sample_rate=int( cfg.realtime_values.get("inputSampleRate") or 24000 ), audio_out_sample_rate=int( cfg.realtime_values.get("outputSampleRate") or 24000 ), ), enable_rtvi=False, ) async def queue_transcript(role: str, content: str) -> None: if content: await worker.queue_frame( OutputTransportMessageUrgentFrame( message={ "type": "transcript", "role": role, "content": content, "timestamp": time_now_iso8601(), }, ) ) @text_input.event_handler("on_text_input") async def on_text_input(_processor, text): await queue_transcript("user", text) await realtime.interrupt() await realtime.send_text(text, run_immediately=True) @text_input.event_handler("on_text_append") async def on_text_append(_processor, text): await queue_transcript("user", text) await realtime.send_text(text, run_immediately=False) @transport.event_handler("on_client_connected") async def on_client_connected(_transport, _client): if greeting: await realtime.speak(greeting) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(_transport, _client): logger.info("Realtime 对端断开,结束管线") await worker.queue_frame(EndFrame()) runner = WorkerRunner(handle_sigint=False) run_status = "completed" try: await runner.add_workers(worker) await runner.run() except Exception: run_status = "failed" raise finally: if recorder: await recorder.finish(status=run_status) logger.info("Realtime 管线已结束")