"""Transportless Pipecat runner for one persisted fixed-text test case.""" from __future__ import annotations import asyncio import json import time from dataclasses import dataclass, field from typing import Any, Awaitable, Callable from uuid import uuid4 from loguru import logger from models import AssistantConfig from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.frames.frames import ( BotStoppedSpeakingFrame, CancelFrame, EndFrame, ErrorFrame, FunctionCallInProgressFrame, FunctionCallResultFrame, LLMContextFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMessagesAppendFrame, ManuallySwitchServiceFrame, TTSSpeakFrame, ) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.worker import PipelineParams, PipelineWorker from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import ( LLMContextAggregatorPair, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.llm_service import FunctionCallParams from pipecat.workers.runner import WorkerRunner from db.session import SessionLocal from services.brains import BrainRuntime, build_brain from services.knowledge import search as search_knowledge from services.pipecat.call_lifecycle import ( CallEndCoordinator, FixedSpeechPlaybackMarkerFrame, ) from services.pipecat.processors import ( KnowledgeRetrievalProcessor, UserTurnRoutingProcessor, ) from services.pipecat.service_factory import config_with_resource from services.pipecat.workflow_services import build_workflow_llm_switcher from services.test_runs.errors import TestExecutionError from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort from test_schemas import ContextTurn, FixedInputTurn, TestCaseDefinition IDLE_SETTLE_SECONDS = 0.2 WORKER_START_TIMEOUT_SECONDS = 5.0 INTERNAL_CANCEL_REASON = "text_test_case_complete" AUTOMATIC_KNOWLEDGE_HINT = ( "你已连接内部知识库。系统会在每轮用户问题前自动提供相关资料；" "回答资料事实时只根据检索内容，资料不足要明确说明。" ) ON_DEMAND_KNOWLEDGE_HINT = ( "你已连接内部知识库。当用户问题涉及可能存在于业务知识库中的事实时，" "先调用 search_knowledge_base 检索；回答资料事实时只根据检索内容，" "资料不足要明确说明。" ) @dataclass class RawToolCall: id: str function_name: str arguments: Any result: Any = None outcome: str = "error" duration_ms: int = 0 _started_at: float = field(default_factory=time.monotonic, repr=False) def as_result(self) -> dict[str, Any]: return { "id": self.id, "functionName": self.function_name, "argumentsJson": json.dumps( self.arguments, ensure_ascii=False, indent=2, default=str ), "resultJson": json.dumps( self.result, ensure_ascii=False, indent=2, default=str ), "outcome": self.outcome, "durationMs": self.duration_ms, } @dataclass class RawTurnResult: id: str index: int user_input: str assistant_reply: str tool_calls: list[RawToolCall] @dataclass class TextCaseResult: turns: list[RawTurnResult] transcript: list[dict[str, str]] @dataclass class _ResponseWindow: active_llm: int = 0 active_assistant: int = 0 pending_tools: set[str] = field(default_factory=set) activity_count: int = 0 last_activity_at: float = field(default_factory=time.monotonic) def touch(self) -> None: self.activity_count += 1 self.last_activity_at = time.monotonic() @property def idle(self) -> bool: return self.active_llm == 0 and self.active_assistant == 0 and not self.pending_tools class _TextCaptureProcessor(FrameProcessor): """Capture assistant/tool output and expose a reliable response boundary.""" def __init__(self, context: LLMContext): super().__init__() self.context = context self.window = _ResponseWindow() self.outputs: list[str] = [] self.tool_calls: list[RawToolCall] = [] self._tool_by_id: dict[str, RawToolCall] = {} self.error: TestExecutionError | None = None self.on_fixed_speech_complete: Callable[[], Awaitable[None]] | None = None async def process_frame(self, frame, direction: FrameDirection): await super().process_frame(frame, direction) if isinstance(frame, TTSSpeakFrame): text = frame.text.strip() if text: self.outputs.append(text) if frame.append_to_context: self.context.add_message({"role": "assistant", "content": text}) self.window.touch() if self.on_fixed_speech_complete is not None: await self.on_fixed_speech_complete() return if isinstance(frame, FixedSpeechPlaybackMarkerFrame): await frame.completion.mark_played() self.window.touch() return if isinstance(frame, LLMFullResponseStartFrame): self.window.active_llm += 1 self.window.touch() elif isinstance(frame, LLMFullResponseEndFrame): self.window.active_llm = max(0, self.window.active_llm - 1) self.window.touch() await self.push_frame(frame, direction) # There is no TTS/output transport. This releases provider/tool # follow-up logic that normally waits for a speaking boundary. await self.push_frame(BotStoppedSpeakingFrame(), FrameDirection.UPSTREAM) return elif isinstance(frame, FunctionCallInProgressFrame): call = RawToolCall( id=frame.tool_call_id, function_name=frame.function_name, arguments=frame.arguments or {}, ) self.tool_calls.append(call) self._tool_by_id[call.id] = call self.window.pending_tools.add(call.id) self.window.touch() elif isinstance(frame, FunctionCallResultFrame): call = self._tool_by_id.get(frame.tool_call_id) if call is None: call = RawToolCall( id=frame.tool_call_id, function_name=frame.function_name, arguments=frame.arguments or {}, ) self.tool_calls.append(call) self._tool_by_id[call.id] = call call.result = frame.result call.duration_ms = max(0, round((time.monotonic() - call._started_at) * 1000)) status = frame.result.get("status") if isinstance(frame.result, dict) else None call.outcome = "error" if status in {"error", "failed", "timeout"} else "success" self.window.pending_tools.discard(call.id) self.window.touch() elif isinstance(frame, ErrorFrame): self.error = TestExecutionError( code="PIPELINE_ERROR", message=frame.error or "Pipeline 执行失败", stage="model" if frame.processor else "pipeline", retryable=True, ) self.window.touch() elif isinstance(frame, (EndFrame, CancelFrame)): self.window.touch() await self.push_frame(frame, direction) def _context_messages(context_turns: list[ContextTurn]) -> list[dict[str, Any]]: messages: list[dict[str, Any]] = [] latest_call_by_name: dict[str, str] = {} for index, turn in enumerate(context_turns): if turn.role in {"agent", "user"}: messages.append( { "role": "assistant" if turn.role == "agent" else "user", "content": turn.content, } ) continue call_id = turn.tool_call_id or latest_call_by_name.get(turn.tool_name or "") call_id = call_id or f"context_tool_{index}_{uuid4().hex[:8]}" if turn.role == "tool_call": latest_call_by_name[turn.tool_name or ""] = call_id messages.append( { "role": "assistant", "tool_calls": [ { "id": call_id, "type": "function", "function": { "name": turn.tool_name, "arguments": json.dumps( json.loads(turn.content), ensure_ascii=False ), }, } ], } ) else: result_payload = json.loads(turn.content) if turn.is_error: result_payload = ( {**result_payload, "status": "error"} if isinstance(result_payload, dict) else {"status": "error", "data": result_payload} ) messages.append( { "role": "tool", "tool_call_id": call_id, "content": json.dumps(result_payload, ensure_ascii=False), } ) return messages async def _wait_for_quiescence( capture: _TextCaptureProcessor, runner_task: asyncio.Task[None], activity_marker: int, *, require_activity: bool = True, ) -> None: while True: if capture.error is not None: raise capture.error if runner_task.done(): await runner_task return has_activity = capture.window.activity_count > activity_marker settled = time.monotonic() - capture.window.last_activity_at >= IDLE_SETTLE_SECONDS if capture.window.idle and settled and (has_activity or not require_activity): return await asyncio.sleep(0.05) class TextPipelineRunner: """Execute all fixed input turns in one isolated, transportless pipeline.""" async def run( self, cfg: AssistantConfig, definition: TestCaseDefinition, *, timeout_seconds: int, ) -> TextCaseResult: try: async with asyncio.timeout(timeout_seconds): return await self._run(cfg, definition) except TimeoutError as exc: raise TestExecutionError( code="PIPELINE_TIMEOUT", message=f"测试用例执行超过 {timeout_seconds} 秒", stage="pipeline", retryable=True, ) from exc async def _run( self, cfg: AssistantConfig, definition: TestCaseDefinition, ) -> TextCaseResult: if cfg.runtimeMode != "pipeline": raise TestExecutionError( code="UNSUPPORTED_RUNTIME_MODE", message="第一版文本测试只支持 Pipeline 运行模式", stage="pipeline", ) if cfg.type not in {"prompt", "workflow"}: raise TestExecutionError( code="UNSUPPORTED_ASSISTANT_TYPE", message=f"第一版文本测试暂不支持 {cfg.type} 类型助手", stage="pipeline", ) brain = build_brain(cfg) seed_messages = _context_messages(definition.context_turns) knowledge_config = cfg.knowledge_retrieval_config or {} knowledge_mode = str(knowledge_config.get("mode") or "automatic") def with_knowledge_hint(text: str) -> str: if cfg.type != "prompt" or not cfg.knowledge_base_id: return text hint = ( AUTOMATIC_KNOWLEDGE_HINT if knowledge_mode == "automatic" else ON_DEMAND_KNOWLEDGE_HINT ) return "\n\n".join(part for part in (text, hint) if part) system_prompt = with_knowledge_hint(brain.system_prompt(cfg)) context = LLMContext( messages=( seed_messages if cfg.type == "workflow" else [{"role": "system", "content": system_prompt}, *seed_messages] ) ) graph_settings = cfg.graph.get("settings") or {} default_resource = cfg.workflow_model_resources.get( str(graph_settings.get("defaultLlmResourceId") or "") ) llm_cfg = ( config_with_resource(cfg, default_resource) if cfg.type == "workflow" and default_resource is not None else cfg ) llm = brain.build_llm(llm_cfg, context) llm_services: dict[str, FrameProcessor] = {} current_llm_service: FrameProcessor = llm if cfg.type == "workflow": llm, llm_services, current_llm_service = build_workflow_llm_switcher(cfg, llm) aggregators = LLMContextAggregatorPair(context) user_aggregator = aggregators.user() assistant_aggregator = aggregators.assistant() capture = _TextCaptureProcessor(context) knowledge = KnowledgeRetrievalProcessor( cfg.knowledge_base_id if knowledge_mode == "automatic" else None, top_n=int(knowledge_config.get("top_n", knowledge_config.get("topN", 5))), score_threshold=float( knowledge_config.get( "score_threshold", knowledge_config.get("scoreThreshold", 0.0) ) ), ) pipeline = Pipeline( [ user_aggregator, UserTurnRoutingProcessor(brain), knowledge, llm, capture, assistant_aggregator, ] ) worker = PipelineWorker( pipeline, params=PipelineParams(enable_metrics=False), enable_rtvi=False, enable_turn_tracking=False, idle_timeout_secs=None, ) runner = WorkerRunner(handle_sigint=False, check_dangling_tasks=False) worker_started = asyncio.Event() @worker.event_handler("on_pipeline_started") async def on_pipeline_started(_worker, _frame): worker_started.set() executor_holder: dict[str, MockToolExecutor] = {} def create_test_executor(store) -> MockToolExecutor: executor = MockToolExecutor(store) executor_holder["executor"] = executor return executor async def queue_call_end(reason: str) -> None: logger.debug(f"文本测试收到会话结束请求: {reason}") await worker.queue_frame(EndFrame(reason=reason)) call_end = CallEndCoordinator(queue_call_end) async def finish_fixed_speech_if_ending() -> None: if call_end.ending: await call_end.finish() capture.on_fixed_speech_complete = finish_fixed_speech_if_ending current_service = current_llm_service async def switch_services( llm_resource_id: str | None, _asr_resource_id: str | None, _tts_resource_id: str | None, ) -> None: nonlocal current_service target = ( llm_services.get(llm_resource_id) if llm_resource_id else current_llm_service ) if target is None: raise ValueError(f"Workflow LLM 资源未加载:{llm_resource_id}") if target is current_service: return await worker.queue_frame(ManuallySwitchServiceFrame(service=target)) current_service = target def set_system_prompt(text: str) -> None: messages = context.get_messages() text = with_knowledge_hint(text) if messages and messages[0].get("role") == "system": messages[0] = {"role": "system", "content": text} else: messages.insert(0, {"role": "system", "content": text}) knowledge_schema: FunctionSchema | None = None if ( cfg.type == "prompt" and cfg.knowledge_base_id and knowledge_mode == "on_demand" ): knowledge_schema = FunctionSchema( name="search_knowledge_base", description=( "在当前助手绑定的知识库中检索资料。" f"知识库：{cfg.knowledge_base_name}。" f"{cfg.knowledge_base_description}" ), properties={ "query": { "type": "string", "description": "完整问题或检索关键词", } }, required=["query"], ) async def search_bound_knowledge(params: FunctionCallParams) -> None: query = str(params.arguments.get("query") or "").strip() if not query: await params.result_callback( {"status": "error", "message": "检索问题为空"} ) return try: async with SessionLocal() as session: results = await search_knowledge( session, cfg.knowledge_base_id or "", query, top_k=int( knowledge_config.get( "top_n", knowledge_config.get("topN", 5), ) ), score_threshold=float( knowledge_config.get( "score_threshold", knowledge_config.get("scoreThreshold", 0.0), ) ), ) await params.result_callback( {"status": "ok", "results": results} ) except Exception as exc: # noqa: BLE001 - return tool errors to LLM logger.warning(f"文本测试知识库检索失败: {exc}") await params.result_callback( {"status": "error", "message": "知识库检索暂时不可用"} ) llm.register_function("search_knowledge_base", search_bound_knowledge) def set_tools(schemas=None) -> None: visible = list(schemas or []) if knowledge_schema is not None: visible.append(knowledge_schema) if visible: context.set_tools(ToolsSchema(standard_tools=visible)) else: context.set_tools() await brain.setup( cfg, BrainRuntime( context=context, llm=llm, queue_frame=worker.queue_frame, set_system_prompt=set_system_prompt, set_tools=set_tools, call_end=call_end, session_id=f"test_{uuid4().hex}", client_tools=TextClientToolPort(), worker=worker, context_aggregator=aggregators, transport=None, switch_services=switch_services, set_knowledge_scope=knowledge.set_scope, set_vision_scope=lambda _scope: None, vision_function=None, set_input_enabled=lambda _enabled: None, apply_turn_config=lambda _enabled, _config: asyncio.sleep(0), flow_global_functions=[], tool_executor_factory=create_test_executor, ), ) executor = executor_holder["executor"] @assistant_aggregator.event_handler("on_assistant_turn_started") async def on_assistant_turn_started(_aggregator): capture.window.active_assistant += 1 capture.window.touch() await brain.on_assistant_text_start(uuid4().hex) @assistant_aggregator.event_handler("on_assistant_turn_stopped") async def on_assistant_turn_stopped(_aggregator, message): content = str(message.content or "").strip() capture.window.active_assistant = max( 0, capture.window.active_assistant - 1 ) if content: capture.outputs.append(content) capture.window.touch() await brain.on_assistant_text_end( uuid4().hex, content, bool(getattr(message, "interrupted", False)), ) if call_end.ending: # Text tests have no audio transport. A fully aggregated # response is therefore the equivalent of playback ending. await call_end.finish() all_tool_behaviors = [ behavior for turn in definition.turns for behavior in turn.behaviors if behavior.type == "tool_call" ] executor.set_turn_behaviors(all_tool_behaviors) runner_task: asyncio.Task[None] | None = None try: await brain.run_preflight() if executor.unmocked_calls: raise TestExecutionError( code="UNMOCKED_TOOL_CALL", message="启动阶段调用了未配置 Mock 的工具: " + "、".join(executor.unmocked_calls), stage="tool", ) await runner.add_workers(worker) runner_task = asyncio.create_task( runner.run(), name=f"text-test-{uuid4().hex[:12]}" ) await asyncio.wait_for( worker_started.wait(), timeout=WORKER_START_TIMEOUT_SECONDS ) marker = capture.window.activity_count await brain.on_connected(greeting_pending=False) await brain.on_client_ready() await _wait_for_quiescence( capture, runner_task, marker, require_activity=False, ) # Startup messages initialize the session but are not a scripted # user turn result in the fixed-input editor contract. capture.outputs.clear() capture.tool_calls.clear() results: list[RawTurnResult] = [] for index, turn in enumerate(definition.turns): if worker.has_finished(): raise TestExecutionError( code="PIPELINE_ENDED_EARLY", message=f"Pipeline 在第 {index + 1} 轮输入前已经结束", stage="pipeline", retryable=False, ) result = await self._run_turn( worker, runner_task, capture, executor, turn, index, ) results.append(result) transcript = [ { "role": str(message.get("role") or ""), "content": str(message.get("content") or ""), } for message in context.get_messages() if message.get("role") in {"user", "assistant"} and str(message.get("content") or "").strip() ] return TextCaseResult(turns=results, transcript=transcript) except TestExecutionError: raise except asyncio.CancelledError: raise except Exception as exc: raise TestExecutionError( code="PIPELINE_EXECUTION_FAILED", message=str(exc) or type(exc).__name__, stage="model", retryable=True, ) from exc finally: if not worker.has_finished(): await worker.cancel(reason=INTERNAL_CANCEL_REASON) if runner_task is not None: await asyncio.gather(runner_task, return_exceptions=True) async def _run_turn( self, worker: PipelineWorker, runner_task: asyncio.Task[None], capture: _TextCaptureProcessor, executor: MockToolExecutor, turn: FixedInputTurn, index: int, ) -> RawTurnResult: tool_behaviors = [ behavior for behavior in turn.behaviors if behavior.type == "tool_call" ] executor.set_turn_behaviors(tool_behaviors) activity_marker = capture.window.activity_count output_marker = len(capture.outputs) tool_marker = len(capture.tool_calls) await worker.queue_frame( LLMMessagesAppendFrame( messages=[{"role": "user", "content": turn.user_input}], run_llm=True, ) ) await _wait_for_quiescence(capture, runner_task, activity_marker) if executor.unmocked_calls: raise TestExecutionError( code="UNMOCKED_TOOL_CALL", message="本轮调用了未配置 Mock 的工具: " + "、".join(executor.unmocked_calls), stage="tool", ) return RawTurnResult( id=turn.id, index=index, user_input=turn.user_input, assistant_reply="\n\n".join(capture.outputs[output_marker:]).strip(), tool_calls=list(capture.tool_calls[tool_marker:]), )