2c2af1f2cd
- Add a new Docker configuration for the UI in launch.json to facilitate development. - Refactor pipeline.py to integrate a TranscriptProcessor for managing user and assistant transcripts, including event handlers for real-time updates and message handling. - Update useVoicePreview.ts to establish a data channel for sending and receiving text messages, improving interaction flow. - Modify AssistantPage.tsx to support displaying chat messages and sending user input, enhancing the user experience during voice interactions. - Revise DebugTranscriptPanel to dynamically render chat messages with timestamps, improving the visual representation of conversation history.
115 lines
4.0 KiB
Python
115 lines
4.0 KiB
Python
"""管线核心:给定一个 transport + 配置,跑完整的语音闭环。
|
|
|
|
关键设计:**transport 由调用方传入**,管线本身不关心是 WebRTC 还是 WS。
|
|
这就是"同时支持多种输出"的落点——加输出方式不用动这里。
|
|
|
|
对应 dograh 的 pipeline_builder.py + run_pipeline.py(已砍掉 workflow 引擎/DB/录音/指标)。
|
|
"""
|
|
|
|
from loguru import logger
|
|
from models import AssistantConfig
|
|
from services.pipecat.service_factory import create_services
|
|
|
|
from pipecat.frames.frames import (
|
|
EndFrame,
|
|
InterruptionTaskFrame,
|
|
TranscriptionFrame,
|
|
TransportMessageUrgentFrame,
|
|
TTSSpeakFrame,
|
|
)
|
|
from pipecat.pipeline.pipeline import Pipeline
|
|
from pipecat.pipeline.runner import PipelineRunner
|
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
|
from pipecat.processors.transcript_processor import TranscriptProcessor
|
|
from pipecat.utils.time import time_now_iso8601
|
|
|
|
|
|
async def run_pipeline(transport, cfg: AssistantConfig) -> None:
|
|
"""在给定 transport 上构建并运行管线,直到连接结束。
|
|
|
|
Args:
|
|
transport: 任意 pipecat transport(WebRTC / WS / 电话…),
|
|
只要有 .input() / .output() / event_handler 即可。
|
|
cfg: 助手配置(随请求内联传入)。
|
|
"""
|
|
logger.info(f"启动管线: assistant={cfg.name} mode={cfg.runtimeMode}")
|
|
|
|
stt, llm, tts = create_services(cfg)
|
|
|
|
context = OpenAILLMContext(messages=[{"role": "system", "content": cfg.prompt}])
|
|
context_aggregator = llm.create_context_aggregator(context)
|
|
|
|
# 转写收集:user 侧收 ASR 最终转写,assistant 侧聚合 TTS 实际播报的文本,
|
|
# 统一通过 data channel 推给前端聊天记录面板。
|
|
transcript = TranscriptProcessor()
|
|
|
|
pipeline = Pipeline(
|
|
[
|
|
transport.input(),
|
|
stt,
|
|
transcript.user(),
|
|
context_aggregator.user(),
|
|
llm,
|
|
tts,
|
|
transport.output(),
|
|
transcript.assistant(),
|
|
context_aggregator.assistant(),
|
|
]
|
|
)
|
|
|
|
task = PipelineTask(
|
|
pipeline,
|
|
params=PipelineParams(
|
|
allow_interruptions=cfg.enableInterrupt,
|
|
enable_metrics=False,
|
|
),
|
|
)
|
|
|
|
@transcript.event_handler("on_transcript_update")
|
|
async def on_transcript_update(_processor, frame):
|
|
# 每条最终转写(用户/助手)推给前端,前端据此渲染聊天记录
|
|
for msg in frame.messages:
|
|
await task.queue_frame(
|
|
TransportMessageUrgentFrame(
|
|
message={
|
|
"type": "transcript",
|
|
"role": msg.role,
|
|
"content": msg.content,
|
|
"timestamp": msg.timestamp,
|
|
}
|
|
)
|
|
)
|
|
|
|
@transport.event_handler("on_app_message")
|
|
async def on_app_message(_transport, message, _sender):
|
|
# 前端文字输入:先打断当前播报,再当作一条用户最终转写注入,
|
|
# 走与语音完全相同的 转写→上下文→LLM→TTS 链路
|
|
if not isinstance(message, dict) or message.get("type") != "user-text":
|
|
return
|
|
text = str(message.get("text") or "").strip()
|
|
if not text:
|
|
return
|
|
await task.queue_frames(
|
|
[
|
|
InterruptionTaskFrame(),
|
|
TranscriptionFrame(
|
|
text=text, user_id="debug", timestamp=time_now_iso8601()
|
|
),
|
|
]
|
|
)
|
|
|
|
@transport.event_handler("on_client_connected")
|
|
async def on_client_connected(_transport, _client):
|
|
if cfg.greeting:
|
|
await task.queue_frame(TTSSpeakFrame(cfg.greeting))
|
|
|
|
@transport.event_handler("on_client_disconnected")
|
|
async def on_client_disconnected(_transport, _client):
|
|
logger.info("对端断开,结束管线")
|
|
await task.queue_frame(EndFrame())
|
|
|
|
runner = PipelineRunner(handle_sigint=False)
|
|
await runner.run(task)
|
|
logger.info("管线已结束")
|