Add support for Xfyun ASR and TTS services in the backend
- Introduce new Xfyun ASR and TTS services, enabling integration with iFlytek's voice recognition and synthesis capabilities. - Update AssistantConfig model to include interface types for STT and TTS. - Enhance credential testing to validate Xfyun credentials. - Modify service factory to create Xfyun services based on configuration. - Update README with new configuration details for Xfyun integration. - Add new frontend components for visualizing audio streams and managing user interactions.
This commit is contained in:
Xin Wang
@@ -10,19 +10,84 @@ from loguru import logger
|
||||
from models import AssistantConfig
|
||||
from services.pipecat.service_factory import create_services
|
||||
|
||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||
from pipecat.frames.frames import (
|
||||
EndFrame,
|
||||
InterruptionTaskFrame,
|
||||
TranscriptionFrame,
|
||||
TransportMessageUrgentFrame,
|
||||
InputTextRawFrame,
|
||||
InputTransportMessageFrame,
|
||||
LLMMessagesAppendFrame,
|
||||
OutputTransportMessageUrgentFrame,
|
||||
TTSSpeakFrame,
|
||||
)
|
||||
from pipecat.pipeline.pipeline import Pipeline
|
||||
from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||
from pipecat.processors.transcript_processor import TranscriptProcessor
|
||||
from pipecat.pipeline.worker import PipelineParams, PipelineWorker
|
||||
from pipecat.processors.aggregators.llm_context import LLMContext
|
||||
from pipecat.processors.aggregators.llm_response_universal import (
|
||||
LLMContextAggregatorPair,
|
||||
LLMUserAggregatorParams,
|
||||
)
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
from pipecat.turns.user_start import (
|
||||
TranscriptionUserTurnStartStrategy,
|
||||
VADUserTurnStartStrategy,
|
||||
)
|
||||
from pipecat.turns.user_turn_strategies import UserTurnStrategies
|
||||
from pipecat.utils.time import time_now_iso8601
|
||||
from pipecat.workers.runner import WorkerRunner
|
||||
|
||||
|
||||
def _text_input(message) -> tuple[str, bool] | None:
|
||||
"""解析现有 user-text 与 RTVI send-text 两种前端文字消息。"""
|
||||
if not isinstance(message, dict):
|
||||
return None
|
||||
if message.get("type") == "user-text":
|
||||
text = str(message.get("text") or "").strip()
|
||||
return (text, True) if text else None
|
||||
if message.get("type") == "send-text":
|
||||
data = message.get("data")
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
text = str(data.get("content") or "").strip()
|
||||
options = data.get("options")
|
||||
run_immediately = not isinstance(options, dict) or options.get(
|
||||
"run_immediately", True
|
||||
)
|
||||
return (text, bool(run_immediately)) if text else None
|
||||
return None
|
||||
|
||||
|
||||
class TextInputProcessor(FrameProcessor):
|
||||
"""把 transport 文字消息转换成级联与实时 LLM 都能消费的帧。"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._register_event_handler("on_text_input")
|
||||
|
||||
async def process_frame(self, frame, direction: FrameDirection):
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
if not isinstance(frame, InputTransportMessageFrame):
|
||||
await self.push_frame(frame, direction)
|
||||
return
|
||||
|
||||
parsed = _text_input(frame.message)
|
||||
if not parsed:
|
||||
await self.push_frame(frame, direction)
|
||||
return
|
||||
|
||||
text, run_immediately = parsed
|
||||
if run_immediately:
|
||||
await self.broadcast_interruption()
|
||||
|
||||
await self.push_frame(
|
||||
LLMMessagesAppendFrame(
|
||||
messages=[{"role": "user", "content": text}],
|
||||
run_llm=run_immediately,
|
||||
)
|
||||
)
|
||||
if run_immediately:
|
||||
await self.push_frame(InputTextRawFrame(text=text))
|
||||
await self._call_event_handler("on_text_input", text)
|
||||
|
||||
|
||||
async def run_pipeline(transport, cfg: AssistantConfig) -> None:
|
||||
@@ -37,78 +102,80 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
|
||||
|
||||
stt, llm, tts = create_services(cfg)
|
||||
|
||||
context = OpenAILLMContext(messages=[{"role": "system", "content": cfg.prompt}])
|
||||
context_aggregator = llm.create_context_aggregator(context)
|
||||
|
||||
# 转写收集:user 侧收 ASR 最终转写,assistant 侧聚合 TTS 实际播报的文本,
|
||||
# 统一通过 data channel 推给前端聊天记录面板。
|
||||
transcript = TranscriptProcessor()
|
||||
context = LLMContext(messages=[{"role": "system", "content": cfg.prompt}])
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
vad_analyzer=SileroVADAnalyzer(),
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
start=[
|
||||
VADUserTurnStartStrategy(enable_interruptions=cfg.enableInterrupt),
|
||||
TranscriptionUserTurnStartStrategy(
|
||||
enable_interruptions=cfg.enableInterrupt
|
||||
),
|
||||
]
|
||||
),
|
||||
),
|
||||
)
|
||||
text_input = TextInputProcessor()
|
||||
|
||||
pipeline = Pipeline(
|
||||
[
|
||||
transport.input(),
|
||||
text_input,
|
||||
stt,
|
||||
transcript.user(),
|
||||
context_aggregator.user(),
|
||||
user_aggregator,
|
||||
llm,
|
||||
tts,
|
||||
transport.output(),
|
||||
transcript.assistant(),
|
||||
context_aggregator.assistant(),
|
||||
assistant_aggregator,
|
||||
]
|
||||
)
|
||||
|
||||
task = PipelineTask(
|
||||
worker = PipelineWorker(
|
||||
pipeline,
|
||||
params=PipelineParams(
|
||||
allow_interruptions=cfg.enableInterrupt,
|
||||
enable_metrics=False,
|
||||
),
|
||||
enable_rtvi=False,
|
||||
)
|
||||
|
||||
@transcript.event_handler("on_transcript_update")
|
||||
async def on_transcript_update(_processor, frame):
|
||||
# 每条最终转写(用户/助手)推给前端,前端据此渲染聊天记录
|
||||
for msg in frame.messages:
|
||||
await task.queue_frame(
|
||||
TransportMessageUrgentFrame(
|
||||
async def queue_transcript(role: str, content: str, timestamp: str) -> None:
|
||||
if content:
|
||||
await worker.queue_frame(
|
||||
OutputTransportMessageUrgentFrame(
|
||||
message={
|
||||
"type": "transcript",
|
||||
"role": msg.role,
|
||||
"content": msg.content,
|
||||
"timestamp": msg.timestamp,
|
||||
}
|
||||
"role": role,
|
||||
"content": content,
|
||||
"timestamp": timestamp,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
@transport.event_handler("on_app_message")
|
||||
async def on_app_message(_transport, message, _sender):
|
||||
# 前端文字输入:先打断当前播报,再当作一条用户最终转写注入,
|
||||
# 走与语音完全相同的 转写→上下文→LLM→TTS 链路
|
||||
if not isinstance(message, dict) or message.get("type") != "user-text":
|
||||
return
|
||||
text = str(message.get("text") or "").strip()
|
||||
if not text:
|
||||
return
|
||||
await task.queue_frames(
|
||||
[
|
||||
InterruptionTaskFrame(),
|
||||
TranscriptionFrame(
|
||||
text=text, user_id="debug", timestamp=time_now_iso8601()
|
||||
),
|
||||
]
|
||||
)
|
||||
@user_aggregator.event_handler("on_user_turn_stopped")
|
||||
async def on_user_turn_stopped(_aggregator, _strategy, message):
|
||||
await queue_transcript("user", message.content, message.timestamp)
|
||||
|
||||
@assistant_aggregator.event_handler("on_assistant_turn_stopped")
|
||||
async def on_assistant_turn_stopped(_aggregator, message):
|
||||
await queue_transcript("assistant", message.content, message.timestamp)
|
||||
|
||||
@text_input.event_handler("on_text_input")
|
||||
async def on_text_input(_processor, text):
|
||||
await queue_transcript("user", text, time_now_iso8601())
|
||||
|
||||
@transport.event_handler("on_client_connected")
|
||||
async def on_client_connected(_transport, _client):
|
||||
if cfg.greeting:
|
||||
await task.queue_frame(TTSSpeakFrame(cfg.greeting))
|
||||
await worker.queue_frame(TTSSpeakFrame(cfg.greeting))
|
||||
|
||||
@transport.event_handler("on_client_disconnected")
|
||||
async def on_client_disconnected(_transport, _client):
|
||||
logger.info("对端断开,结束管线")
|
||||
await task.queue_frame(EndFrame())
|
||||
await worker.queue_frame(EndFrame())
|
||||
|
||||
runner = PipelineRunner(handle_sigint=False)
|
||||
await runner.run(task)
|
||||
runner = WorkerRunner(handle_sigint=False)
|
||||
await runner.add_workers(worker)
|
||||
await runner.run()
|
||||
logger.info("管线已结束")
|
||||
|
||||
Reference in New Issue
Block a user