From 872b8ac64a64ee460a191761c847342d5a46e8a5 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Thu, 21 May 2026 17:23:04 +0800 Subject: [PATCH] move vad config to json --- README.md | 35 +++++++++++++++++++++++++++++++++ config.json | 9 +++++++++ config/siliconflow.json | 9 +++++++++ config/xfyun.json | 9 +++++++++ engine/config.py | 43 +++++++++++++++++++++++++++++++++++++++++ engine/pipeline.py | 31 ++++++++++++++++++++++++++++- 6 files changed, 135 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d88fae..f7644b9 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,41 @@ the TTS in the pipeline. `response.text.final` fires when the turn ends, carrying the full concatenated assistant text and an `interrupted` flag (true when an `input.text` or barge-in cut the turn short). +### Turn detection + +User-turn segmentation (VAD thresholds + how long to wait after silence +before declaring the turn done) is configurable per environment: + +```json +"turn": { + "vad": { + "confidence": 0.7, + "start_secs": 0.2, + "stop_secs": 0.6, + "min_volume": 0.6 + }, + "user_speech_timeout_sec": 1.0 +} +``` + +- `vad.*` maps directly to `pipecat.audio.vad.vad_analyzer.VADParams` and + controls the Silero VAD. `stop_secs` is the duration of silence required + before VAD reports the user stopped speaking; raise it if VAD is + cutting users off mid-clause, lower it for snappier turn-taking. +- `user_speech_timeout_sec` is the additional grace window (used by + `SpeechTimeoutUserTurnStopStrategy`) during which the user may resume + speaking before the aggregator finalizes the turn. The timer is + re-armed every time the user resumes, so brief mid-sentence pauses do + not split one utterance into multiple LLM turns. + +The total "user pause before turn ends" budget is approximately +`vad.stop_secs + user_speech_timeout_sec`. The repo defaults are tuned +slightly more conservatively than upstream pipecat to avoid streaming +ASRs (xfyun in particular) producing many short fragments per logical +utterance. Setting this stop strategy explicitly also replaces pipecat's +default Smart Turn v3 analyzer, so the engine no longer loads the +`smart-turn-v3.*-cpu.onnx` model at startup. + ### Xfyun ASR The STT provider can be switched to iFlytek/Xfyun's streaming voice dictation diff --git a/config.json b/config.json index ef49a80..4a6ff7b 100644 --- a/config.json +++ b/config.json @@ -12,6 +12,15 @@ "session": { "inactivity_timeout_sec": 60 }, + "turn": { + "vad": { + "confidence": 0.7, + "start_secs": 0.2, + "stop_secs": 0.4, + "min_volume": 0.6 + }, + "user_speech_timeout_sec": 0.8 + }, "agent": { "system_prompt": "You are a helpful, friendly voice assistant. Keep responses concise and natural for spoken conversation.", "greeting": "Please introduce yourself briefly.", diff --git a/config/siliconflow.json b/config/siliconflow.json index 4a78706..4fe18a1 100644 --- a/config/siliconflow.json +++ b/config/siliconflow.json @@ -12,6 +12,15 @@ "session": { "inactivity_timeout_sec": 60 }, + "turn": { + "vad": { + "confidence": 0.7, + "start_secs": 0.2, + "stop_secs": 0.6, + "min_volume": 0.6 + }, + "user_speech_timeout_sec": 1.0 + }, "agent": { "system_prompt": "你是一个有用的语音对话助手名字叫小白", "greeting": "你好，我是小白，请问有什么可以帮你？", diff --git a/config/xfyun.json b/config/xfyun.json index 2d3eba0..6494bba 100644 --- a/config/xfyun.json +++ b/config/xfyun.json @@ -12,6 +12,15 @@ "session": { "inactivity_timeout_sec": 60 }, + "turn": { + "vad": { + "confidence": 0.7, + "start_secs": 0.2, + "stop_secs": 0.6, + "min_volume": 0.6 + }, + "user_speech_timeout_sec": 1.0 + }, "agent": { "system_prompt": "你是一个有用的语音对话助手名字叫小白", "greeting": "你好，我是小白，请问有什么可以帮你？", diff --git a/engine/config.py b/engine/config.py index 71d2160..d8cc655 100644 --- a/engine/config.py +++ b/engine/config.py @@ -28,6 +28,39 @@ class SessionConfig: inactivity_timeout_sec: int = 60 +@dataclass(frozen=True) +class VADConfig: + """Voice Activity Detection thresholds for the Silero analyzer. + + These map directly to ``pipecat.audio.vad.vad_analyzer.VADParams``. + Defaults are tuned a touch more conservative than upstream pipecat so + short pauses in continuous speech don't end the user turn prematurely. + """ + + confidence: float = 0.7 + start_secs: float = 0.2 + stop_secs: float = 0.6 + min_volume: float = 0.6 + + +@dataclass(frozen=True) +class TurnConfig: + """User-turn segmentation policy. + + ``user_speech_timeout_sec`` is the grace window (in seconds) after VAD + has confirmed silence during which the user is allowed to resume + speaking before the aggregator finalizes the turn. Used by + ``SpeechTimeoutUserTurnStopStrategy``. Higher = more tolerant of + natural mid-sentence pauses; lower = snappier turn-taking. + + The combined "user pause before turn ends" budget is roughly + ``vad.stop_secs + user_speech_timeout_sec``. + """ + + vad: VADConfig = field(default_factory=VADConfig) + user_speech_timeout_sec: float = 1.0 + + @dataclass(frozen=True) class AgentConfig: system_prompt: str = "You are a helpful, friendly voice assistant." @@ -90,6 +123,7 @@ class EngineConfig: server: ServerConfig = field(default_factory=ServerConfig) audio: AudioConfig = field(default_factory=AudioConfig) session: SessionConfig = field(default_factory=SessionConfig) + turn: TurnConfig = field(default_factory=TurnConfig) agent: AgentConfig = field(default_factory=AgentConfig) services: ServicesConfig = field(default_factory=ServicesConfig) @@ -116,10 +150,19 @@ def config_from_dict(data: dict) -> EngineConfig: if stt.get("language") == "": stt["language"] = None + turn = _dict(data.get("turn")) + vad = _dict(turn.get("vad")) + return EngineConfig( server=ServerConfig(**_dict(data.get("server"))), audio=AudioConfig(**_dict(data.get("audio"))), session=SessionConfig(**_dict(data.get("session"))), + turn=TurnConfig( + vad=VADConfig(**vad), + user_speech_timeout_sec=float( + turn.get("user_speech_timeout_sec", TurnConfig().user_speech_timeout_sec) + ), + ), agent=AgentConfig(**agent), services=ServicesConfig( llm=LLMConfig(**_dict(services.get("llm"))), diff --git a/engine/pipeline.py b/engine/pipeline.py index 4110ba3..517e54b 100644 --- a/engine/pipeline.py +++ b/engine/pipeline.py @@ -3,6 +3,7 @@ from __future__ import annotations from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import ( LLMRunFrame, OutputTransportMessageUrgentFrame, @@ -24,6 +25,10 @@ from pipecat.transports.websocket.fastapi import ( FastAPIWebsocketParams, FastAPIWebsocketTransport, ) +from pipecat.turns.user_stop.speech_timeout_user_turn_stop_strategy import ( + SpeechTimeoutUserTurnStopStrategy, +) +from pipecat.turns.user_turn_strategies import UserTurnStrategies from .config import EngineConfig from .product_protocol import ProductWebsocketSerializer @@ -84,9 +89,33 @@ async def run_pipeline_with_serializer( messages.append({"role": "system", "content": config.agent.greeting}) context = LLMContext(messages) + + vad_params = VADParams( + confidence=config.turn.vad.confidence, + start_secs=config.turn.vad.start_secs, + stop_secs=config.turn.vad.stop_secs, + min_volume=config.turn.vad.min_volume, + ) + # Replace pipecat's default stop strategy (Smart Turn v3) with a simple + # silence-timeout strategy. Smart Turn v3 was finalizing every short + # Chinese phrase as a complete turn, which caused one logical utterance + # to become several LLM calls and several user bubbles in the UI. The + # timeout strategy waits for `user_speech_timeout_sec` of silence + # (re-armed every time the user resumes speaking) before declaring the + # turn finished — which is what we actually want for streaming ASRs. + user_turn_strategies = UserTurnStrategies( + stop=[ + SpeechTimeoutUserTurnStopStrategy( + user_speech_timeout=config.turn.user_speech_timeout_sec, + ), + ], + ) user_aggregator, assistant_aggregator = LLMContextAggregatorPair( context, - user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()), + user_params=LLMUserAggregatorParams( + vad_analyzer=SileroVADAnalyzer(params=vad_params), + user_turn_strategies=user_turn_strategies, + ), ) pipeline = Pipeline(