From acfb07f85963451a1f4d75b4e4fb1ab12d6ba1d5 Mon Sep 17 00:00:00 2001 From: zack Date: Thu, 5 Mar 2026 14:52:50 -0500 Subject: [PATCH] feat(assemblyai): add vad_threshold parameter for U3 Pro Add vad_threshold parameter to AssemblyAIConnectionParams to support voice activity detection threshold configuration for the u3-rt-pro model. This parameter allows users to align AssemblyAI's VAD threshold with their external VAD systems (e.g., Silero VAD) to avoid the "dead zone" where AssemblyAI transcribes speech that the external VAD hasn't detected yet, which can delay interruption handling. - Range: 0.0 to 1.0 (lower = more sensitive) - Default: 0.3 (API default when not sent) - Only applicable to u3-rt-pro model - Automatically included in WebSocket query parameters Recommended usage: Set vad_threshold to match your VAD's activation threshold (e.g., both at 0.3) for optimal performance. --- src/pipecat/services/assemblyai/models.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/pipecat/services/assemblyai/models.py b/src/pipecat/services/assemblyai/models.py index 3a022dce1..2c6b3a1dc 100644 --- a/src/pipecat/services/assemblyai/models.py +++ b/src/pipecat/services/assemblyai/models.py @@ -127,8 +127,6 @@ class AssemblyAIConnectionParams(BaseModel): Parameters: sample_rate: Audio sample rate in Hz. Defaults to 16000. encoding: Audio encoding format. Defaults to "pcm_s16le". - formatted_finals: Whether to enable transcript formatting. Defaults to True. - word_finalization_max_wait_time: Maximum time to wait for word finalization in milliseconds. end_of_turn_confidence_threshold: Confidence threshold for end-of-turn detection. min_turn_silence: Minimum silence duration when confident about end-of-turn. min_end_of_turn_silence_when_confident: DEPRECATED. Use min_turn_silence instead. @@ -139,16 +137,24 @@ class AssemblyAIConnectionParams(BaseModel): language_detection: Enable automatic language detection. Only applicable to universal-streaming-multilingual. When enabled, Turn messages include language_code and language_confidence fields. Defaults to None (not sent). - format_turns: Whether to format transcript turns. Defaults to True. + format_turns: Whether to format transcript turns. Only applicable to + universal-streaming-english and universal-streaming-multilingual models. + For u3-rt-pro, formatting is automatic and built-in. Defaults to True. speaker_labels: Enable speaker diarization. When enabled, final transcripts (end_of_turn=True) include a speaker field identifying the speaker (e.g., "Speaker A", "Speaker B"). Defaults to None (not sent). + vad_threshold: Voice activity detection confidence threshold. Only applicable to + u3-rt-pro. The confidence threshold (0.0 to 1.0) for classifying audio frames + as silence. Frames with VAD confidence below this value are considered silent. + Increase for noisy environments to reduce false speech detection. Defaults to + 0.3 (API default). For best performance when using with external VAD (e.g., Silero), + align this value with your VAD's activation threshold to avoid the "dead zone" + where AssemblyAI transcribes speech that your VAD hasn't detected yet. + Defaults to None (not sent). """ sample_rate: int = 16000 encoding: Literal["pcm_s16le", "pcm_mulaw"] = "pcm_s16le" - formatted_finals: bool = True - word_finalization_max_wait_time: Optional[int] = None end_of_turn_confidence_threshold: Optional[float] = None min_turn_silence: Optional[int] = None min_end_of_turn_silence_when_confident: Optional[int] = None # Deprecated @@ -161,6 +167,7 @@ class AssemblyAIConnectionParams(BaseModel): language_detection: Optional[bool] = None format_turns: bool = True speaker_labels: Optional[bool] = None + vad_threshold: Optional[float] = None @model_validator(mode="after") def handle_deprecated_param(self):