wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit for vad parameters on saaras:v3

Browse Source
This commit is contained in:
dhruvladia-sarvam
2026-04-20 13:52:48 +05:30
parent 6d3dfd8f64
commit f2a19cb1a3
2 changed files with 165 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pyproject.toml
View File

@@ -106,7 +106,7 @@ rime = [ "pipecat-ai[websockets-base]" ]
runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<1", "pipecat-ai-small-webrtc-prebuilt>=2.4.0"]
sagemaker = ["aws_sdk_sagemaker_runtime_http2; python_version>='3.12'"]
sambanova = []
sarvam = [ "sarvamai==0.1.26", "pipecat-ai[websockets-base]" ]
sarvam = [ "sarvamai==0.1.28", "pipecat-ai[websockets-base]" ]
sentry = [ "sentry-sdk>=2.28.0,<3" ]
silero = []
simli = [ "simli-ai~=2.0.1"]

167
src/pipecat/services/sarvam/stt.py
View File

@@ -91,6 +91,7 @@ class ModelConfig:
supports_prompt: Whether the model accepts prompt parameter.
supports_mode: Whether the model accepts mode parameter.
supports_language: Whether the model accepts language parameter.
supports_vad_params: Whether the model accepts fine-grained VAD parameters.
default_language: Default language code (None = auto-detect).
default_mode: Default mode (None = not applicable).
use_translate_endpoint: Whether to use speech_to_text_translate_streaming endpoint.
@@ -100,6 +101,7 @@ class ModelConfig:
supports_prompt: bool
supports_mode: bool
supports_language: bool
supports_vad_params: bool
default_language: str | None
default_mode: str | None
use_translate_endpoint: bool
@@ -111,6 +113,7 @@ MODEL_CONFIGS: dict[str, ModelConfig] = {
supports_prompt=False,
supports_mode=False,
supports_language=True,
supports_vad_params=False,
default_language="unknown",
default_mode=None,
use_translate_endpoint=False,
@@ -120,6 +123,7 @@ MODEL_CONFIGS: dict[str, ModelConfig] = {
supports_prompt=True,
supports_mode=False,
supports_language=False,
supports_vad_params=False,
default_language=None, # Auto-detects language
default_mode=None,
use_translate_endpoint=True,
@@ -129,6 +133,7 @@ MODEL_CONFIGS: dict[str, ModelConfig] = {
supports_prompt=False,
supports_mode=True,
supports_language=True,
supports_vad_params=True,
default_language="unknown",
default_mode="transcribe",
use_translate_endpoint=False,
@@ -146,11 +151,43 @@ class SarvamSTTSettings(STTSettings):
Only applicable to models that support prompts (e.g., saaras:v2.5).
vad_signals: Enable VAD signals in response.
high_vad_sensitivity: Enable high VAD sensitivity.
positive_speech_threshold: VAD probability threshold (0.0-1.0) above which
a frame is considered speech. Only for saaras:v3.
negative_speech_threshold: VAD probability threshold (0.0-1.0) below which
a frame is considered silence. Only for saaras:v3.
min_speech_frames: Minimum consecutive speech frames to start a speech
segment. Only for saaras:v3.
first_turn_min_speech_frames: Minimum speech frames for the first user
turn. Only for saaras:v3.
negative_frames_count: Number of silence frames within the window to end
a speech segment. Only for saaras:v3.
negative_frames_window: Sliding window size (in frames) for counting
negative frames. Only for saaras:v3.
start_speech_volume_threshold: Volume level (dB) below which audio is
too quiet to be speech. Only for saaras:v3.
interrupt_min_speech_frames: Minimum speech frames to register a
barge-in/interruption. Only for saaras:v3.
pre_speech_pad_frames: Number of audio frames to prepend before detected
speech onset. Only for saaras:v3.
num_initial_ignored_frames: Number of leading audio frames to skip at
connection start. Only for saaras:v3.
"""
prompt: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
vad_signals: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
high_vad_sensitivity: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
positive_speech_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
negative_speech_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
min_speech_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
first_turn_min_speech_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
negative_frames_count: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
negative_frames_window: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
start_speech_volume_threshold: float | None | _NotGiven = field(
default_factory=lambda: NOT_GIVEN
)
interrupt_min_speech_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
pre_speech_pad_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
num_initial_ignored_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class SarvamSTTService(STTService):
@@ -190,7 +227,27 @@ class SarvamSTTService(STTService):
mode: Mode of operation for saaras:v3 models only. Options: transcribe, translate,
verbatim, translit, codemix. Defaults to "transcribe" for saaras:v3.
vad_signals: Enable VAD signals in response. Defaults to None.
high_vad_sensitivity: Enable high VAD (Voice Activity Detection) sensitivity. Defaults to None.
high_vad_sensitivity: Enable high VAD sensitivity. Defaults to None.
positive_speech_threshold: VAD probability threshold (0.0-1.0) above which
a frame is considered speech. Only for saaras:v3.
negative_speech_threshold: VAD probability threshold (0.0-1.0) below which
a frame is considered silence. Only for saaras:v3.
min_speech_frames: Minimum consecutive speech frames to start a segment.
Only for saaras:v3.
first_turn_min_speech_frames: Minimum speech frames for the first user
turn. Only for saaras:v3.
negative_frames_count: Silence frames within the window to end a segment.
Only for saaras:v3.
negative_frames_window: Sliding window size (in frames) for counting
negative frames. Only for saaras:v3.
start_speech_volume_threshold: Volume (dB) below which audio is too quiet.
Only for saaras:v3.
interrupt_min_speech_frames: Minimum speech frames for barge-in.
Only for saaras:v3.
pre_speech_pad_frames: Audio frames to prepend before speech onset.
Only for saaras:v3.
num_initial_ignored_frames: Leading frames to skip at connection start.
Only for saaras:v3.
"""
language: Language | None = None
@@ -198,6 +255,16 @@ class SarvamSTTService(STTService):
mode: Literal["transcribe", "translate", "verbatim", "translit", "codemix"] | None = None
vad_signals: bool | None = None
high_vad_sensitivity: bool | None = None
positive_speech_threshold: float | None = None
negative_speech_threshold: float | None = None
min_speech_frames: int | None = None
first_turn_min_speech_frames: int | None = None
negative_frames_count: int | None = None
negative_frames_window: int | None = None
start_speech_volume_threshold: float | None = None
interrupt_min_speech_frames: int | None = None
pre_speech_pad_frames: int | None = None
num_initial_ignored_frames: int | None = None
def __init__(
self,
@@ -249,6 +316,16 @@ class SarvamSTTService(STTService):
prompt=None,
vad_signals=None,
high_vad_sensitivity=None,
positive_speech_threshold=None,
negative_speech_threshold=None,
min_speech_frames=None,
first_turn_min_speech_frames=None,
negative_frames_count=None,
negative_frames_window=None,
start_speech_volume_threshold=None,
interrupt_min_speech_frames=None,
pre_speech_pad_frames=None,
num_initial_ignored_frames=None,
)
# --- 2. Deprecated direct-arg overrides ---
@@ -266,6 +343,18 @@ class SarvamSTTService(STTService):
mode = params.mode
default_settings.vad_signals = params.vad_signals
default_settings.high_vad_sensitivity = params.high_vad_sensitivity
default_settings.positive_speech_threshold = params.positive_speech_threshold
default_settings.negative_speech_threshold = params.negative_speech_threshold
default_settings.min_speech_frames = params.min_speech_frames
default_settings.first_turn_min_speech_frames = params.first_turn_min_speech_frames
default_settings.negative_frames_count = params.negative_frames_count
default_settings.negative_frames_window = params.negative_frames_window
default_settings.start_speech_volume_threshold = (
params.start_speech_volume_threshold
)
default_settings.interrupt_min_speech_frames = params.interrupt_min_speech_frames
default_settings.pre_speech_pad_frames = params.pre_speech_pad_frames
default_settings.num_initial_ignored_frames = params.num_initial_ignored_frames
# --- 4. Settings delta (canonical API, always wins) ---
if settings is not None:
@@ -289,6 +378,26 @@ class SarvamSTTService(STTService):
f"Model '{resolved_model}' does not support language parameter (auto-detects language)."
)
if not self._config.supports_vad_params:
vad_param_names = [
"positive_speech_threshold",
"negative_speech_threshold",
"min_speech_frames",
"first_turn_min_speech_frames",
"negative_frames_count",
"negative_frames_window",
"start_speech_volume_threshold",
"interrupt_min_speech_frames",
"pre_speech_pad_frames",
"num_initial_ignored_frames",
]
for param_name in vad_param_names:
if getattr(default_settings, param_name) is not None:
raise ValueError(
f"Model '{resolved_model}' does not support {param_name} parameter. "
f"Fine-grained VAD parameters are only supported by saaras:v3."
)
# Resolve mode default from model config
if mode is None:
mode = self._config.default_mode
@@ -393,10 +502,44 @@ class SarvamSTTService(STTService):
f"Model '{self._settings.model}' does not support prompt parameter."
)
if isinstance(delta, self.Settings) and not self._config.supports_vad_params:
vad_param_names = [
"positive_speech_threshold",
"negative_speech_threshold",
"min_speech_frames",
"first_turn_min_speech_frames",
"negative_frames_count",
"negative_frames_window",
"start_speech_volume_threshold",
"interrupt_min_speech_frames",
"pre_speech_pad_frames",
"num_initial_ignored_frames",
]
for param_name in vad_param_names:
val = getattr(delta, param_name, NOT_GIVEN)
if is_given(val) and val is not None:
raise ValueError(
f"Model '{self._settings.model}' does not support {param_name} "
f"parameter. Fine-grained VAD parameters are only supported by saaras:v3."
)
changed = await super()._update_settings(delta)
# Language and prompt are WebSocket connect-time parameters; reconnect to apply.
reconnect_fields = {"language", "prompt"}
# These are all WebSocket connect-time parameters; reconnect to apply.
reconnect_fields = {
"language",
"prompt",
"positive_speech_threshold",
"negative_speech_threshold",
"min_speech_frames",
"first_turn_min_speech_frames",
"negative_frames_count",
"negative_frames_window",
"start_speech_volume_threshold",
"interrupt_min_speech_frames",
"pre_speech_pad_frames",
"num_initial_ignored_frames",
}
if changed.keys() & reconnect_fields:
await self._disconnect()
await self._connect()
@@ -535,6 +678,24 @@ class SarvamSTTService(STTService):
"true" if self._settings.high_vad_sensitivity else "false"
)
# Fine-grained VAD parameters (saaras:v3 only, sent as strings per SDK spec)
if self._config.supports_vad_params:
_vad_params = {
"positive_speech_threshold": self._settings.positive_speech_threshold,
"negative_speech_threshold": self._settings.negative_speech_threshold,
"min_speech_frames": self._settings.min_speech_frames,
"first_turn_min_speech_frames": self._settings.first_turn_min_speech_frames,
"negative_frames_count": self._settings.negative_frames_count,
"negative_frames_window": self._settings.negative_frames_window,
"start_speech_volume_threshold": self._settings.start_speech_volume_threshold,
"interrupt_min_speech_frames": self._settings.interrupt_min_speech_frames,
"pre_speech_pad_frames": self._settings.pre_speech_pad_frames,
"num_initial_ignored_frames": self._settings.num_initial_ignored_frames,
}
for k, v in _vad_params.items():
if v is not None:
connect_kwargs[k] = str(v)
# Add language_code for models that support it
language_string = self._get_language_string()
if language_string is not None: