Initial commit for vad parameters on saaras:v3

pyproject.toml

@@ -106,7 +106,7 @@ rime = [ "pipecat-ai[websockets-base]" ]
 runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<1", "pipecat-ai-small-webrtc-prebuilt>=2.4.0"]
 sagemaker = ["aws_sdk_sagemaker_runtime_http2; python_version>='3.12'"]
 sambanova = []
-sarvam = [ "sarvamai==0.1.26", "pipecat-ai[websockets-base]" ]
+sarvam = [ "sarvamai==0.1.28", "pipecat-ai[websockets-base]" ]
 sentry = [ "sentry-sdk>=2.28.0,<3" ]
 silero = []
 simli = [ "simli-ai~=2.0.1"]

src/pipecat/services/sarvam/stt.py

@@ -91,6 +91,7 @@ class ModelConfig:
         supports_prompt: Whether the model accepts prompt parameter.
         supports_mode: Whether the model accepts mode parameter.
         supports_language: Whether the model accepts language parameter.
+        supports_vad_params: Whether the model accepts fine-grained VAD parameters.
         default_language: Default language code (None = auto-detect).
         default_mode: Default mode (None = not applicable).
         use_translate_endpoint: Whether to use speech_to_text_translate_streaming endpoint.
@@ -100,6 +101,7 @@ class ModelConfig:
     supports_prompt: bool
     supports_mode: bool
     supports_language: bool
+    supports_vad_params: bool
     default_language: str | None
     default_mode: str | None
     use_translate_endpoint: bool
@@ -111,6 +113,7 @@ MODEL_CONFIGS: dict[str, ModelConfig] = {
         supports_prompt=False,
         supports_mode=False,
         supports_language=True,
+        supports_vad_params=False,
         default_language="unknown",
         default_mode=None,
         use_translate_endpoint=False,
@@ -120,6 +123,7 @@ MODEL_CONFIGS: dict[str, ModelConfig] = {
         supports_prompt=True,
         supports_mode=False,
         supports_language=False,
+        supports_vad_params=False,
         default_language=None,  # Auto-detects language
         default_mode=None,
         use_translate_endpoint=True,
@@ -129,6 +133,7 @@ MODEL_CONFIGS: dict[str, ModelConfig] = {
         supports_prompt=False,
         supports_mode=True,
         supports_language=True,
+        supports_vad_params=True,
         default_language="unknown",
         default_mode="transcribe",
         use_translate_endpoint=False,
@@ -146,11 +151,43 @@ class SarvamSTTSettings(STTSettings):
             Only applicable to models that support prompts (e.g., saaras:v2.5).
         vad_signals: Enable VAD signals in response.
         high_vad_sensitivity: Enable high VAD sensitivity.
+        positive_speech_threshold: VAD probability threshold (0.0-1.0) above which
+            a frame is considered speech. Only for saaras:v3.
+        negative_speech_threshold: VAD probability threshold (0.0-1.0) below which
+            a frame is considered silence. Only for saaras:v3.
+        min_speech_frames: Minimum consecutive speech frames to start a speech
+            segment. Only for saaras:v3.
+        first_turn_min_speech_frames: Minimum speech frames for the first user
+            turn. Only for saaras:v3.
+        negative_frames_count: Number of silence frames within the window to end
+            a speech segment. Only for saaras:v3.
+        negative_frames_window: Sliding window size (in frames) for counting
+            negative frames. Only for saaras:v3.
+        start_speech_volume_threshold: Volume level (dB) below which audio is
+            too quiet to be speech. Only for saaras:v3.
+        interrupt_min_speech_frames: Minimum speech frames to register a
+            barge-in/interruption. Only for saaras:v3.
+        pre_speech_pad_frames: Number of audio frames to prepend before detected
+            speech onset. Only for saaras:v3.
+        num_initial_ignored_frames: Number of leading audio frames to skip at
+            connection start. Only for saaras:v3.
     """
 
     prompt: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
     vad_signals: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
     high_vad_sensitivity: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    positive_speech_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    negative_speech_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    min_speech_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    first_turn_min_speech_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    negative_frames_count: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    negative_frames_window: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    start_speech_volume_threshold: float | None | _NotGiven = field(
+        default_factory=lambda: NOT_GIVEN
+    )
+    interrupt_min_speech_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    pre_speech_pad_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    num_initial_ignored_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
 
 
 class SarvamSTTService(STTService):
@@ -190,7 +227,27 @@ class SarvamSTTService(STTService):
             mode: Mode of operation for saaras:v3 models only. Options: transcribe, translate,
                 verbatim, translit, codemix. Defaults to "transcribe" for saaras:v3.
             vad_signals: Enable VAD signals in response. Defaults to None.
-            high_vad_sensitivity: Enable high VAD (Voice Activity Detection) sensitivity. Defaults to None.
+            high_vad_sensitivity: Enable high VAD sensitivity. Defaults to None.
+            positive_speech_threshold: VAD probability threshold (0.0-1.0) above which
+                a frame is considered speech. Only for saaras:v3.
+            negative_speech_threshold: VAD probability threshold (0.0-1.0) below which
+                a frame is considered silence. Only for saaras:v3.
+            min_speech_frames: Minimum consecutive speech frames to start a segment.
+                Only for saaras:v3.
+            first_turn_min_speech_frames: Minimum speech frames for the first user
+                turn. Only for saaras:v3.
+            negative_frames_count: Silence frames within the window to end a segment.
+                Only for saaras:v3.
+            negative_frames_window: Sliding window size (in frames) for counting
+                negative frames. Only for saaras:v3.
+            start_speech_volume_threshold: Volume (dB) below which audio is too quiet.
+                Only for saaras:v3.
+            interrupt_min_speech_frames: Minimum speech frames for barge-in.
+                Only for saaras:v3.
+            pre_speech_pad_frames: Audio frames to prepend before speech onset.
+                Only for saaras:v3.
+            num_initial_ignored_frames: Leading frames to skip at connection start.
+                Only for saaras:v3.
         """
 
         language: Language | None = None
@@ -198,6 +255,16 @@ class SarvamSTTService(STTService):
         mode: Literal["transcribe", "translate", "verbatim", "translit", "codemix"] | None = None
         vad_signals: bool | None = None
         high_vad_sensitivity: bool | None = None
+        positive_speech_threshold: float | None = None
+        negative_speech_threshold: float | None = None
+        min_speech_frames: int | None = None
+        first_turn_min_speech_frames: int | None = None
+        negative_frames_count: int | None = None
+        negative_frames_window: int | None = None
+        start_speech_volume_threshold: float | None = None
+        interrupt_min_speech_frames: int | None = None
+        pre_speech_pad_frames: int | None = None
+        num_initial_ignored_frames: int | None = None
 
     def __init__(
         self,
@@ -249,6 +316,16 @@ class SarvamSTTService(STTService):
             prompt=None,
             vad_signals=None,
             high_vad_sensitivity=None,
+            positive_speech_threshold=None,
+            negative_speech_threshold=None,
+            min_speech_frames=None,
+            first_turn_min_speech_frames=None,
+            negative_frames_count=None,
+            negative_frames_window=None,
+            start_speech_volume_threshold=None,
+            interrupt_min_speech_frames=None,
+            pre_speech_pad_frames=None,
+            num_initial_ignored_frames=None,
         )
 
         # --- 2. Deprecated direct-arg overrides ---
@@ -266,6 +343,18 @@ class SarvamSTTService(STTService):
                     mode = params.mode
                 default_settings.vad_signals = params.vad_signals
                 default_settings.high_vad_sensitivity = params.high_vad_sensitivity
+                default_settings.positive_speech_threshold = params.positive_speech_threshold
+                default_settings.negative_speech_threshold = params.negative_speech_threshold
+                default_settings.min_speech_frames = params.min_speech_frames
+                default_settings.first_turn_min_speech_frames = params.first_turn_min_speech_frames
+                default_settings.negative_frames_count = params.negative_frames_count
+                default_settings.negative_frames_window = params.negative_frames_window
+                default_settings.start_speech_volume_threshold = (
+                    params.start_speech_volume_threshold
+                )
+                default_settings.interrupt_min_speech_frames = params.interrupt_min_speech_frames
+                default_settings.pre_speech_pad_frames = params.pre_speech_pad_frames
+                default_settings.num_initial_ignored_frames = params.num_initial_ignored_frames
 
         # --- 4. Settings delta (canonical API, always wins) ---
         if settings is not None:
@@ -289,6 +378,26 @@ class SarvamSTTService(STTService):
                 f"Model '{resolved_model}' does not support language parameter (auto-detects language)."
             )
 
+        if not self._config.supports_vad_params:
+            vad_param_names = [
+                "positive_speech_threshold",
+                "negative_speech_threshold",
+                "min_speech_frames",
+                "first_turn_min_speech_frames",
+                "negative_frames_count",
+                "negative_frames_window",
+                "start_speech_volume_threshold",
+                "interrupt_min_speech_frames",
+                "pre_speech_pad_frames",
+                "num_initial_ignored_frames",
+            ]
+            for param_name in vad_param_names:
+                if getattr(default_settings, param_name) is not None:
+                    raise ValueError(
+                        f"Model '{resolved_model}' does not support {param_name} parameter. "
+                        f"Fine-grained VAD parameters are only supported by saaras:v3."
+                    )
+
         # Resolve mode default from model config
         if mode is None:
             mode = self._config.default_mode
@@ -393,10 +502,44 @@ class SarvamSTTService(STTService):
                     f"Model '{self._settings.model}' does not support prompt parameter."
                 )
 
+        if isinstance(delta, self.Settings) and not self._config.supports_vad_params:
+            vad_param_names = [
+                "positive_speech_threshold",
+                "negative_speech_threshold",
+                "min_speech_frames",
+                "first_turn_min_speech_frames",
+                "negative_frames_count",
+                "negative_frames_window",
+                "start_speech_volume_threshold",
+                "interrupt_min_speech_frames",
+                "pre_speech_pad_frames",
+                "num_initial_ignored_frames",
+            ]
+            for param_name in vad_param_names:
+                val = getattr(delta, param_name, NOT_GIVEN)
+                if is_given(val) and val is not None:
+                    raise ValueError(
+                        f"Model '{self._settings.model}' does not support {param_name} "
+                        f"parameter. Fine-grained VAD parameters are only supported by saaras:v3."
+                    )
+
         changed = await super()._update_settings(delta)
 
-        # Language and prompt are WebSocket connect-time parameters; reconnect to apply.
+        # These are all WebSocket connect-time parameters; reconnect to apply.
-        reconnect_fields = {"language", "prompt"}
+        reconnect_fields = {
+            "language",
+            "prompt",
+            "positive_speech_threshold",
+            "negative_speech_threshold",
+            "min_speech_frames",
+            "first_turn_min_speech_frames",
+            "negative_frames_count",
+            "negative_frames_window",
+            "start_speech_volume_threshold",
+            "interrupt_min_speech_frames",
+            "pre_speech_pad_frames",
+            "num_initial_ignored_frames",
+        }
         if changed.keys() & reconnect_fields:
             await self._disconnect()
             await self._connect()
@@ -535,6 +678,24 @@ class SarvamSTTService(STTService):
                     "true" if self._settings.high_vad_sensitivity else "false"
                 )
 
+            # Fine-grained VAD parameters (saaras:v3 only, sent as strings per SDK spec)
+            if self._config.supports_vad_params:
+                _vad_params = {
+                    "positive_speech_threshold": self._settings.positive_speech_threshold,
+                    "negative_speech_threshold": self._settings.negative_speech_threshold,
+                    "min_speech_frames": self._settings.min_speech_frames,
+                    "first_turn_min_speech_frames": self._settings.first_turn_min_speech_frames,
+                    "negative_frames_count": self._settings.negative_frames_count,
+                    "negative_frames_window": self._settings.negative_frames_window,
+                    "start_speech_volume_threshold": self._settings.start_speech_volume_threshold,
+                    "interrupt_min_speech_frames": self._settings.interrupt_min_speech_frames,
+                    "pre_speech_pad_frames": self._settings.pre_speech_pad_frames,
+                    "num_initial_ignored_frames": self._settings.num_initial_ignored_frames,
+                }
+                for k, v in _vad_params.items():
+                    if v is not None:
+                        connect_kwargs[k] = str(v)
+
             # Add language_code for models that support it
             language_string = self._get_language_string()
             if language_string is not None: