diff --git a/pyproject.toml b/pyproject.toml index fca7db757..b1ff2cbd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -106,7 +106,7 @@ rime = [ "pipecat-ai[websockets-base]" ] runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<1", "pipecat-ai-small-webrtc-prebuilt>=2.4.0"] sagemaker = ["aws_sdk_sagemaker_runtime_http2; python_version>='3.12'"] sambanova = [] -sarvam = [ "sarvamai==0.1.26", "pipecat-ai[websockets-base]" ] +sarvam = [ "sarvamai==0.1.28", "pipecat-ai[websockets-base]" ] sentry = [ "sentry-sdk>=2.28.0,<3" ] silero = [] simli = [ "simli-ai~=2.0.1"] diff --git a/src/pipecat/services/sarvam/stt.py b/src/pipecat/services/sarvam/stt.py index faadd914b..4ebe703fa 100644 --- a/src/pipecat/services/sarvam/stt.py +++ b/src/pipecat/services/sarvam/stt.py @@ -91,6 +91,7 @@ class ModelConfig: supports_prompt: Whether the model accepts prompt parameter. supports_mode: Whether the model accepts mode parameter. supports_language: Whether the model accepts language parameter. + supports_vad_params: Whether the model accepts fine-grained VAD parameters. default_language: Default language code (None = auto-detect). default_mode: Default mode (None = not applicable). use_translate_endpoint: Whether to use speech_to_text_translate_streaming endpoint. @@ -100,6 +101,7 @@ class ModelConfig: supports_prompt: bool supports_mode: bool supports_language: bool + supports_vad_params: bool default_language: str | None default_mode: str | None use_translate_endpoint: bool @@ -111,6 +113,7 @@ MODEL_CONFIGS: dict[str, ModelConfig] = { supports_prompt=False, supports_mode=False, supports_language=True, + supports_vad_params=False, default_language="unknown", default_mode=None, use_translate_endpoint=False, @@ -120,6 +123,7 @@ MODEL_CONFIGS: dict[str, ModelConfig] = { supports_prompt=True, supports_mode=False, supports_language=False, + supports_vad_params=False, default_language=None, # Auto-detects language default_mode=None, use_translate_endpoint=True, @@ -129,6 +133,7 @@ MODEL_CONFIGS: dict[str, ModelConfig] = { supports_prompt=False, supports_mode=True, supports_language=True, + supports_vad_params=True, default_language="unknown", default_mode="transcribe", use_translate_endpoint=False, @@ -146,11 +151,43 @@ class SarvamSTTSettings(STTSettings): Only applicable to models that support prompts (e.g., saaras:v2.5). vad_signals: Enable VAD signals in response. high_vad_sensitivity: Enable high VAD sensitivity. + positive_speech_threshold: VAD probability threshold (0.0-1.0) above which + a frame is considered speech. Only for saaras:v3. + negative_speech_threshold: VAD probability threshold (0.0-1.0) below which + a frame is considered silence. Only for saaras:v3. + min_speech_frames: Minimum consecutive speech frames to start a speech + segment. Only for saaras:v3. + first_turn_min_speech_frames: Minimum speech frames for the first user + turn. Only for saaras:v3. + negative_frames_count: Number of silence frames within the window to end + a speech segment. Only for saaras:v3. + negative_frames_window: Sliding window size (in frames) for counting + negative frames. Only for saaras:v3. + start_speech_volume_threshold: Volume level (dB) below which audio is + too quiet to be speech. Only for saaras:v3. + interrupt_min_speech_frames: Minimum speech frames to register a + barge-in/interruption. Only for saaras:v3. + pre_speech_pad_frames: Number of audio frames to prepend before detected + speech onset. Only for saaras:v3. + num_initial_ignored_frames: Number of leading audio frames to skip at + connection start. Only for saaras:v3. """ prompt: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) vad_signals: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) high_vad_sensitivity: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + positive_speech_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + negative_speech_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + min_speech_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + first_turn_min_speech_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + negative_frames_count: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + negative_frames_window: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + start_speech_volume_threshold: float | None | _NotGiven = field( + default_factory=lambda: NOT_GIVEN + ) + interrupt_min_speech_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + pre_speech_pad_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) + num_initial_ignored_frames: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) class SarvamSTTService(STTService): @@ -190,7 +227,27 @@ class SarvamSTTService(STTService): mode: Mode of operation for saaras:v3 models only. Options: transcribe, translate, verbatim, translit, codemix. Defaults to "transcribe" for saaras:v3. vad_signals: Enable VAD signals in response. Defaults to None. - high_vad_sensitivity: Enable high VAD (Voice Activity Detection) sensitivity. Defaults to None. + high_vad_sensitivity: Enable high VAD sensitivity. Defaults to None. + positive_speech_threshold: VAD probability threshold (0.0-1.0) above which + a frame is considered speech. Only for saaras:v3. + negative_speech_threshold: VAD probability threshold (0.0-1.0) below which + a frame is considered silence. Only for saaras:v3. + min_speech_frames: Minimum consecutive speech frames to start a segment. + Only for saaras:v3. + first_turn_min_speech_frames: Minimum speech frames for the first user + turn. Only for saaras:v3. + negative_frames_count: Silence frames within the window to end a segment. + Only for saaras:v3. + negative_frames_window: Sliding window size (in frames) for counting + negative frames. Only for saaras:v3. + start_speech_volume_threshold: Volume (dB) below which audio is too quiet. + Only for saaras:v3. + interrupt_min_speech_frames: Minimum speech frames for barge-in. + Only for saaras:v3. + pre_speech_pad_frames: Audio frames to prepend before speech onset. + Only for saaras:v3. + num_initial_ignored_frames: Leading frames to skip at connection start. + Only for saaras:v3. """ language: Language | None = None @@ -198,6 +255,16 @@ class SarvamSTTService(STTService): mode: Literal["transcribe", "translate", "verbatim", "translit", "codemix"] | None = None vad_signals: bool | None = None high_vad_sensitivity: bool | None = None + positive_speech_threshold: float | None = None + negative_speech_threshold: float | None = None + min_speech_frames: int | None = None + first_turn_min_speech_frames: int | None = None + negative_frames_count: int | None = None + negative_frames_window: int | None = None + start_speech_volume_threshold: float | None = None + interrupt_min_speech_frames: int | None = None + pre_speech_pad_frames: int | None = None + num_initial_ignored_frames: int | None = None def __init__( self, @@ -249,6 +316,16 @@ class SarvamSTTService(STTService): prompt=None, vad_signals=None, high_vad_sensitivity=None, + positive_speech_threshold=None, + negative_speech_threshold=None, + min_speech_frames=None, + first_turn_min_speech_frames=None, + negative_frames_count=None, + negative_frames_window=None, + start_speech_volume_threshold=None, + interrupt_min_speech_frames=None, + pre_speech_pad_frames=None, + num_initial_ignored_frames=None, ) # --- 2. Deprecated direct-arg overrides --- @@ -266,6 +343,18 @@ class SarvamSTTService(STTService): mode = params.mode default_settings.vad_signals = params.vad_signals default_settings.high_vad_sensitivity = params.high_vad_sensitivity + default_settings.positive_speech_threshold = params.positive_speech_threshold + default_settings.negative_speech_threshold = params.negative_speech_threshold + default_settings.min_speech_frames = params.min_speech_frames + default_settings.first_turn_min_speech_frames = params.first_turn_min_speech_frames + default_settings.negative_frames_count = params.negative_frames_count + default_settings.negative_frames_window = params.negative_frames_window + default_settings.start_speech_volume_threshold = ( + params.start_speech_volume_threshold + ) + default_settings.interrupt_min_speech_frames = params.interrupt_min_speech_frames + default_settings.pre_speech_pad_frames = params.pre_speech_pad_frames + default_settings.num_initial_ignored_frames = params.num_initial_ignored_frames # --- 4. Settings delta (canonical API, always wins) --- if settings is not None: @@ -289,6 +378,26 @@ class SarvamSTTService(STTService): f"Model '{resolved_model}' does not support language parameter (auto-detects language)." ) + if not self._config.supports_vad_params: + vad_param_names = [ + "positive_speech_threshold", + "negative_speech_threshold", + "min_speech_frames", + "first_turn_min_speech_frames", + "negative_frames_count", + "negative_frames_window", + "start_speech_volume_threshold", + "interrupt_min_speech_frames", + "pre_speech_pad_frames", + "num_initial_ignored_frames", + ] + for param_name in vad_param_names: + if getattr(default_settings, param_name) is not None: + raise ValueError( + f"Model '{resolved_model}' does not support {param_name} parameter. " + f"Fine-grained VAD parameters are only supported by saaras:v3." + ) + # Resolve mode default from model config if mode is None: mode = self._config.default_mode @@ -393,10 +502,44 @@ class SarvamSTTService(STTService): f"Model '{self._settings.model}' does not support prompt parameter." ) + if isinstance(delta, self.Settings) and not self._config.supports_vad_params: + vad_param_names = [ + "positive_speech_threshold", + "negative_speech_threshold", + "min_speech_frames", + "first_turn_min_speech_frames", + "negative_frames_count", + "negative_frames_window", + "start_speech_volume_threshold", + "interrupt_min_speech_frames", + "pre_speech_pad_frames", + "num_initial_ignored_frames", + ] + for param_name in vad_param_names: + val = getattr(delta, param_name, NOT_GIVEN) + if is_given(val) and val is not None: + raise ValueError( + f"Model '{self._settings.model}' does not support {param_name} " + f"parameter. Fine-grained VAD parameters are only supported by saaras:v3." + ) + changed = await super()._update_settings(delta) - # Language and prompt are WebSocket connect-time parameters; reconnect to apply. - reconnect_fields = {"language", "prompt"} + # These are all WebSocket connect-time parameters; reconnect to apply. + reconnect_fields = { + "language", + "prompt", + "positive_speech_threshold", + "negative_speech_threshold", + "min_speech_frames", + "first_turn_min_speech_frames", + "negative_frames_count", + "negative_frames_window", + "start_speech_volume_threshold", + "interrupt_min_speech_frames", + "pre_speech_pad_frames", + "num_initial_ignored_frames", + } if changed.keys() & reconnect_fields: await self._disconnect() await self._connect() @@ -535,6 +678,24 @@ class SarvamSTTService(STTService): "true" if self._settings.high_vad_sensitivity else "false" ) + # Fine-grained VAD parameters (saaras:v3 only, sent as strings per SDK spec) + if self._config.supports_vad_params: + _vad_params = { + "positive_speech_threshold": self._settings.positive_speech_threshold, + "negative_speech_threshold": self._settings.negative_speech_threshold, + "min_speech_frames": self._settings.min_speech_frames, + "first_turn_min_speech_frames": self._settings.first_turn_min_speech_frames, + "negative_frames_count": self._settings.negative_frames_count, + "negative_frames_window": self._settings.negative_frames_window, + "start_speech_volume_threshold": self._settings.start_speech_volume_threshold, + "interrupt_min_speech_frames": self._settings.interrupt_min_speech_frames, + "pre_speech_pad_frames": self._settings.pre_speech_pad_frames, + "num_initial_ignored_frames": self._settings.num_initial_ignored_frames, + } + for k, v in _vad_params.items(): + if v is not None: + connect_kwargs[k] = str(v) + # Add language_code for models that support it language_string = self._get_language_string() if language_string is not None: