diff --git a/CHANGELOG.md b/CHANGELOG.md index bb7223b7c..3748892b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Prevented `HeyGenVideoService` from automatically disconnecting after 5 minutes. +### Added + +- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and + example wiring; leverages the enhancement model for robust detection with no + ONNX dependency or added processing complexity. + ## [0.0.94] - 2025-11-10 ### Changed diff --git a/examples/foundational/07ad-interruptible-aicoustics.py b/examples/foundational/07ad-interruptible-aicoustics.py index 16a246699..edcd9498f 100644 --- a/examples/foundational/07ad-interruptible-aicoustics.py +++ b/examples/foundational/07ad-interruptible-aicoustics.py @@ -15,7 +15,6 @@ from loguru import logger from pipecat.audio.filters.aic_filter import AICFilter from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline @@ -48,7 +47,7 @@ def _create_aic_filter() -> AICFilter: return AICFilter( license_key=license_key, - enhancement_level=1.0, + enhancement_level=0.5, ) @@ -56,27 +55,33 @@ def _create_aic_filter() -> AICFilter: # instantiated. The function will be called when the desired transport gets # selected. transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - audio_in_filter=_create_aic_filter(), - ), - "twilio": lambda: FastAPIWebsocketParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - audio_in_filter=_create_aic_filter(), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - audio_in_filter=_create_aic_filter(), - ), + "daily": lambda: ( + lambda aic: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=aic, + ) + )(_create_aic_filter()), + "twilio": lambda: ( + lambda aic: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=aic, + ) + )(_create_aic_filter()), + "webrtc": lambda: ( + lambda aic: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=aic, + ) + )(_create_aic_filter()), } diff --git a/pyproject.toml b/pyproject.toml index a203b47b0..e313c789d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ Source = "https://github.com/pipecat-ai/pipecat" Website = "https://pipecat.ai" [project.optional-dependencies] -aic = [ "aic-sdk~=1.0.1" ] +aic = [ "aic-sdk~=1.1.0" ] anthropic = [ "anthropic~=0.49.0" ] assemblyai = [ "pipecat-ai[websockets-base]" ] asyncai = [ "pipecat-ai[websockets-base]" ] diff --git a/src/pipecat/audio/filters/aic_filter.py b/src/pipecat/audio/filters/aic_filter.py index 8c2f3e7f4..2f4699912 100644 --- a/src/pipecat/audio/filters/aic_filter.py +++ b/src/pipecat/audio/filters/aic_filter.py @@ -68,6 +68,58 @@ class AICFilter(BaseAudioFilter): # Model will be created in start() since the API now requires sample_rate self._aic = None + def get_vad_factory(self): + """Return a zero-arg factory that will create the VAD once the model exists. + + Returns: + A zero-argument callable that, when invoked, returns an initialized + VoiceActivityDetector bound to the underlying AIC model. Raises a + RuntimeError if the model has not been initialized (i.e. start() + has not been called successfully). + """ + + def _factory(): + if self._aic is None: + raise RuntimeError("AIC model not initialized yet. Call start(sample_rate) first.") + return self._aic.create_vad() + + return _factory + + def create_vad_analyzer( + self, + *, + lookback_buffer_size: Optional[float] = None, + sensitivity: Optional[float] = None, + ): + """Return an analyzer that will lazily instantiate the AIC VAD when ready. + + AIC VAD parameters: + - lookback_buffer_size: + Number of window-length audio buffers used as a lookback buffer. + Higher values increase prediction stability but add latency. + Range: 1.0 .. 20.0, Default (SDK): 6.0 + - sensitivity: + Energy threshold sensitivity. Energy threshold = 10 ** (-sensitivity). + Range: 1.0 .. 15.0, Default (SDK): 6.0 + + Args: + lookback_buffer_size: Optional lookback buffer size to configure on the VAD. + Range: 1.0 .. 20.0. If None, SDK default is used. + sensitivity: Optional sensitivity (energy threshold) to configure on the VAD. + Range: 1.0 .. 15.0. If None, SDK default is used. + + Returns: + A lazily-initialized AICVADAnalyzer that will bind to the VAD backend + once the filter's model has been created (after start(sample_rate)). + """ + from pipecat.audio.vad.aic_vad import AICVADAnalyzer + + return AICVADAnalyzer( + vad_factory=self.get_vad_factory(), + lookback_buffer_size=lookback_buffer_size, + sensitivity=sensitivity, + ) + async def start(self, sample_rate: int): """Initialize the filter with the transport's sample rate. @@ -185,7 +237,7 @@ class AICFilter(BaseAudioFilter): ) # Process planar in-place; returns ndarray (same shape) - out_f32 = self._aic.process(block_f32) + out_f32 = await self._aic.process_async(block_f32) # Convert back to int16 bytes, planar layout out_i16 = np.clip(out_f32 * 32768.0, -32768, 32767).astype(np.int16) diff --git a/src/pipecat/audio/vad/aic_vad.py b/src/pipecat/audio/vad/aic_vad.py new file mode 100644 index 000000000..4907e4f55 --- /dev/null +++ b/src/pipecat/audio/vad/aic_vad.py @@ -0,0 +1,158 @@ +"""AIC-integrated VAD analyzer that lazily binds to the AIC SDK backend. + +This analyzer queries the backend's is_speech_detected() and maps it to a float +confidence (1.0/0.0). It uses 10 ms windows based on the sample rate and applies +optional AIC VAD parameters (lookback_buffer_size, sensitivity) when available. +""" + +from typing import Any, Callable, Optional + +from loguru import logger + +from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams + +try: + from aic import AICVadParameter +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use the AIC filter, you need to `pip install pipecat-ai[aic]`.") + raise Exception(f"Missing module: {e}") + + +class AICVADAnalyzer(VADAnalyzer): + """VAD analyzer that lazily instantiates the AIC VoiceActivityDetector via a factory. + + The analyzer can be constructed before the AIC Model exists. Once the filter has + started and the Model is available, the provided factory will succeed and the + backend VAD will be created. We then switch to single-sample updates where + num_frames_required() returns 1 and confidence is derived from the backend's + boolean is_speech_detected() state. + + AIC VAD runtime parameters: + - lookback_buffer_size: + Controls the lookback buffer size used by the VAD, i.e. the number of + window-length audio buffers used as a lookback buffer. Larger values improve + stability but increase latency. + Range: 1.0 .. 20.0 + Default (SDK): 6.0 + - sensitivity: + Controls the energy threshold sensitivity. Higher values make the detector + less sensitive (require more energy to count as speech). + Range: 1.0 .. 15.0 + Formula: Energy threshold = 10 ** (-sensitivity) + Default (SDK): 6.0 + """ + + def __init__( + self, + *, + vad_factory: Optional[Callable[[], Any]] = None, + lookback_buffer_size: Optional[float] = None, + sensitivity: Optional[float] = None, + ): + """Create an AIC VAD analyzer. + + Args: + vad_factory: + Zero-arg callable that returns an initialized AIC VoiceActivityDetector. + This may raise until the filter's Model has been created; the analyzer + will retry on set_sample_rate/first use. + lookback_buffer_size: + Optional override for AIC VAD lookback buffer size. + Range: 1.0 .. 20.0. Larger values increase stability at the cost of latency. + If None, the SDK default (6.0) is used. + sensitivity: + Optional override for AIC VAD sensitivity (energy threshold). + Range: 1.0 .. 15.0. Energy threshold = 10 ** (-sensitivity). + If None, the SDK default (6.0) is used. + """ + # Use fixed VAD parameters for AIC: no user override + fixed_params = VADParams(confidence=0.5, start_secs=0.0, stop_secs=0.0, min_volume=0.0) + super().__init__(sample_rate=None, params=fixed_params) + self._vad_factory = vad_factory + self._backend_vad: Optional[Any] = None + self._pending_lookback: Optional[float] = lookback_buffer_size + self._pending_sensitivity: Optional[float] = sensitivity + + def bind_vad_factory(self, vad_factory: Callable[[], Any]): + """Attach or replace the factory post-construction.""" + self._vad_factory = vad_factory + self._ensure_backend_initialized() + + def _apply_backend_params(self): + """Apply optional AIC VAD parameters if available.""" + if self._backend_vad is None or AICVadParameter is None: + return + try: + if self._pending_lookback is not None: + self._backend_vad.set_parameter( + AICVadParameter.LOOKBACK_BUFFER_SIZE, float(self._pending_lookback) + ) + if self._pending_sensitivity is not None: + self._backend_vad.set_parameter( + AICVadParameter.SENSITIVITY, float(self._pending_sensitivity) + ) + except Exception as e: # noqa: BLE001 + logger.debug(f"AIC VAD parameter application deferred/failed: {e}") + + def _ensure_backend_initialized(self): + if self._backend_vad is not None: + return + if not self._vad_factory: + return + try: + self._backend_vad = self._vad_factory() + self._apply_backend_params() + # With backend ready, recompute internal frame sizing + super().set_params(self._params) + logger.debug("AIC VAD backend initialized in analyzer.") + except Exception as e: # noqa: BLE001 + # Filter may not be started yet; try again later + logger.debug(f"Deferring AIC VAD backend initialization: {e}") + + def set_sample_rate(self, sample_rate: int): + """Set the sample rate for audio processing. + + Args: + sample_rate: Audio sample rate in Hz. + """ + # Set rate and attempt backend initialization once we know SR + self._sample_rate = self._init_sample_rate or sample_rate + self._ensure_backend_initialized() + # Ensure params are initialized even if backend not ready yet + try: + super().set_params(self._params) + except Exception: + pass + + def num_frames_required(self) -> int: + """Get the number of audio frames required for analysis. + + Returns: + Number of frames needed for VAD processing. + """ + # Use 10 ms windows based on sample rate + return int(self.sample_rate * 0.01) if self.sample_rate > 0 else 160 + + def voice_confidence(self, buffer: bytes) -> float: + """Calculate voice activity confidence for the given audio buffer. + + Args: + buffer: Audio buffer to analyze. + + Returns: + Voice confidence score is 0.0 or 1.0. + """ + # Ensure backend exists (filter might have started since last call) + self._ensure_backend_initialized() + if self._backend_vad is None: + return 0.0 + + # We do not need to analyze 'buffer' here since the model's VAD is updated + # as part of the enhancement pipeline. Simply query the boolean and map it. + try: + is_speech = self._backend_vad.is_speech_detected() + return 1.0 if is_speech else 0.0 + except Exception as e: # noqa: BLE001 + logger.error(f"AIC VAD inference error: {e}") + return 0.0