From a7b2052b38f054af4c30e6fb2c00f5b2146f71bc Mon Sep 17 00:00:00 2001 From: Corvin Jaedicke Date: Thu, 13 Nov 2025 14:20:35 +0100 Subject: [PATCH] add ai-coustics VAD --- .../07ad-interruptible-aicoustics.py | 51 +++--- pyproject.toml | 4 + src/pipecat/audio/filters/aic_filter.py | 52 ++++++ src/pipecat/audio/vad/aic_vad.py | 158 ++++++++++++++++++ uv.lock | 36 ++-- 5 files changed, 261 insertions(+), 40 deletions(-) create mode 100644 src/pipecat/audio/vad/aic_vad.py diff --git a/examples/foundational/07ad-interruptible-aicoustics.py b/examples/foundational/07ad-interruptible-aicoustics.py index aa647ff33..0f6832a28 100644 --- a/examples/foundational/07ad-interruptible-aicoustics.py +++ b/examples/foundational/07ad-interruptible-aicoustics.py @@ -15,7 +15,6 @@ from loguru import logger from pipecat.audio.filters.aic_filter import AICFilter from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline @@ -48,7 +47,7 @@ def _create_aic_filter() -> AICFilter: return AICFilter( license_key=license_key, - enhancement_level=1.0, + enhancement_level=0.5, ) @@ -56,27 +55,33 @@ def _create_aic_filter() -> AICFilter: # instantiated. The function will be called when the desired transport gets # selected. transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - audio_in_filter=_create_aic_filter(), - ), - "twilio": lambda: FastAPIWebsocketParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - audio_in_filter=_create_aic_filter(), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - audio_in_filter=_create_aic_filter(), - ), + "daily": lambda: ( + lambda aic: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=aic, + ) + )(_create_aic_filter()), + "twilio": lambda: ( + lambda aic: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=aic, + ) + )(_create_aic_filter()), + "webrtc": lambda: ( + lambda aic: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=aic, + ) + )(_create_aic_filter()), } diff --git a/pyproject.toml b/pyproject.toml index 7b382f112..4e7a6130d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -128,6 +128,7 @@ dev = [ "setuptools~=78.1.1", "setuptools_scm~=8.3.1", "python-dotenv>=1.0.1,<2.0.0", + "pipecat-ai[aic,daily,deepgram,local-smart-turn-v3,openai,runner,silero,webrtc]", ] docs = [ @@ -205,3 +206,6 @@ convention = "google" command_line = "--module pytest" source = [ "src" ] omit = [ "*/tests/*" ] + +[tool.uv.sources] +pipecat-ai = { workspace = true } diff --git a/src/pipecat/audio/filters/aic_filter.py b/src/pipecat/audio/filters/aic_filter.py index 3f1af2f57..2f4699912 100644 --- a/src/pipecat/audio/filters/aic_filter.py +++ b/src/pipecat/audio/filters/aic_filter.py @@ -68,6 +68,58 @@ class AICFilter(BaseAudioFilter): # Model will be created in start() since the API now requires sample_rate self._aic = None + def get_vad_factory(self): + """Return a zero-arg factory that will create the VAD once the model exists. + + Returns: + A zero-argument callable that, when invoked, returns an initialized + VoiceActivityDetector bound to the underlying AIC model. Raises a + RuntimeError if the model has not been initialized (i.e. start() + has not been called successfully). + """ + + def _factory(): + if self._aic is None: + raise RuntimeError("AIC model not initialized yet. Call start(sample_rate) first.") + return self._aic.create_vad() + + return _factory + + def create_vad_analyzer( + self, + *, + lookback_buffer_size: Optional[float] = None, + sensitivity: Optional[float] = None, + ): + """Return an analyzer that will lazily instantiate the AIC VAD when ready. + + AIC VAD parameters: + - lookback_buffer_size: + Number of window-length audio buffers used as a lookback buffer. + Higher values increase prediction stability but add latency. + Range: 1.0 .. 20.0, Default (SDK): 6.0 + - sensitivity: + Energy threshold sensitivity. Energy threshold = 10 ** (-sensitivity). + Range: 1.0 .. 15.0, Default (SDK): 6.0 + + Args: + lookback_buffer_size: Optional lookback buffer size to configure on the VAD. + Range: 1.0 .. 20.0. If None, SDK default is used. + sensitivity: Optional sensitivity (energy threshold) to configure on the VAD. + Range: 1.0 .. 15.0. If None, SDK default is used. + + Returns: + A lazily-initialized AICVADAnalyzer that will bind to the VAD backend + once the filter's model has been created (after start(sample_rate)). + """ + from pipecat.audio.vad.aic_vad import AICVADAnalyzer + + return AICVADAnalyzer( + vad_factory=self.get_vad_factory(), + lookback_buffer_size=lookback_buffer_size, + sensitivity=sensitivity, + ) + async def start(self, sample_rate: int): """Initialize the filter with the transport's sample rate. diff --git a/src/pipecat/audio/vad/aic_vad.py b/src/pipecat/audio/vad/aic_vad.py new file mode 100644 index 000000000..4907e4f55 --- /dev/null +++ b/src/pipecat/audio/vad/aic_vad.py @@ -0,0 +1,158 @@ +"""AIC-integrated VAD analyzer that lazily binds to the AIC SDK backend. + +This analyzer queries the backend's is_speech_detected() and maps it to a float +confidence (1.0/0.0). It uses 10 ms windows based on the sample rate and applies +optional AIC VAD parameters (lookback_buffer_size, sensitivity) when available. +""" + +from typing import Any, Callable, Optional + +from loguru import logger + +from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams + +try: + from aic import AICVadParameter +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use the AIC filter, you need to `pip install pipecat-ai[aic]`.") + raise Exception(f"Missing module: {e}") + + +class AICVADAnalyzer(VADAnalyzer): + """VAD analyzer that lazily instantiates the AIC VoiceActivityDetector via a factory. + + The analyzer can be constructed before the AIC Model exists. Once the filter has + started and the Model is available, the provided factory will succeed and the + backend VAD will be created. We then switch to single-sample updates where + num_frames_required() returns 1 and confidence is derived from the backend's + boolean is_speech_detected() state. + + AIC VAD runtime parameters: + - lookback_buffer_size: + Controls the lookback buffer size used by the VAD, i.e. the number of + window-length audio buffers used as a lookback buffer. Larger values improve + stability but increase latency. + Range: 1.0 .. 20.0 + Default (SDK): 6.0 + - sensitivity: + Controls the energy threshold sensitivity. Higher values make the detector + less sensitive (require more energy to count as speech). + Range: 1.0 .. 15.0 + Formula: Energy threshold = 10 ** (-sensitivity) + Default (SDK): 6.0 + """ + + def __init__( + self, + *, + vad_factory: Optional[Callable[[], Any]] = None, + lookback_buffer_size: Optional[float] = None, + sensitivity: Optional[float] = None, + ): + """Create an AIC VAD analyzer. + + Args: + vad_factory: + Zero-arg callable that returns an initialized AIC VoiceActivityDetector. + This may raise until the filter's Model has been created; the analyzer + will retry on set_sample_rate/first use. + lookback_buffer_size: + Optional override for AIC VAD lookback buffer size. + Range: 1.0 .. 20.0. Larger values increase stability at the cost of latency. + If None, the SDK default (6.0) is used. + sensitivity: + Optional override for AIC VAD sensitivity (energy threshold). + Range: 1.0 .. 15.0. Energy threshold = 10 ** (-sensitivity). + If None, the SDK default (6.0) is used. + """ + # Use fixed VAD parameters for AIC: no user override + fixed_params = VADParams(confidence=0.5, start_secs=0.0, stop_secs=0.0, min_volume=0.0) + super().__init__(sample_rate=None, params=fixed_params) + self._vad_factory = vad_factory + self._backend_vad: Optional[Any] = None + self._pending_lookback: Optional[float] = lookback_buffer_size + self._pending_sensitivity: Optional[float] = sensitivity + + def bind_vad_factory(self, vad_factory: Callable[[], Any]): + """Attach or replace the factory post-construction.""" + self._vad_factory = vad_factory + self._ensure_backend_initialized() + + def _apply_backend_params(self): + """Apply optional AIC VAD parameters if available.""" + if self._backend_vad is None or AICVadParameter is None: + return + try: + if self._pending_lookback is not None: + self._backend_vad.set_parameter( + AICVadParameter.LOOKBACK_BUFFER_SIZE, float(self._pending_lookback) + ) + if self._pending_sensitivity is not None: + self._backend_vad.set_parameter( + AICVadParameter.SENSITIVITY, float(self._pending_sensitivity) + ) + except Exception as e: # noqa: BLE001 + logger.debug(f"AIC VAD parameter application deferred/failed: {e}") + + def _ensure_backend_initialized(self): + if self._backend_vad is not None: + return + if not self._vad_factory: + return + try: + self._backend_vad = self._vad_factory() + self._apply_backend_params() + # With backend ready, recompute internal frame sizing + super().set_params(self._params) + logger.debug("AIC VAD backend initialized in analyzer.") + except Exception as e: # noqa: BLE001 + # Filter may not be started yet; try again later + logger.debug(f"Deferring AIC VAD backend initialization: {e}") + + def set_sample_rate(self, sample_rate: int): + """Set the sample rate for audio processing. + + Args: + sample_rate: Audio sample rate in Hz. + """ + # Set rate and attempt backend initialization once we know SR + self._sample_rate = self._init_sample_rate or sample_rate + self._ensure_backend_initialized() + # Ensure params are initialized even if backend not ready yet + try: + super().set_params(self._params) + except Exception: + pass + + def num_frames_required(self) -> int: + """Get the number of audio frames required for analysis. + + Returns: + Number of frames needed for VAD processing. + """ + # Use 10 ms windows based on sample rate + return int(self.sample_rate * 0.01) if self.sample_rate > 0 else 160 + + def voice_confidence(self, buffer: bytes) -> float: + """Calculate voice activity confidence for the given audio buffer. + + Args: + buffer: Audio buffer to analyze. + + Returns: + Voice confidence score is 0.0 or 1.0. + """ + # Ensure backend exists (filter might have started since last call) + self._ensure_backend_initialized() + if self._backend_vad is None: + return 0.0 + + # We do not need to analyze 'buffer' here since the model's VAD is updated + # as part of the enhancement pipeline. Simply query the boolean and map it. + try: + is_speech = self._backend_vad.is_speech_detected() + return 1.0 if is_speech else 0.0 + except Exception as e: # noqa: BLE001 + logger.error(f"AIC VAD inference error: {e}") + return 0.0 diff --git a/uv.lock b/uv.lock index eabe03714..7fb531dea 100644 --- a/uv.lock +++ b/uv.lock @@ -4624,6 +4624,7 @@ dev = [ { name = "coverage" }, { name = "grpcio-tools" }, { name = "pip-tools" }, + { name = "pipecat-ai", extra = ["aic", "daily", "deepgram", "local-smart-turn-v3", "openai", "runner", "silero", "webrtc"] }, { name = "pre-commit" }, { name = "pyright" }, { name = "pytest" }, @@ -4697,23 +4698,23 @@ requires-dist = [ { name = "opentelemetry-sdk", marker = "extra == 'tracing'", specifier = ">=1.33.0" }, { name = "ormsgpack", marker = "extra == 'fish'", specifier = "~=1.7.0" }, { name = "pillow", specifier = ">=11.1.0,<12" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'assemblyai'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'asyncai'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'aws'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'cartesia'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'elevenlabs'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'fish'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'gladia'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'google'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'heygen'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'lmnt'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'neuphonic'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'openai'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'playht'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'rime'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'sarvam'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'soniox'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'websocket'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'assemblyai'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'asyncai'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'aws'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'cartesia'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'elevenlabs'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'fish'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'gladia'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'google'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'heygen'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'lmnt'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'neuphonic'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'openai'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'playht'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'rime'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'sarvam'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'soniox'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'websocket'", editable = "." }, { name = "pipecat-ai-krisp", marker = "extra == 'krisp'", specifier = "~=0.4.0" }, { name = "pipecat-ai-small-webrtc-prebuilt", marker = "extra == 'runner'", specifier = ">=1.0.0" }, { name = "protobuf", specifier = "~=5.29.3" }, @@ -4753,6 +4754,7 @@ dev = [ { name = "coverage", specifier = "~=7.9.1" }, { name = "grpcio-tools", specifier = "~=1.67.1" }, { name = "pip-tools", specifier = "~=7.4.1" }, + { name = "pipecat-ai", extras = ["aic", "daily", "deepgram", "local-smart-turn-v3", "openai", "runner", "silero", "webrtc"], editable = "." }, { name = "pre-commit", specifier = "~=4.2.0" }, { name = "pyright", specifier = ">=1.1.404,<1.2" }, { name = "pytest", specifier = "~=8.4.1" },