wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add ai-coustics VAD

Browse Source
This commit is contained in:
Corvin Jaedicke
2025-11-13 14:20:35 +01:00
parent 3c76917c1e
commit a7b2052b38
5 changed files with 261 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
examples/foundational/07ad-interruptible-aicoustics.py
View File

@@ -15,7 +15,6 @@ from loguru import logger
from pipecat.audio.filters.aic_filter import AICFilter
from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams
from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.frames.frames import LLMRunFrame
from pipecat.pipeline.pipeline import Pipeline
@@ -48,7 +47,7 @@ def _create_aic_filter() -> AICFilter:
return AICFilter(
license_key=license_key,
enhancement_level=1.0,
enhancement_level=0.5,
)
@@ -56,27 +55,33 @@ def _create_aic_filter() -> AICFilter:
# instantiated. The function will be called when the desired transport gets
# selected.
transport_params = {
"daily": lambda: DailyParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
audio_in_filter=_create_aic_filter(),
),
"twilio": lambda: FastAPIWebsocketParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
audio_in_filter=_create_aic_filter(),
),
"webrtc": lambda: TransportParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
audio_in_filter=_create_aic_filter(),
),
"daily": lambda: (
lambda aic: DailyParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0),
turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
audio_in_filter=aic,
)
)(_create_aic_filter()),
"twilio": lambda: (
lambda aic: FastAPIWebsocketParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0),
turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
audio_in_filter=aic,
)
)(_create_aic_filter()),
"webrtc": lambda: (
lambda aic: TransportParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0),
turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
audio_in_filter=aic,
)
)(_create_aic_filter()),
}

4
pyproject.toml
View File

@@ -128,6 +128,7 @@ dev = [
"setuptools~=78.1.1",
"setuptools_scm~=8.3.1",
"python-dotenv>=1.0.1,<2.0.0",
"pipecat-ai[aic,daily,deepgram,local-smart-turn-v3,openai,runner,silero,webrtc]",
]
docs = [
@@ -205,3 +206,6 @@ convention = "google"
command_line = "--module pytest"
source = [ "src" ]
omit = [ "*/tests/*" ]
[tool.uv.sources]
pipecat-ai = { workspace = true }

52
src/pipecat/audio/filters/aic_filter.py
View File

@@ -68,6 +68,58 @@ class AICFilter(BaseAudioFilter):
# Model will be created in start() since the API now requires sample_rate
self._aic = None
def get_vad_factory(self):
"""Return a zero-arg factory that will create the VAD once the model exists.
Returns:
A zero-argument callable that, when invoked, returns an initialized
VoiceActivityDetector bound to the underlying AIC model. Raises a
RuntimeError if the model has not been initialized (i.e. start()
has not been called successfully).
"""
def _factory():
if self._aic is None:
raise RuntimeError("AIC model not initialized yet. Call start(sample_rate) first.")
return self._aic.create_vad()
return _factory
def create_vad_analyzer(
self,
*,
lookback_buffer_size: Optional[float] = None,
sensitivity: Optional[float] = None,
):
"""Return an analyzer that will lazily instantiate the AIC VAD when ready.
AIC VAD parameters:
- lookback_buffer_size:
Number of window-length audio buffers used as a lookback buffer.
Higher values increase prediction stability but add latency.
Range: 1.0 .. 20.0, Default (SDK): 6.0
- sensitivity:
Energy threshold sensitivity. Energy threshold = 10 ** (-sensitivity).
Range: 1.0 .. 15.0, Default (SDK): 6.0
Args:
lookback_buffer_size: Optional lookback buffer size to configure on the VAD.
Range: 1.0 .. 20.0. If None, SDK default is used.
sensitivity: Optional sensitivity (energy threshold) to configure on the VAD.
Range: 1.0 .. 15.0. If None, SDK default is used.
Returns:
A lazily-initialized AICVADAnalyzer that will bind to the VAD backend
once the filter's model has been created (after start(sample_rate)).
"""
from pipecat.audio.vad.aic_vad import AICVADAnalyzer
return AICVADAnalyzer(
vad_factory=self.get_vad_factory(),
lookback_buffer_size=lookback_buffer_size,
sensitivity=sensitivity,
)
async def start(self, sample_rate: int):
"""Initialize the filter with the transport's sample rate.

158
src/pipecat/audio/vad/aic_vad.py Normal file
View File

@@ -0,0 +1,158 @@
"""AIC-integrated VAD analyzer that lazily binds to the AIC SDK backend.
This analyzer queries the backend's is_speech_detected() and maps it to a float
confidence (1.0/0.0). It uses 10 ms windows based on the sample rate and applies
optional AIC VAD parameters (lookback_buffer_size, sensitivity) when available.
"""
from typing import Any, Callable, Optional
from loguru import logger
from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams
try:
from aic import AICVadParameter
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use the AIC filter, you need to `pip install pipecat-ai[aic]`.")
raise Exception(f"Missing module: {e}")
class AICVADAnalyzer(VADAnalyzer):
"""VAD analyzer that lazily instantiates the AIC VoiceActivityDetector via a factory.
The analyzer can be constructed before the AIC Model exists. Once the filter has
started and the Model is available, the provided factory will succeed and the
backend VAD will be created. We then switch to single-sample updates where
num_frames_required() returns 1 and confidence is derived from the backend's
boolean is_speech_detected() state.
AIC VAD runtime parameters:
- lookback_buffer_size:
Controls the lookback buffer size used by the VAD, i.e. the number of
window-length audio buffers used as a lookback buffer. Larger values improve
stability but increase latency.
Range: 1.0 .. 20.0
Default (SDK): 6.0
- sensitivity:
Controls the energy threshold sensitivity. Higher values make the detector
less sensitive (require more energy to count as speech).
Range: 1.0 .. 15.0
Formula: Energy threshold = 10 ** (-sensitivity)
Default (SDK): 6.0
"""
def __init__(
self,
*,
vad_factory: Optional[Callable[[], Any]] = None,
lookback_buffer_size: Optional[float] = None,
sensitivity: Optional[float] = None,
):
"""Create an AIC VAD analyzer.
Args:
vad_factory:
Zero-arg callable that returns an initialized AIC VoiceActivityDetector.
This may raise until the filter's Model has been created; the analyzer
will retry on set_sample_rate/first use.
lookback_buffer_size:
Optional override for AIC VAD lookback buffer size.
Range: 1.0 .. 20.0. Larger values increase stability at the cost of latency.
If None, the SDK default (6.0) is used.
sensitivity:
Optional override for AIC VAD sensitivity (energy threshold).
Range: 1.0 .. 15.0. Energy threshold = 10 ** (-sensitivity).
If None, the SDK default (6.0) is used.
"""
# Use fixed VAD parameters for AIC: no user override
fixed_params = VADParams(confidence=0.5, start_secs=0.0, stop_secs=0.0, min_volume=0.0)
super().__init__(sample_rate=None, params=fixed_params)
self._vad_factory = vad_factory
self._backend_vad: Optional[Any] = None
self._pending_lookback: Optional[float] = lookback_buffer_size
self._pending_sensitivity: Optional[float] = sensitivity
def bind_vad_factory(self, vad_factory: Callable[[], Any]):
"""Attach or replace the factory post-construction."""
self._vad_factory = vad_factory
self._ensure_backend_initialized()
def _apply_backend_params(self):
"""Apply optional AIC VAD parameters if available."""
if self._backend_vad is None or AICVadParameter is None:
return
try:
if self._pending_lookback is not None:
self._backend_vad.set_parameter(
AICVadParameter.LOOKBACK_BUFFER_SIZE, float(self._pending_lookback)
)
if self._pending_sensitivity is not None:
self._backend_vad.set_parameter(
AICVadParameter.SENSITIVITY, float(self._pending_sensitivity)
)
except Exception as e: # noqa: BLE001
logger.debug(f"AIC VAD parameter application deferred/failed: {e}")
def _ensure_backend_initialized(self):
if self._backend_vad is not None:
return
if not self._vad_factory:
return
try:
self._backend_vad = self._vad_factory()
self._apply_backend_params()
# With backend ready, recompute internal frame sizing
super().set_params(self._params)
logger.debug("AIC VAD backend initialized in analyzer.")
except Exception as e: # noqa: BLE001
# Filter may not be started yet; try again later
logger.debug(f"Deferring AIC VAD backend initialization: {e}")
def set_sample_rate(self, sample_rate: int):
"""Set the sample rate for audio processing.
Args:
sample_rate: Audio sample rate in Hz.
"""
# Set rate and attempt backend initialization once we know SR
self._sample_rate = self._init_sample_rate or sample_rate
self._ensure_backend_initialized()
# Ensure params are initialized even if backend not ready yet
try:
super().set_params(self._params)
except Exception:
pass
def num_frames_required(self) -> int:
"""Get the number of audio frames required for analysis.
Returns:
Number of frames needed for VAD processing.
"""
# Use 10 ms windows based on sample rate
return int(self.sample_rate * 0.01) if self.sample_rate > 0 else 160
def voice_confidence(self, buffer: bytes) -> float:
"""Calculate voice activity confidence for the given audio buffer.
Args:
buffer: Audio buffer to analyze.
Returns:
Voice confidence score is 0.0 or 1.0.
"""
# Ensure backend exists (filter might have started since last call)
self._ensure_backend_initialized()
if self._backend_vad is None:
return 0.0
# We do not need to analyze 'buffer' here since the model's VAD is updated
# as part of the enhancement pipeline. Simply query the boolean and map it.
try:
is_speech = self._backend_vad.is_speech_detected()
return 1.0 if is_speech else 0.0
except Exception as e: # noqa: BLE001
logger.error(f"AIC VAD inference error: {e}")
return 0.0

36
uv.lock generated
View File

@@ -4624,6 +4624,7 @@ dev = [
{ name = "coverage" },
{ name = "grpcio-tools" },
{ name = "pip-tools" },
{ name = "pipecat-ai", extra = ["aic", "daily", "deepgram", "local-smart-turn-v3", "openai", "runner", "silero", "webrtc"] },
{ name = "pre-commit" },
{ name = "pyright" },
{ name = "pytest" },
@@ -4697,23 +4698,23 @@ requires-dist = [
{ name = "opentelemetry-sdk", marker = "extra == 'tracing'", specifier = ">=1.33.0" },
{ name = "ormsgpack", marker = "extra == 'fish'", specifier = "~=1.7.0" },
{ name = "pillow", specifier = ">=11.1.0,<12" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'assemblyai'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'asyncai'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'aws'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'cartesia'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'elevenlabs'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'fish'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'gladia'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'google'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'heygen'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'lmnt'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'neuphonic'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'openai'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'playht'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'rime'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'sarvam'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'soniox'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'websocket'" },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'assemblyai'", editable = "." },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'asyncai'", editable = "." },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'aws'", editable = "." },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'cartesia'", editable = "." },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'elevenlabs'", editable = "." },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'fish'", editable = "." },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'gladia'", editable = "." },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'google'", editable = "." },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'heygen'", editable = "." },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'lmnt'", editable = "." },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'neuphonic'", editable = "." },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'openai'", editable = "." },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'playht'", editable = "." },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'rime'", editable = "." },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'sarvam'", editable = "." },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'soniox'", editable = "." },
{ name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'websocket'", editable = "." },
{ name = "pipecat-ai-krisp", marker = "extra == 'krisp'", specifier = "~=0.4.0" },
{ name = "pipecat-ai-small-webrtc-prebuilt", marker = "extra == 'runner'", specifier = ">=1.0.0" },
{ name = "protobuf", specifier = "~=5.29.3" },
@@ -4753,6 +4754,7 @@ dev = [
{ name = "coverage", specifier = "~=7.9.1" },
{ name = "grpcio-tools", specifier = "~=1.67.1" },
{ name = "pip-tools", specifier = "~=7.4.1" },
{ name = "pipecat-ai", extras = ["aic", "daily", "deepgram", "local-smart-turn-v3", "openai", "runner", "silero", "webrtc"], editable = "." },
{ name = "pre-commit", specifier = "~=4.2.0" },
{ name = "pyright", specifier = ">=1.1.404,<1.2" },
{ name = "pytest", specifier = "~=8.4.1" },