Merge pull request #3408 from ai-coustics/aic-v2

Add ai-coustics AIC SDK v2 support with model downloading
This commit is contained in:
Mark Backman
2026-01-27 10:38:26 -05:00
committed by GitHub
10 changed files with 1074 additions and 207 deletions

View File

@@ -9,129 +9,145 @@
This module provides an audio filter implementation using ai-coustics' AIC SDK to
enhance audio streams in real time. It mirrors the structure of other filters like
the Koala filter and integrates with Pipecat's input transport pipeline.
Classes:
AICFilter: For aic-sdk (uses 'aic_sdk' module)
"""
from pathlib import Path
from typing import List, Optional
import numpy as np
from aic_sdk import (
Model,
ParameterFixedError,
ProcessorAsync,
ProcessorConfig,
ProcessorParameter,
set_sdk_id,
)
from loguru import logger
from pipecat.audio.filters.base_audio_filter import BaseAudioFilter
from pipecat.audio.vad.aic_vad import AICVADAnalyzer
from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame
try:
# AIC SDK (https://ai-coustics.github.io/aic-sdk-py/api/)
from aic import AICModelType, AICParameter, Model
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use the AIC filter, you need to `pip install pipecat-ai[aic]`.")
raise Exception(f"Missing module: {e}")
class AICFilter(BaseAudioFilter):
"""Audio filter using ai-coustics' AIC SDK for real-time enhancement.
Buffers incoming audio to the model's preferred block size and processes
planar frames in-place using float32 samples in the linear -1..+1 range.
frames using float32 samples normalized to the range -1 to +1.
"""
def __init__(
self,
*,
license_key: str = "",
model_type: AICModelType = AICModelType.QUAIL_STT,
enhancement_level: Optional[float] = 1.0,
voice_gain: Optional[float] = 1.0,
noise_gate_enable: Optional[bool] = True,
license_key: str,
model_id: Optional[str] = None,
model_path: Optional[Path] = None,
model_download_dir: Optional[Path] = None,
) -> None:
"""Initialize the AIC filter.
Args:
license_key: ai-coustics license key for authentication.
model_type: Model variant to load.
enhancement_level: Optional overall enhancement strength (0.0..1.0).
voice_gain: Optional linear gain applied to detected speech (0.0..4.0).
noise_gate_enable: Optional enable/disable noise gate (default: True).
model_id: Model identifier to download from CDN. Required if model_path
is not provided. See https://artifacts.ai-coustics.io/ for available models.
model_path: Optional path to a local .aicmodel file. If provided,
model_id is ignored and no download occurs.
model_download_dir: Directory for downloading models as a Path object.
Defaults to a cache directory in user's home folder.
.. deprecated:: 1.3.0
The `noise_gate_enable` parameter is deprecated and no longer has any effect.
It will be removed in a future version.
Raises:
ValueError: If neither model_id nor model_path is provided.
"""
# Set SDK ID for telemetry identification (6 = pipecat)
set_sdk_id(6)
if model_id is None and model_path is None:
raise ValueError(
"Either 'model_id' or 'model_path' must be provided. "
"See https://artifacts.ai-coustics.io/ for available models."
)
self._license_key = license_key
self._model_type = model_type
self._model_id = model_id
self._model_path = model_path
self._model_download_dir = model_download_dir or (
Path.home() / ".cache" / "pipecat" / "aic-models"
)
self._enhancement_level = enhancement_level
self._voice_gain = voice_gain
if noise_gate_enable is not None:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"Parameter `noise_gate_enable` is deprecated and no longer has any effect. "
"It will be removed in a future version. Use AIC VAD instead (create_vad_analyzer()).",
DeprecationWarning,
)
self._noise_gate_enable = noise_gate_enable
self._enabled = True
self._bypass = False
self._sample_rate = 0
self._aic_ready = False
self._frames_per_block = 0
self._audio_buffer = bytearray()
# Model will be created in start() since the API now requires sample_rate
self._aic = None
def get_vad_factory(self):
"""Return a zero-arg factory that will create the VAD once the model exists.
# Audio format constants
self._bytes_per_sample = 2 # int16 = 2 bytes
self._dtype = np.int16
self._scale = (
32768.0 # 2^15, for normalizing int16 (-32768 to 32767) to float32 (-1.0 to 1.0)
)
# AIC SDK objects
self._model = None
self._processor = None
self._processor_ctx = None
self._vad_ctx = None
# Pre-allocated buffers (resized in start() once frames_per_block is known)
self._in_f32 = None
self._out_i16 = None
def get_vad_context(self):
"""Return the VAD context once the processor exists.
Returns:
A zero-argument callable that, when invoked, returns an initialized
VoiceActivityDetector bound to the underlying AIC model. Raises a
RuntimeError if the model has not been initialized (i.e. start()
has not been called successfully).
The VadContext instance bound to the underlying processor.
Raises RuntimeError if the processor has not been initialized.
"""
def _factory():
if self._aic is None:
raise RuntimeError("AIC model not initialized yet. Call start(sample_rate) first.")
return self._aic.create_vad()
return _factory
if self._vad_ctx is None:
raise RuntimeError("AIC processor not initialized yet. Call start(sample_rate) first.")
return self._vad_ctx
def create_vad_analyzer(
self,
*,
lookback_buffer_size: Optional[float] = None,
speech_hold_duration: Optional[float] = None,
minimum_speech_duration: Optional[float] = None,
sensitivity: Optional[float] = None,
):
"""Return an analyzer that will lazily instantiate the AIC VAD when ready.
AIC VAD parameters:
- lookback_buffer_size:
Number of window-length audio buffers used as a lookback buffer.
Higher values increase prediction stability but add latency.
Range: 1.0 .. 20.0, Default (SDK): 6.0
- speech_hold_duration:
How long VAD continues detecting after speech ends (in seconds).
Range: 0.0 to 100x model window length, Default (SDK): 0.05s
- minimum_speech_duration:
Minimum duration of speech required before VAD reports speech detected
(in seconds). Range: 0.0 to 1.0, Default (SDK): 0.0s
- sensitivity:
Energy threshold sensitivity. Energy threshold = 10 ** (-sensitivity).
Range: 1.0 .. 15.0, Default (SDK): 6.0
Range: 1.0 to 15.0, Default (SDK): 6.0
Args:
lookback_buffer_size: Optional lookback buffer size to configure on the VAD.
Range: 1.0 .. 20.0. If None, SDK default is used.
speech_hold_duration: Optional speech hold duration to configure on the VAD.
If None, SDK default (0.05s) is used.
minimum_speech_duration: Optional minimum speech duration before VAD reports
speech detected. If None, SDK default (0.0s) is used.
sensitivity: Optional sensitivity (energy threshold) to configure on the VAD.
Range: 1.0 .. 15.0. If None, SDK default is used.
Range: 1.0 to 15.0. If None, SDK default (6.0) is used.
Returns:
A lazily-initialized AICVADAnalyzer that will bind to the VAD backend
once the filter's model has been created (after start(sample_rate)).
A lazily-initialized AICVADAnalyzer that will bind to the VAD context
once the filter's processor has been created (after start(sample_rate)).
"""
from pipecat.audio.vad.aic_vad import AICVADAnalyzer
return AICVADAnalyzer(
vad_factory=self.get_vad_factory(),
lookback_buffer_size=lookback_buffer_size,
vad_context_factory=lambda: self.get_vad_context(),
speech_hold_duration=speech_hold_duration,
minimum_speech_duration=minimum_speech_duration,
sensitivity=sensitivity,
)
@@ -146,52 +162,83 @@ class AICFilter(BaseAudioFilter):
"""
self._sample_rate = sample_rate
# Load or download model
if self._model_path:
logger.debug(f"Loading AIC model from: {self._model_path}")
self._model = Model.from_file(str(self._model_path))
else:
logger.debug(f"Downloading AIC model: {self._model_id}")
self._model_download_dir.mkdir(parents=True, exist_ok=True)
model_path = await Model.download_async(self._model_id, str(self._model_download_dir))
logger.debug(f"Model downloaded to: {model_path}")
self._model = Model.from_file(model_path)
# Get optimal frames for this sample rate
self._frames_per_block = self._model.get_optimal_num_frames(self._sample_rate)
# Allocate processing buffers now that we know the block size
self._in_f32 = np.zeros((1, self._frames_per_block), dtype=np.float32)
self._out_i16 = np.zeros(self._frames_per_block, dtype=np.int16)
# Create configuration
config = ProcessorConfig.optimal(
self._model,
sample_rate=self._sample_rate,
)
# Create async processor
try:
# Create model with required runtime parameters
self._aic = Model(
model_type=self._model_type,
license_key=self._license_key or None,
sample_rate=self._sample_rate,
channels=1,
)
self._frames_per_block = self._aic.optimal_num_frames()
# Optional parameter configuration
if self._enhancement_level is not None:
self._aic.set_parameter(
AICParameter.ENHANCEMENT_LEVEL,
float(self._enhancement_level if self._enabled else 0.0),
)
if self._voice_gain is not None:
self._aic.set_parameter(AICParameter.VOICE_GAIN, float(self._voice_gain))
self._aic_ready = True
# Log processor information
logger.debug(f"ai-coustics filter started:")
logger.debug(f" Sample rate: {self._sample_rate} Hz")
logger.debug(f" Frames per chunk: {self._frames_per_block}")
logger.debug(f" Enhancement strength: {int(self._enhancement_level * 100)}%")
logger.debug(f" Optimal input buffer size: {self._aic.optimal_num_frames()} samples")
logger.debug(f" Optimal sample rate: {self._aic.optimal_sample_rate()} Hz")
logger.debug(
f" Current algorithmic latency: {self._aic.processing_latency() / self._sample_rate * 1000:.2f}ms"
)
self._processor = ProcessorAsync(self._model, self._license_key, config)
except Exception as e: # noqa: BLE001 - surfacing SDK initialization errors
logger.error(f"AIC model initialization failed: {e}")
self._aic_ready = False
self._processor = None
self._aic_ready = self._processor is not None
if not self._aic_ready:
logger.debug(f"ai-coustics filter is not ready.")
return
# Get contexts for parameter control and VAD
self._processor_ctx = self._processor.get_processor_context()
self._vad_ctx = self._processor.get_vad_context()
# Apply initial parameters
try:
self._processor_ctx.set_parameter(
ProcessorParameter.Bypass, 1.0 if self._bypass else 0.0
)
except ParameterFixedError as e:
logger.error(f"AIC parameter update failed: {e}")
# Log processor information
logger.debug(f"ai-coustics filter started:")
logger.debug(f" Model ID: {self._model.get_id()}")
logger.debug(f" Sample rate: {self._sample_rate} Hz")
logger.debug(f" Frames per chunk: {self._frames_per_block}")
logger.debug(f" Optimal sample rate: {self._model.get_optimal_sample_rate()} Hz")
logger.debug(
f" Optimal number of frames for {self._sample_rate} Hz: {self._model.get_optimal_num_frames(self._sample_rate)}"
)
logger.debug(
f" Output delay: {self._processor_ctx.get_output_delay()} samples "
f"({self._processor_ctx.get_output_delay() / self._sample_rate * 1000:.2f}ms)"
)
async def stop(self):
"""Clean up the AIC model when stopping.
"""Clean up the AIC processor when stopping.
Returns:
None
"""
try:
if self._aic is not None:
self._aic.close()
if self._processor_ctx is not None:
self._processor_ctx.reset()
finally:
self._aic = None
self._processor = None
self._processor_ctx = None
self._vad_ctx = None
self._model = None
self._aic_ready = False
self._audio_buffer.clear()
@@ -205,11 +252,12 @@ class AICFilter(BaseAudioFilter):
None
"""
if isinstance(frame, FilterEnableFrame):
self._enabled = frame.enable
if self._aic is not None:
self._bypass = not frame.enable
if self._processor_ctx is not None:
try:
level = float(self._enhancement_level if self._enabled else 0.0)
self._aic.set_parameter(AICParameter.ENHANCEMENT_LEVEL, level)
self._processor_ctx.set_parameter(
ProcessorParameter.Bypass, 1.0 if self._bypass else 0.0
)
except Exception as e: # noqa: BLE001
logger.error(f"AIC set_parameter failed: {e}")
@@ -220,43 +268,41 @@ class AICFilter(BaseAudioFilter):
model's required block length. Returns enhanced audio data.
Args:
audio: Raw audio data as bytes to be filtered (int16 PCM, planar).
audio: Raw audio data as bytes (int16 PCM).
Returns:
Enhanced audio data as bytes (int16 PCM, planar).
Enhanced audio data as bytes (int16 PCM).
"""
if not self._aic_ready or self._aic is None:
if not self._aic_ready or self._processor is None:
return audio
self._audio_buffer.extend(audio)
available_frames = len(self._audio_buffer) // self._bytes_per_sample
num_blocks = available_frames // self._frames_per_block
if num_blocks == 0:
return b""
filtered_chunks: List[bytes] = []
mv = memoryview(self._audio_buffer)
block_size = self._frames_per_block * self._bytes_per_sample
# Number of int16 samples currently buffered
available_frames = len(self._audio_buffer) // 2
for i in range(num_blocks):
start = i * block_size
block_i16 = np.frombuffer(mv[start : start + block_size], dtype=self._dtype)
while available_frames >= self._frames_per_block:
# Consume exactly one block worth of frames
samples_to_consume = self._frames_per_block * 1
bytes_to_consume = samples_to_consume * 2
block_bytes = bytes(self._audio_buffer[:bytes_to_consume])
# Reuse input buffer, in-place divide
np.copyto(self._in_f32[0], block_i16)
self._in_f32 /= self._scale
# Convert to float32 in -1..+1 range and reshape to planar (channels, frames)
block_i16 = np.frombuffer(block_bytes, dtype=np.int16)
block_f32 = (block_i16.astype(np.float32) / 32768.0).reshape(
(1, self._frames_per_block)
)
out_f32 = await self._processor.process_async(self._in_f32)
# Process planar in-place; returns ndarray (same shape)
out_f32 = await self._aic.process_async(block_f32)
# Convert float32 output back to int16
np.multiply(out_f32, self._scale, out=self._in_f32) # reuse in_f32 as temp
np.clip(self._in_f32, -self._scale, self._scale - 1, out=self._in_f32)
np.copyto(self._out_i16, self._in_f32[0].astype(self._dtype))
# Convert back to int16 bytes, planar layout
out_i16 = np.clip(out_f32 * 32768.0, -32768, 32767).astype(np.int16)
filtered_chunks.append(out_i16.reshape(-1).tobytes())
filtered_chunks.append(self._out_i16.tobytes())
# Slide buffer
self._audio_buffer = self._audio_buffer[bytes_to_consume:]
available_frames = len(self._audio_buffer) // 2
# Do not flush incomplete frames; keep them buffered for the next call
self._audio_buffer = self._audio_buffer[num_blocks * block_size :]
return b"".join(filtered_chunks)

View File

@@ -1,44 +1,44 @@
"""AIC-integrated VAD analyzer that lazily binds to the AIC SDK backend.
This analyzer queries the backend's is_speech_detected() and maps it to a float
confidence (1.0/0.0). It uses 10 ms windows based on the sample rate and applies
optional AIC VAD parameters (lookback_buffer_size, sensitivity) when available.
This module provides VAD analyzer implementations that query the AIC SDK's
is_speech_detected() and map it to a float confidence (1.0/0.0).
Classes:
AICVADAnalyzer: For aic-sdk (uses 'aic_sdk' module)
"""
from typing import Any, Callable, Optional
from aic_sdk import VadParameter
from loguru import logger
from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams
try:
from aic import AICVadParameter
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use the AIC filter, you need to `pip install pipecat-ai[aic]`.")
raise Exception(f"Missing module: {e}")
class AICVADAnalyzer(VADAnalyzer):
"""VAD analyzer that lazily instantiates the AIC VoiceActivityDetector via a factory.
"""VAD analyzer that lazily binds to the AIC VadContext via a factory.
The analyzer can be constructed before the AIC Model exists. Once the filter has
started and the Model is available, the provided factory will succeed and the
backend VAD will be created. We then switch to single-sample updates where
num_frames_required() returns 1 and confidence is derived from the backend's
boolean is_speech_detected() state.
The analyzer can be constructed before the AIC Processor exists. Once the filter has
started and the Processor is available, the provided factory will succeed and the
VadContext will be obtained. The context's is_speech_detected() boolean state is
then mapped to 1.0 (speech) or 0.0 (no speech) to satisfy the VADAnalyzer interface.
AIC VAD runtime parameters:
- lookback_buffer_size:
Controls the lookback buffer size used by the VAD, i.e. the number of
window-length audio buffers used as a lookback buffer. Larger values improve
stability but increase latency.
Range: 1.0 .. 20.0
Default (SDK): 6.0
- speech_hold_duration:
Controls for how long the VAD continues to detect speech after the audio signal
no longer contains speech (in seconds).
Range: 0.0 to 100x model window length
Default (SDK): 0.05s (50ms)
- minimum_speech_duration:
Controls for how long speech needs to be present in the audio signal before the
VAD considers it speech (in seconds).
Range: 0.0 to 1.0
Default (SDK): 0.0s
- sensitivity:
Controls the energy threshold sensitivity. Higher values make the detector
less sensitive (require more energy to count as speech).
Range: 1.0 .. 15.0
Controls the sensitivity (energy threshold) of the VAD. This value is used by
the VAD as the threshold a speech audio signal's energy has to exceed in order
to be considered speech.
Range: 1.0 to 15.0
Formula: Energy threshold = 10 ** (-sensitivity)
Default (SDK): 6.0
"""
@@ -46,69 +46,80 @@ class AICVADAnalyzer(VADAnalyzer):
def __init__(
self,
*,
vad_factory: Optional[Callable[[], Any]] = None,
lookback_buffer_size: Optional[float] = None,
vad_context_factory: Optional[Callable[[], Any]] = None,
speech_hold_duration: Optional[float] = None,
minimum_speech_duration: Optional[float] = None,
sensitivity: Optional[float] = None,
):
"""Create an AIC VAD analyzer.
Args:
vad_factory:
Zero-arg callable that returns an initialized AIC VoiceActivityDetector.
This may raise until the filter's Model has been created; the analyzer
vad_context_factory:
Zero-arg callable that returns the AIC VadContext.
This may raise until the filter's Processor has been created; the analyzer
will retry on set_sample_rate/first use.
lookback_buffer_size:
Optional override for AIC VAD lookback buffer size.
Range: 1.0 .. 20.0. Larger values increase stability at the cost of latency.
If None, the SDK default (6.0) is used.
speech_hold_duration:
Optional override for AIC VAD speech hold duration (in seconds).
Range: 0.0 to 100x model window length.
If None, the SDK default (0.05s) is used.
minimum_speech_duration:
Optional override for minimum speech duration before VAD reports
speech detected (in seconds).
Range: 0.0 to 1.0.
If None, the SDK default (0.0s) is used.
sensitivity:
Optional override for AIC VAD sensitivity (energy threshold).
Range: 1.0 .. 15.0. Energy threshold = 10 ** (-sensitivity).
Range: 1.0 to 15.0. Energy threshold = 10 ** (-sensitivity).
If None, the SDK default (6.0) is used.
"""
# Use fixed VAD parameters for AIC: no user override
fixed_params = VADParams(confidence=0.5, start_secs=0.0, stop_secs=0.0, min_volume=0.0)
super().__init__(sample_rate=None, params=fixed_params)
self._vad_factory = vad_factory
self._backend_vad: Optional[Any] = None
self._pending_lookback: Optional[float] = lookback_buffer_size
self._vad_context_factory = vad_context_factory
self._vad_ctx: Optional[Any] = None
self._pending_speech_hold_duration: Optional[float] = speech_hold_duration
self._pending_minimum_speech_duration: Optional[float] = minimum_speech_duration
self._pending_sensitivity: Optional[float] = sensitivity
def bind_vad_factory(self, vad_factory: Callable[[], Any]):
def bind_vad_context_factory(self, vad_context_factory: Callable[[], Any]):
"""Attach or replace the factory post-construction."""
self._vad_factory = vad_factory
self._ensure_backend_initialized()
self._vad_context_factory = vad_context_factory
self._ensure_vad_context_initialized()
def _apply_backend_params(self):
def _apply_vad_params(self):
"""Apply optional AIC VAD parameters if available."""
if self._backend_vad is None or AICVadParameter is None:
if self._vad_ctx is None or VadParameter is None:
return
try:
if self._pending_lookback is not None:
self._backend_vad.set_parameter(
AICVadParameter.LOOKBACK_BUFFER_SIZE, float(self._pending_lookback)
if self._pending_speech_hold_duration is not None:
self._vad_ctx.set_parameter(
VadParameter.SpeechHoldDuration, self._pending_speech_hold_duration
)
if self._pending_minimum_speech_duration is not None:
self._vad_ctx.set_parameter(
VadParameter.MinimumSpeechDuration, self._pending_minimum_speech_duration
)
if self._pending_sensitivity is not None:
self._backend_vad.set_parameter(
AICVadParameter.SENSITIVITY, float(self._pending_sensitivity)
)
self._vad_ctx.set_parameter(VadParameter.Sensitivity, self._pending_sensitivity)
except Exception as e: # noqa: BLE001
logger.debug(f"AIC VAD parameter application deferred/failed: {e}")
def _ensure_backend_initialized(self):
if self._backend_vad is not None:
def _ensure_vad_context_initialized(self):
if self._vad_ctx is not None:
return
if not self._vad_factory:
if not self._vad_context_factory:
return
try:
self._backend_vad = self._vad_factory()
self._apply_backend_params()
# With backend ready, recompute internal frame sizing
self._vad_ctx = self._vad_context_factory()
self._apply_vad_params()
# With VAD context ready, recompute internal frame sizing
super().set_params(self._params)
logger.debug("AIC VAD backend initialized in analyzer.")
logger.debug("AIC VAD context initialized in analyzer.")
except Exception as e: # noqa: BLE001
# Filter may not be started yet; try again later
logger.debug(f"Deferring AIC VAD backend initialization: {e}")
logger.debug(f"Deferring AIC VAD context initialization: {e}")
def set_sample_rate(self, sample_rate: int):
"""Set the sample rate for audio processing.
@@ -116,10 +127,10 @@ class AICVADAnalyzer(VADAnalyzer):
Args:
sample_rate: Audio sample rate in Hz.
"""
# Set rate and attempt backend initialization once we know SR
# Set rate and attempt VAD context initialization once we know SR
self._sample_rate = self._init_sample_rate or sample_rate
self._ensure_backend_initialized()
# Ensure params are initialized even if backend not ready yet
self._ensure_vad_context_initialized()
# Ensure params are initialized even if VAD context not ready yet
try:
super().set_params(self._params)
except Exception:
@@ -135,23 +146,29 @@ class AICVADAnalyzer(VADAnalyzer):
return int(self.sample_rate * 0.01) if self.sample_rate > 0 else 160
def voice_confidence(self, buffer: bytes) -> float:
"""Calculate voice activity confidence for the given audio buffer.
"""Return voice activity detection result for the given audio buffer.
Note:
The AIC SDK provides binary speech detection (not a probability score).
This method returns 1.0 when speech is detected and 0.0 otherwise,
rather than a true confidence value.
Args:
buffer: Audio buffer to analyze.
buffer: Audio buffer (unused - AIC VAD state is updated internally
by the enhancement pipeline).
Returns:
Voice confidence score is 0.0 or 1.0.
1.0 if speech is detected, 0.0 otherwise.
"""
# Ensure backend exists (filter might have started since last call)
self._ensure_backend_initialized()
if self._backend_vad is None:
# Ensure VAD context exists (filter might have started since last call)
self._ensure_vad_context_initialized()
if self._vad_ctx is None:
return 0.0
# We do not need to analyze 'buffer' here since the model's VAD is updated
# We do not need to analyze 'buffer' here since the processor's VAD is updated
# as part of the enhancement pipeline. Simply query the boolean and map it.
try:
is_speech = self._backend_vad.is_speech_detected()
is_speech = self._vad_ctx.is_speech_detected()
return 1.0 if is_speech else 0.0
except Exception as e: # noqa: BLE001
logger.error(f"AIC VAD inference error: {e}")