address feedback.
This commit is contained in:
@@ -11,7 +11,7 @@ enhance audio streams in real time. It mirrors the structure of other filters li
|
|||||||
the Koala filter and integrates with Pipecat's input transport pipeline.
|
the Koala filter and integrates with Pipecat's input transport pipeline.
|
||||||
|
|
||||||
Classes:
|
Classes:
|
||||||
AICFilter: For aic-sdk >= 2.0.0 (uses 'aic_sdk' module)
|
AICFilter: For aic-sdk (uses 'aic_sdk' module)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
@@ -30,7 +30,7 @@ class AICFilter(BaseAudioFilter):
|
|||||||
"""Audio filter using ai-coustics' AIC SDK for real-time enhancement.
|
"""Audio filter using ai-coustics' AIC SDK for real-time enhancement.
|
||||||
|
|
||||||
Buffers incoming audio to the model's preferred block size and processes
|
Buffers incoming audio to the model's preferred block size and processes
|
||||||
planar frames in-place using float32 samples in the linear -1..+1 range.
|
frames using float32 samples normalized to the -1..+1 range.
|
||||||
|
|
||||||
.. note::
|
.. note::
|
||||||
This class requires aic-sdk >= 2.0.0 (uses 'aic_sdk' module).
|
This class requires aic-sdk >= 2.0.0 (uses 'aic_sdk' module).
|
||||||
@@ -84,12 +84,21 @@ class AICFilter(BaseAudioFilter):
|
|||||||
self._frames_per_block = 0
|
self._frames_per_block = 0
|
||||||
self._audio_buffer = bytearray()
|
self._audio_buffer = bytearray()
|
||||||
|
|
||||||
|
# Audio format constants
|
||||||
|
self._bytes_per_sample = 2 # int16 = 2 bytes
|
||||||
|
self._dtype = np.int16
|
||||||
|
self._scale = 32768.0 # 2^15, for normalizing int16 (-32768..32767) to float32 (-1.0..1.0)
|
||||||
|
|
||||||
# AIC SDK objects
|
# AIC SDK objects
|
||||||
self._model = None
|
self._model = None
|
||||||
self._processor = None
|
self._processor = None
|
||||||
self._processor_ctx = None
|
self._processor_ctx = None
|
||||||
self._vad_ctx = None
|
self._vad_ctx = None
|
||||||
|
|
||||||
|
# Pre-allocated buffers (resized in start() once frames_per_block is known)
|
||||||
|
self._in_f32 = None
|
||||||
|
self._out_i16 = None
|
||||||
|
|
||||||
def get_vad_context(self):
|
def get_vad_context(self):
|
||||||
"""Return the VAD context once the processor exists.
|
"""Return the VAD context once the processor exists.
|
||||||
|
|
||||||
@@ -156,6 +165,13 @@ class AICFilter(BaseAudioFilter):
|
|||||||
logger.debug(f"Model downloaded to: {model_path}")
|
logger.debug(f"Model downloaded to: {model_path}")
|
||||||
self._model = Model.from_file(model_path)
|
self._model = Model.from_file(model_path)
|
||||||
|
|
||||||
|
# Get optimal frames for this sample rate
|
||||||
|
self._frames_per_block = self._model.get_optimal_num_frames(self._sample_rate)
|
||||||
|
|
||||||
|
# Allocate processing buffers now that we know the block size
|
||||||
|
self._in_f32 = np.zeros((1, self._frames_per_block), dtype=np.float32)
|
||||||
|
self._out_i16 = np.zeros(self._frames_per_block, dtype=np.int16)
|
||||||
|
|
||||||
# Create configuration
|
# Create configuration
|
||||||
config = ProcessorConfig.optimal(
|
config = ProcessorConfig.optimal(
|
||||||
self._model,
|
self._model,
|
||||||
@@ -163,7 +179,7 @@ class AICFilter(BaseAudioFilter):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Create async processor
|
# Create async processor
|
||||||
self._processor = ProcessorAsync(self._model, self._license_key or "", config)
|
self._processor = ProcessorAsync(self._model, self._license_key, config)
|
||||||
|
|
||||||
# Get contexts for parameter control and VAD
|
# Get contexts for parameter control and VAD
|
||||||
self._processor_ctx = self._processor.get_processor_context()
|
self._processor_ctx = self._processor.get_processor_context()
|
||||||
@@ -171,12 +187,10 @@ class AICFilter(BaseAudioFilter):
|
|||||||
|
|
||||||
# Apply initial parameters
|
# Apply initial parameters
|
||||||
if self._enhancement_level is not None:
|
if self._enhancement_level is not None:
|
||||||
level = float(self._enhancement_level if self._enabled else 0.0)
|
level = self._enhancement_level if self._enabled else 0.0
|
||||||
self._processor_ctx.set_parameter(ProcessorParameter.EnhancementLevel, level)
|
self._processor_ctx.set_parameter(ProcessorParameter.EnhancementLevel, level)
|
||||||
if self._voice_gain is not None:
|
if self._voice_gain is not None:
|
||||||
self._processor_ctx.set_parameter(
|
self._processor_ctx.set_parameter(ProcessorParameter.VoiceGain, self._voice_gain)
|
||||||
ProcessorParameter.VoiceGain, float(self._voice_gain)
|
|
||||||
)
|
|
||||||
|
|
||||||
self._aic_ready = True
|
self._aic_ready = True
|
||||||
|
|
||||||
@@ -225,7 +239,7 @@ class AICFilter(BaseAudioFilter):
|
|||||||
self._enabled = frame.enable
|
self._enabled = frame.enable
|
||||||
if self._processor_ctx is not None:
|
if self._processor_ctx is not None:
|
||||||
try:
|
try:
|
||||||
level = float(self._enhancement_level if self._enabled else 0.0)
|
level = self._enhancement_level if self._enabled else 0.0
|
||||||
self._processor_ctx.set_parameter(ProcessorParameter.EnhancementLevel, level)
|
self._processor_ctx.set_parameter(ProcessorParameter.EnhancementLevel, level)
|
||||||
except Exception as e: # noqa: BLE001
|
except Exception as e: # noqa: BLE001
|
||||||
logger.error(f"AIC set_parameter failed: {e}")
|
logger.error(f"AIC set_parameter failed: {e}")
|
||||||
@@ -237,52 +251,41 @@ class AICFilter(BaseAudioFilter):
|
|||||||
model's required block length. Returns enhanced audio data.
|
model's required block length. Returns enhanced audio data.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
audio: Raw audio data as bytes to be filtered (int16 PCM, planar).
|
audio: Raw audio data as bytes (int16 PCM).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Enhanced audio data as bytes (int16 PCM, planar).
|
Enhanced audio data as bytes (int16 PCM).
|
||||||
"""
|
"""
|
||||||
if not self._aic_ready or self._processor is None:
|
if not self._aic_ready or self._processor is None:
|
||||||
return audio
|
return audio
|
||||||
|
|
||||||
self._audio_buffer.extend(audio)
|
self._audio_buffer.extend(audio)
|
||||||
|
available_frames = len(self._audio_buffer) // self._bytes_per_sample
|
||||||
|
num_blocks = available_frames // self._frames_per_block
|
||||||
|
|
||||||
|
if num_blocks == 0:
|
||||||
|
return b""
|
||||||
|
|
||||||
filtered_chunks: List[bytes] = []
|
filtered_chunks: List[bytes] = []
|
||||||
|
mv = memoryview(self._audio_buffer)
|
||||||
|
block_size = self._frames_per_block * self._bytes_per_sample
|
||||||
|
|
||||||
# Number of int16 samples currently buffered
|
for i in range(num_blocks):
|
||||||
available_frames = len(self._audio_buffer) // 2
|
start = i * block_size
|
||||||
|
block_i16 = np.frombuffer(mv[start : start + block_size], dtype=self._dtype)
|
||||||
|
|
||||||
while available_frames >= self._frames_per_block:
|
# Reuse input buffer, in-place divide
|
||||||
# Consume exactly one block worth of frames
|
np.copyto(self._in_f32[0], block_i16)
|
||||||
samples_to_consume = self._frames_per_block * 1
|
self._in_f32 /= self._scale
|
||||||
bytes_to_consume = samples_to_consume * 2
|
|
||||||
block_bytes = bytes(self._audio_buffer[:bytes_to_consume])
|
|
||||||
|
|
||||||
# Convert to float32 in -1..+1 range and reshape to (channels, frames)
|
out_f32 = await self._processor.process_async(self._in_f32)
|
||||||
block_i16 = np.frombuffer(block_bytes, dtype=np.int16)
|
|
||||||
# Convert to float32 and normalize
|
|
||||||
block_f32 = block_i16.astype(np.float32)
|
|
||||||
|
|
||||||
block_f32 *= (1.0 / 32768.0)
|
# Convert float32 output back to int16
|
||||||
|
np.multiply(out_f32, self._scale, out=self._in_f32) # reuse in_f32 as temp
|
||||||
|
np.clip(self._in_f32, -self._scale, self._scale - 1, out=self._in_f32)
|
||||||
|
np.copyto(self._out_i16, self._in_f32[0].astype(self._dtype))
|
||||||
|
|
||||||
# Reshape to (1, frames) for AIC SDK
|
filtered_chunks.append(self._out_i16.tobytes())
|
||||||
block_f32 = block_f32.reshape((1, self._frames_per_block))
|
|
||||||
|
|
||||||
# Process via async processor; returns ndarray (same shape)
|
self._audio_buffer = self._audio_buffer[num_blocks * block_size :]
|
||||||
out_f32 = await self._processor.process_async(block_f32)
|
|
||||||
|
|
||||||
# Convert back to int16 bytes
|
|
||||||
# Denormalize and convert back to int16
|
|
||||||
out_f32 *= 32768.0
|
|
||||||
|
|
||||||
# In-place clip to valid int16 range (-32768 to 32767)
|
|
||||||
np.clip(out_f32, -32768.0, 32767, out=out_f32)
|
|
||||||
out_i16 = out_f32.astype(dtype)
|
|
||||||
filtered_chunks.append(out_i16.reshape(-1).tobytes())
|
|
||||||
|
|
||||||
# Slide buffer
|
|
||||||
self._audio_buffer = self._audio_buffer[bytes_to_consume:]
|
|
||||||
available_frames = len(self._audio_buffer) // 2
|
|
||||||
|
|
||||||
# Do not flush incomplete frames; keep them buffered for the next call
|
|
||||||
return b"".join(filtered_chunks)
|
return b"".join(filtered_chunks)
|
||||||
|
|||||||
Reference in New Issue
Block a user