diff --git a/changelog/3406.fixed.md b/changelog/3406.fixed.md new file mode 100644 index 000000000..b4eae4548 --- /dev/null +++ b/changelog/3406.fixed.md @@ -0,0 +1 @@ +- Fixed an issue where if you were using `OpenRouterLLMService` with a Gemini model, it wouldn't handle multiple `"system"` messages as expected (and as we do in `GoogleLLMService`), which is to convert subsequent ones into `"user"` messages. Instead, the latest `"system"` message would overwrite the previous ones. diff --git a/changelog/3408.added.md b/changelog/3408.added.md new file mode 100644 index 000000000..04f1311cc --- /dev/null +++ b/changelog/3408.added.md @@ -0,0 +1,4 @@ +- Additions for `AICFilter` and `AICVADAnalyzer`: + - Added model downloading support to `AICFilter` with `model_id` and `model_download_dir` parameters. + - Added `model_path` parameter to `AICFilter` for loading local `.aicmodel` files. + - Added unit tests for `AICFilter` and `AICVADAnalyzer`. diff --git a/changelog/3408.changed.md b/changelog/3408.changed.md new file mode 100644 index 000000000..9436b6074 --- /dev/null +++ b/changelog/3408.changed.md @@ -0,0 +1 @@ +- Updated `AICFilter` and `AICVADAnalyzer` to use aic-sdk ~= 2.0.1. diff --git a/changelog/3408.removed.md b/changelog/3408.removed.md new file mode 100644 index 000000000..f578bf5d0 --- /dev/null +++ b/changelog/3408.removed.md @@ -0,0 +1 @@ +- Removed deprecated `AICFilter` parameters: `enhancement_level`, `voice_gain`, `noise_gate_enable`. diff --git a/changelog/3536.fixed.md b/changelog/3536.fixed.md new file mode 100644 index 000000000..f70bd62b6 --- /dev/null +++ b/changelog/3536.fixed.md @@ -0,0 +1 @@ +- Fixed a logging issue where non-ASCII characters (e.g., Japanese, Chinese, etc.) were being unnecessarily escaped to Unicode sequences when function call occurred. diff --git a/changelog/3541.fixed.md b/changelog/3541.fixed.md new file mode 100644 index 000000000..ac5b59529 --- /dev/null +++ b/changelog/3541.fixed.md @@ -0,0 +1 @@ +- Fixed how audio tracks are synchronized inside the `AudioBufferProcessor` to fix timing issues where silence and audio were misaligned between user and bot buffers. diff --git a/changelog/3560.changed.md b/changelog/3560.changed.md new file mode 100644 index 000000000..b4ba6aa8b --- /dev/null +++ b/changelog/3560.changed.md @@ -0,0 +1 @@ +- `FrameSerializer` now subclasses from `BaseObject` to enable event support. diff --git a/changelog/3562.changed.md b/changelog/3562.changed.md new file mode 100644 index 000000000..533f37b2f --- /dev/null +++ b/changelog/3562.changed.md @@ -0,0 +1,2 @@ +- Added support for TTFS in `SpeechmaticsSTTService` and set the default mode to `EXTERNAL` to support Pipecat-controlled VAD. +- Changed dependency to `speechmatics-voice[smart]>=0.2.8` diff --git a/changelog/3567.fixed.md b/changelog/3567.fixed.md new file mode 100644 index 000000000..92ecabeca --- /dev/null +++ b/changelog/3567.fixed.md @@ -0,0 +1 @@ +- Fixed race condition in `OpenAIRealtimeBetaLLMService` that could cause an error when truncating the conversation. diff --git a/changelog/3574.fixed.md b/changelog/3574.fixed.md new file mode 100644 index 000000000..187f172b7 --- /dev/null +++ b/changelog/3574.fixed.md @@ -0,0 +1 @@ +- Fixed an infinite loop in `WebsocketService` that blocked the event loop when a remote server closed the connection gracefully. \ No newline at end of file diff --git a/changelog/3575.fixed.md b/changelog/3575.fixed.md new file mode 100644 index 000000000..03c42b1ad --- /dev/null +++ b/changelog/3575.fixed.md @@ -0,0 +1 @@ +- Fixed `LLMUserAggregator` and `LLMAssistantAggregator` not emitting pending transcripts via `on_user_turn_stopped` and `on_assistant_turn_stopped` events when the conversation ends (`EndFrame`) or is cancelled (`CancelFrame`). diff --git a/examples/foundational/07zd-interruptible-aicoustics.py b/examples/foundational/07zd-interruptible-aicoustics.py index 8a49e734f..ca205fc1d 100644 --- a/examples/foundational/07zd-interruptible-aicoustics.py +++ b/examples/foundational/07zd-interruptible-aicoustics.py @@ -50,7 +50,7 @@ def _create_aic_filter() -> AICFilter: return AICFilter( license_key=license_key, - enhancement_level=0.5, + model_id="quail-vf-l-16khz", ) @@ -62,7 +62,9 @@ transport_params = { lambda aic: DailyParams( audio_in_enabled=True, audio_out_enabled=True, - vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0), + vad_analyzer=aic.create_vad_analyzer( + speech_hold_duration=0.05, minimum_speech_duration=0.0, sensitivity=6.0 + ), audio_in_filter=aic, ) )(_create_aic_filter()), @@ -70,7 +72,9 @@ transport_params = { lambda aic: FastAPIWebsocketParams( audio_in_enabled=True, audio_out_enabled=True, - vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0), + vad_analyzer=aic.create_vad_analyzer( + speech_hold_duration=0.05, minimum_speech_duration=0.0, sensitivity=6.0 + ), audio_in_filter=aic, ) )(_create_aic_filter()), @@ -78,7 +82,9 @@ transport_params = { lambda aic: TransportParams( audio_in_enabled=True, audio_out_enabled=True, - vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0), + vad_analyzer=aic.create_vad_analyzer( + speech_hold_duration=0.05, minimum_speech_duration=0.0, sensitivity=6.0 + ), audio_in_filter=aic, ) )(_create_aic_filter()), diff --git a/examples/foundational/README.md b/examples/foundational/README.md index bc25d42ca..5800efff3 100644 --- a/examples/foundational/README.md +++ b/examples/foundational/README.md @@ -4,7 +4,7 @@ This directory contains examples showing how to build voice and multimodal agent ## Setup -1. Follow the [README](../../README.md#%EF%B8%8F-contributing-to-the-framework) steps to get your local environment configured. +1. Follow the [README](https://github.com/pipecat-ai/pipecat/blob/main/README.md#%EF%B8%8F-contributing-to-the-framework) steps to get your local environment configured. > **Run from root directory**: Make sure you are running the steps from the root directory. @@ -140,4 +140,4 @@ uv run python --host 0.0.0.0 --port 8080 - **Connection errors**: Verify API keys in `.env` file - **Port conflicts**: Use `--port` to change the port -For more examples, visit our the [`pipecat-examples repository](https://github.com/pipecat-ai/pipecat-examples). +For more examples, visit our the [pipecat-examples repository](https://github.com/pipecat-ai/pipecat-examples). diff --git a/pyproject.toml b/pyproject.toml index e99ab62bc..339b498f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,13 +48,13 @@ Issues = "https://github.com/pipecat-ai/pipecat/issues" Changelog = "https://github.com/pipecat-ai/pipecat/blob/main/CHANGELOG.md" [project.optional-dependencies] -aic = [ "aic-sdk~=1.2.0" ] +aic = [ "aic-sdk~=2.0.1" ] anthropic = [ "anthropic~=0.49.0" ] assemblyai = [ "pipecat-ai[websockets-base]" ] asyncai = [ "pipecat-ai[websockets-base]" ] aws = [ "aioboto3~=15.5.0", "pipecat-ai[websockets-base]" ] aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.2.0; python_version>='3.12'" ] -azure = [ "azure-cognitiveservices-speech~=1.44.0"] +azure = [ "azure-cognitiveservices-speech~=1.47.0"] cartesia = [ "cartesia~=2.0.3", "pipecat-ai[websockets-base]" ] camb = [ "camb-sdk>=1.5.4" ] cerebras = [] @@ -109,7 +109,7 @@ silero = [ "onnxruntime>=1.20.1,<2" ] simli = [ "simli-ai~=1.0.3"] soniox = [ "pipecat-ai[websockets-base]" ] soundfile = [ "soundfile~=0.13.1" ] -speechmatics = [ "speechmatics-voice[smart]>=0.2.6" ] +speechmatics = [ "speechmatics-voice[smart]>=0.2.8" ] strands = [ "strands-agents>=1.9.1,<2" ] tavus=[] together = [] diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index e544c732e..d4290be00 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -133,8 +133,7 @@ TESTS_07 = [ ("07zb-interruptible-inworld-http.py", EVAL_SIMPLE_MATH), ("07zc-interruptible-asyncai.py", EVAL_SIMPLE_MATH), ("07zc-interruptible-asyncai-http.py", EVAL_SIMPLE_MATH), - # Need license key to run - # ("07zd-interruptible-aicoustics.py", EVAL_SIMPLE_MATH), + ("07zd-interruptible-aicoustics.py", EVAL_SIMPLE_MATH), ("07ze-interruptible-hume.py", EVAL_SIMPLE_MATH), ("07zf-interruptible-gradium.py", EVAL_SIMPLE_MATH), ("07zg-interruptible-camb.py", EVAL_SIMPLE_MATH), diff --git a/src/pipecat/audio/filters/aic_filter.py b/src/pipecat/audio/filters/aic_filter.py index e4242521b..7f0626776 100644 --- a/src/pipecat/audio/filters/aic_filter.py +++ b/src/pipecat/audio/filters/aic_filter.py @@ -9,129 +9,145 @@ This module provides an audio filter implementation using ai-coustics' AIC SDK to enhance audio streams in real time. It mirrors the structure of other filters like the Koala filter and integrates with Pipecat's input transport pipeline. + +Classes: + AICFilter: For aic-sdk (uses 'aic_sdk' module) """ +from pathlib import Path from typing import List, Optional import numpy as np +from aic_sdk import ( + Model, + ParameterFixedError, + ProcessorAsync, + ProcessorConfig, + ProcessorParameter, + set_sdk_id, +) from loguru import logger from pipecat.audio.filters.base_audio_filter import BaseAudioFilter +from pipecat.audio.vad.aic_vad import AICVADAnalyzer from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame -try: - # AIC SDK (https://ai-coustics.github.io/aic-sdk-py/api/) - from aic import AICModelType, AICParameter, Model -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error("In order to use the AIC filter, you need to `pip install pipecat-ai[aic]`.") - raise Exception(f"Missing module: {e}") - class AICFilter(BaseAudioFilter): """Audio filter using ai-coustics' AIC SDK for real-time enhancement. Buffers incoming audio to the model's preferred block size and processes - planar frames in-place using float32 samples in the linear -1..+1 range. + frames using float32 samples normalized to the range -1 to +1. """ def __init__( self, *, - license_key: str = "", - model_type: AICModelType = AICModelType.QUAIL_STT, - enhancement_level: Optional[float] = 1.0, - voice_gain: Optional[float] = 1.0, - noise_gate_enable: Optional[bool] = True, + license_key: str, + model_id: Optional[str] = None, + model_path: Optional[Path] = None, + model_download_dir: Optional[Path] = None, ) -> None: """Initialize the AIC filter. Args: license_key: ai-coustics license key for authentication. - model_type: Model variant to load. - enhancement_level: Optional overall enhancement strength (0.0..1.0). - voice_gain: Optional linear gain applied to detected speech (0.0..4.0). - noise_gate_enable: Optional enable/disable noise gate (default: True). + model_id: Model identifier to download from CDN. Required if model_path + is not provided. See https://artifacts.ai-coustics.io/ for available models. + model_path: Optional path to a local .aicmodel file. If provided, + model_id is ignored and no download occurs. + model_download_dir: Directory for downloading models as a Path object. + Defaults to a cache directory in user's home folder. - .. deprecated:: 1.3.0 - The `noise_gate_enable` parameter is deprecated and no longer has any effect. - It will be removed in a future version. + Raises: + ValueError: If neither model_id nor model_path is provided. """ + # Set SDK ID for telemetry identification (6 = pipecat) + set_sdk_id(6) + + if model_id is None and model_path is None: + raise ValueError( + "Either 'model_id' or 'model_path' must be provided. " + "See https://artifacts.ai-coustics.io/ for available models." + ) + self._license_key = license_key - self._model_type = model_type + self._model_id = model_id + self._model_path = model_path + self._model_download_dir = model_download_dir or ( + Path.home() / ".cache" / "pipecat" / "aic-models" + ) - self._enhancement_level = enhancement_level - self._voice_gain = voice_gain - if noise_gate_enable is not None: - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "Parameter `noise_gate_enable` is deprecated and no longer has any effect. " - "It will be removed in a future version. Use AIC VAD instead (create_vad_analyzer()).", - DeprecationWarning, - ) - - self._noise_gate_enable = noise_gate_enable - - self._enabled = True + self._bypass = False self._sample_rate = 0 self._aic_ready = False self._frames_per_block = 0 self._audio_buffer = bytearray() - # Model will be created in start() since the API now requires sample_rate - self._aic = None - def get_vad_factory(self): - """Return a zero-arg factory that will create the VAD once the model exists. + # Audio format constants + self._bytes_per_sample = 2 # int16 = 2 bytes + self._dtype = np.int16 + self._scale = ( + 32768.0 # 2^15, for normalizing int16 (-32768 to 32767) to float32 (-1.0 to 1.0) + ) + + # AIC SDK objects + self._model = None + self._processor = None + self._processor_ctx = None + self._vad_ctx = None + + # Pre-allocated buffers (resized in start() once frames_per_block is known) + self._in_f32 = None + self._out_i16 = None + + def get_vad_context(self): + """Return the VAD context once the processor exists. Returns: - A zero-argument callable that, when invoked, returns an initialized - VoiceActivityDetector bound to the underlying AIC model. Raises a - RuntimeError if the model has not been initialized (i.e. start() - has not been called successfully). + The VadContext instance bound to the underlying processor. + Raises RuntimeError if the processor has not been initialized. """ - - def _factory(): - if self._aic is None: - raise RuntimeError("AIC model not initialized yet. Call start(sample_rate) first.") - return self._aic.create_vad() - - return _factory + if self._vad_ctx is None: + raise RuntimeError("AIC processor not initialized yet. Call start(sample_rate) first.") + return self._vad_ctx def create_vad_analyzer( self, *, - lookback_buffer_size: Optional[float] = None, + speech_hold_duration: Optional[float] = None, + minimum_speech_duration: Optional[float] = None, sensitivity: Optional[float] = None, ): """Return an analyzer that will lazily instantiate the AIC VAD when ready. AIC VAD parameters: - - lookback_buffer_size: - Number of window-length audio buffers used as a lookback buffer. - Higher values increase prediction stability but add latency. - Range: 1.0 .. 20.0, Default (SDK): 6.0 + - speech_hold_duration: + How long VAD continues detecting after speech ends (in seconds). + Range: 0.0 to 100x model window length, Default (SDK): 0.05s + - minimum_speech_duration: + Minimum duration of speech required before VAD reports speech detected + (in seconds). Range: 0.0 to 1.0, Default (SDK): 0.0s - sensitivity: Energy threshold sensitivity. Energy threshold = 10 ** (-sensitivity). - Range: 1.0 .. 15.0, Default (SDK): 6.0 + Range: 1.0 to 15.0, Default (SDK): 6.0 Args: - lookback_buffer_size: Optional lookback buffer size to configure on the VAD. - Range: 1.0 .. 20.0. If None, SDK default is used. + speech_hold_duration: Optional speech hold duration to configure on the VAD. + If None, SDK default (0.05s) is used. + minimum_speech_duration: Optional minimum speech duration before VAD reports + speech detected. If None, SDK default (0.0s) is used. sensitivity: Optional sensitivity (energy threshold) to configure on the VAD. - Range: 1.0 .. 15.0. If None, SDK default is used. + Range: 1.0 to 15.0. If None, SDK default (6.0) is used. Returns: - A lazily-initialized AICVADAnalyzer that will bind to the VAD backend - once the filter's model has been created (after start(sample_rate)). + A lazily-initialized AICVADAnalyzer that will bind to the VAD context + once the filter's processor has been created (after start(sample_rate)). """ - from pipecat.audio.vad.aic_vad import AICVADAnalyzer - return AICVADAnalyzer( - vad_factory=self.get_vad_factory(), - lookback_buffer_size=lookback_buffer_size, + vad_context_factory=lambda: self.get_vad_context(), + speech_hold_duration=speech_hold_duration, + minimum_speech_duration=minimum_speech_duration, sensitivity=sensitivity, ) @@ -146,52 +162,83 @@ class AICFilter(BaseAudioFilter): """ self._sample_rate = sample_rate + # Load or download model + if self._model_path: + logger.debug(f"Loading AIC model from: {self._model_path}") + self._model = Model.from_file(str(self._model_path)) + else: + logger.debug(f"Downloading AIC model: {self._model_id}") + self._model_download_dir.mkdir(parents=True, exist_ok=True) + model_path = await Model.download_async(self._model_id, str(self._model_download_dir)) + logger.debug(f"Model downloaded to: {model_path}") + self._model = Model.from_file(model_path) + + # Get optimal frames for this sample rate + self._frames_per_block = self._model.get_optimal_num_frames(self._sample_rate) + + # Allocate processing buffers now that we know the block size + self._in_f32 = np.zeros((1, self._frames_per_block), dtype=np.float32) + self._out_i16 = np.zeros(self._frames_per_block, dtype=np.int16) + + # Create configuration + config = ProcessorConfig.optimal( + self._model, + sample_rate=self._sample_rate, + ) + + # Create async processor try: - # Create model with required runtime parameters - self._aic = Model( - model_type=self._model_type, - license_key=self._license_key or None, - sample_rate=self._sample_rate, - channels=1, - ) - self._frames_per_block = self._aic.optimal_num_frames() - - # Optional parameter configuration - if self._enhancement_level is not None: - self._aic.set_parameter( - AICParameter.ENHANCEMENT_LEVEL, - float(self._enhancement_level if self._enabled else 0.0), - ) - if self._voice_gain is not None: - self._aic.set_parameter(AICParameter.VOICE_GAIN, float(self._voice_gain)) - - self._aic_ready = True - - # Log processor information - logger.debug(f"ai-coustics filter started:") - logger.debug(f" Sample rate: {self._sample_rate} Hz") - logger.debug(f" Frames per chunk: {self._frames_per_block}") - logger.debug(f" Enhancement strength: {int(self._enhancement_level * 100)}%") - logger.debug(f" Optimal input buffer size: {self._aic.optimal_num_frames()} samples") - logger.debug(f" Optimal sample rate: {self._aic.optimal_sample_rate()} Hz") - logger.debug( - f" Current algorithmic latency: {self._aic.processing_latency() / self._sample_rate * 1000:.2f}ms" - ) + self._processor = ProcessorAsync(self._model, self._license_key, config) except Exception as e: # noqa: BLE001 - surfacing SDK initialization errors logger.error(f"AIC model initialization failed: {e}") - self._aic_ready = False + self._processor = None + + self._aic_ready = self._processor is not None + + if not self._aic_ready: + logger.debug(f"ai-coustics filter is not ready.") + return + + # Get contexts for parameter control and VAD + self._processor_ctx = self._processor.get_processor_context() + self._vad_ctx = self._processor.get_vad_context() + + # Apply initial parameters + try: + self._processor_ctx.set_parameter( + ProcessorParameter.Bypass, 1.0 if self._bypass else 0.0 + ) + except ParameterFixedError as e: + logger.error(f"AIC parameter update failed: {e}") + + # Log processor information + logger.debug(f"ai-coustics filter started:") + logger.debug(f" Model ID: {self._model.get_id()}") + logger.debug(f" Sample rate: {self._sample_rate} Hz") + logger.debug(f" Frames per chunk: {self._frames_per_block}") + logger.debug(f" Optimal sample rate: {self._model.get_optimal_sample_rate()} Hz") + logger.debug( + f" Optimal number of frames for {self._sample_rate} Hz: {self._model.get_optimal_num_frames(self._sample_rate)}" + ) + logger.debug( + f" Output delay: {self._processor_ctx.get_output_delay()} samples " + f"({self._processor_ctx.get_output_delay() / self._sample_rate * 1000:.2f}ms)" + ) async def stop(self): - """Clean up the AIC model when stopping. + """Clean up the AIC processor when stopping. Returns: None """ try: - if self._aic is not None: - self._aic.close() + if self._processor_ctx is not None: + self._processor_ctx.reset() finally: - self._aic = None + self._processor = None + self._processor_ctx = None + self._vad_ctx = None + self._model = None self._aic_ready = False self._audio_buffer.clear() @@ -205,11 +252,12 @@ class AICFilter(BaseAudioFilter): None """ if isinstance(frame, FilterEnableFrame): - self._enabled = frame.enable - if self._aic is not None: + self._bypass = not frame.enable + if self._processor_ctx is not None: try: - level = float(self._enhancement_level if self._enabled else 0.0) - self._aic.set_parameter(AICParameter.ENHANCEMENT_LEVEL, level) + self._processor_ctx.set_parameter( + ProcessorParameter.Bypass, 1.0 if self._bypass else 0.0 + ) except Exception as e: # noqa: BLE001 logger.error(f"AIC set_parameter failed: {e}") @@ -220,43 +268,41 @@ class AICFilter(BaseAudioFilter): model's required block length. Returns enhanced audio data. Args: - audio: Raw audio data as bytes to be filtered (int16 PCM, planar). + audio: Raw audio data as bytes (int16 PCM). Returns: - Enhanced audio data as bytes (int16 PCM, planar). + Enhanced audio data as bytes (int16 PCM). """ - if not self._aic_ready or self._aic is None: + if not self._aic_ready or self._processor is None: return audio self._audio_buffer.extend(audio) + available_frames = len(self._audio_buffer) // self._bytes_per_sample + num_blocks = available_frames // self._frames_per_block + + if num_blocks == 0: + return b"" filtered_chunks: List[bytes] = [] + mv = memoryview(self._audio_buffer) + block_size = self._frames_per_block * self._bytes_per_sample - # Number of int16 samples currently buffered - available_frames = len(self._audio_buffer) // 2 + for i in range(num_blocks): + start = i * block_size + block_i16 = np.frombuffer(mv[start : start + block_size], dtype=self._dtype) - while available_frames >= self._frames_per_block: - # Consume exactly one block worth of frames - samples_to_consume = self._frames_per_block * 1 - bytes_to_consume = samples_to_consume * 2 - block_bytes = bytes(self._audio_buffer[:bytes_to_consume]) + # Reuse input buffer, in-place divide + np.copyto(self._in_f32[0], block_i16) + self._in_f32 /= self._scale - # Convert to float32 in -1..+1 range and reshape to planar (channels, frames) - block_i16 = np.frombuffer(block_bytes, dtype=np.int16) - block_f32 = (block_i16.astype(np.float32) / 32768.0).reshape( - (1, self._frames_per_block) - ) + out_f32 = await self._processor.process_async(self._in_f32) - # Process planar in-place; returns ndarray (same shape) - out_f32 = await self._aic.process_async(block_f32) + # Convert float32 output back to int16 + np.multiply(out_f32, self._scale, out=self._in_f32) # reuse in_f32 as temp + np.clip(self._in_f32, -self._scale, self._scale - 1, out=self._in_f32) + np.copyto(self._out_i16, self._in_f32[0].astype(self._dtype)) - # Convert back to int16 bytes, planar layout - out_i16 = np.clip(out_f32 * 32768.0, -32768, 32767).astype(np.int16) - filtered_chunks.append(out_i16.reshape(-1).tobytes()) + filtered_chunks.append(self._out_i16.tobytes()) - # Slide buffer - self._audio_buffer = self._audio_buffer[bytes_to_consume:] - available_frames = len(self._audio_buffer) // 2 - - # Do not flush incomplete frames; keep them buffered for the next call + self._audio_buffer = self._audio_buffer[num_blocks * block_size :] return b"".join(filtered_chunks) diff --git a/src/pipecat/audio/vad/aic_vad.py b/src/pipecat/audio/vad/aic_vad.py index 4907e4f55..813029e2b 100644 --- a/src/pipecat/audio/vad/aic_vad.py +++ b/src/pipecat/audio/vad/aic_vad.py @@ -1,44 +1,44 @@ """AIC-integrated VAD analyzer that lazily binds to the AIC SDK backend. -This analyzer queries the backend's is_speech_detected() and maps it to a float -confidence (1.0/0.0). It uses 10 ms windows based on the sample rate and applies -optional AIC VAD parameters (lookback_buffer_size, sensitivity) when available. +This module provides VAD analyzer implementations that query the AIC SDK's +is_speech_detected() and map it to a float confidence (1.0/0.0). + +Classes: + AICVADAnalyzer: For aic-sdk (uses 'aic_sdk' module) """ from typing import Any, Callable, Optional +from aic_sdk import VadParameter from loguru import logger from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams -try: - from aic import AICVadParameter -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error("In order to use the AIC filter, you need to `pip install pipecat-ai[aic]`.") - raise Exception(f"Missing module: {e}") - class AICVADAnalyzer(VADAnalyzer): - """VAD analyzer that lazily instantiates the AIC VoiceActivityDetector via a factory. + """VAD analyzer that lazily binds to the AIC VadContext via a factory. - The analyzer can be constructed before the AIC Model exists. Once the filter has - started and the Model is available, the provided factory will succeed and the - backend VAD will be created. We then switch to single-sample updates where - num_frames_required() returns 1 and confidence is derived from the backend's - boolean is_speech_detected() state. + The analyzer can be constructed before the AIC Processor exists. Once the filter has + started and the Processor is available, the provided factory will succeed and the + VadContext will be obtained. The context's is_speech_detected() boolean state is + then mapped to 1.0 (speech) or 0.0 (no speech) to satisfy the VADAnalyzer interface. AIC VAD runtime parameters: - - lookback_buffer_size: - Controls the lookback buffer size used by the VAD, i.e. the number of - window-length audio buffers used as a lookback buffer. Larger values improve - stability but increase latency. - Range: 1.0 .. 20.0 - Default (SDK): 6.0 + - speech_hold_duration: + Controls for how long the VAD continues to detect speech after the audio signal + no longer contains speech (in seconds). + Range: 0.0 to 100x model window length + Default (SDK): 0.05s (50ms) + - minimum_speech_duration: + Controls for how long speech needs to be present in the audio signal before the + VAD considers it speech (in seconds). + Range: 0.0 to 1.0 + Default (SDK): 0.0s - sensitivity: - Controls the energy threshold sensitivity. Higher values make the detector - less sensitive (require more energy to count as speech). - Range: 1.0 .. 15.0 + Controls the sensitivity (energy threshold) of the VAD. This value is used by + the VAD as the threshold a speech audio signal's energy has to exceed in order + to be considered speech. + Range: 1.0 to 15.0 Formula: Energy threshold = 10 ** (-sensitivity) Default (SDK): 6.0 """ @@ -46,69 +46,80 @@ class AICVADAnalyzer(VADAnalyzer): def __init__( self, *, - vad_factory: Optional[Callable[[], Any]] = None, - lookback_buffer_size: Optional[float] = None, + vad_context_factory: Optional[Callable[[], Any]] = None, + speech_hold_duration: Optional[float] = None, + minimum_speech_duration: Optional[float] = None, sensitivity: Optional[float] = None, ): """Create an AIC VAD analyzer. Args: - vad_factory: - Zero-arg callable that returns an initialized AIC VoiceActivityDetector. - This may raise until the filter's Model has been created; the analyzer + vad_context_factory: + Zero-arg callable that returns the AIC VadContext. + This may raise until the filter's Processor has been created; the analyzer will retry on set_sample_rate/first use. - lookback_buffer_size: - Optional override for AIC VAD lookback buffer size. - Range: 1.0 .. 20.0. Larger values increase stability at the cost of latency. - If None, the SDK default (6.0) is used. + speech_hold_duration: + Optional override for AIC VAD speech hold duration (in seconds). + Range: 0.0 to 100x model window length. + If None, the SDK default (0.05s) is used. + minimum_speech_duration: + Optional override for minimum speech duration before VAD reports + speech detected (in seconds). + Range: 0.0 to 1.0. + If None, the SDK default (0.0s) is used. sensitivity: Optional override for AIC VAD sensitivity (energy threshold). - Range: 1.0 .. 15.0. Energy threshold = 10 ** (-sensitivity). + Range: 1.0 to 15.0. Energy threshold = 10 ** (-sensitivity). If None, the SDK default (6.0) is used. """ # Use fixed VAD parameters for AIC: no user override fixed_params = VADParams(confidence=0.5, start_secs=0.0, stop_secs=0.0, min_volume=0.0) super().__init__(sample_rate=None, params=fixed_params) - self._vad_factory = vad_factory - self._backend_vad: Optional[Any] = None - self._pending_lookback: Optional[float] = lookback_buffer_size + + self._vad_context_factory = vad_context_factory + self._vad_ctx: Optional[Any] = None + self._pending_speech_hold_duration: Optional[float] = speech_hold_duration + self._pending_minimum_speech_duration: Optional[float] = minimum_speech_duration self._pending_sensitivity: Optional[float] = sensitivity - def bind_vad_factory(self, vad_factory: Callable[[], Any]): + def bind_vad_context_factory(self, vad_context_factory: Callable[[], Any]): """Attach or replace the factory post-construction.""" - self._vad_factory = vad_factory - self._ensure_backend_initialized() + self._vad_context_factory = vad_context_factory + self._ensure_vad_context_initialized() - def _apply_backend_params(self): + def _apply_vad_params(self): """Apply optional AIC VAD parameters if available.""" - if self._backend_vad is None or AICVadParameter is None: + if self._vad_ctx is None or VadParameter is None: return + try: - if self._pending_lookback is not None: - self._backend_vad.set_parameter( - AICVadParameter.LOOKBACK_BUFFER_SIZE, float(self._pending_lookback) + if self._pending_speech_hold_duration is not None: + self._vad_ctx.set_parameter( + VadParameter.SpeechHoldDuration, self._pending_speech_hold_duration + ) + if self._pending_minimum_speech_duration is not None: + self._vad_ctx.set_parameter( + VadParameter.MinimumSpeechDuration, self._pending_minimum_speech_duration ) if self._pending_sensitivity is not None: - self._backend_vad.set_parameter( - AICVadParameter.SENSITIVITY, float(self._pending_sensitivity) - ) + self._vad_ctx.set_parameter(VadParameter.Sensitivity, self._pending_sensitivity) except Exception as e: # noqa: BLE001 logger.debug(f"AIC VAD parameter application deferred/failed: {e}") - def _ensure_backend_initialized(self): - if self._backend_vad is not None: + def _ensure_vad_context_initialized(self): + if self._vad_ctx is not None: return - if not self._vad_factory: + if not self._vad_context_factory: return try: - self._backend_vad = self._vad_factory() - self._apply_backend_params() - # With backend ready, recompute internal frame sizing + self._vad_ctx = self._vad_context_factory() + self._apply_vad_params() + # With VAD context ready, recompute internal frame sizing super().set_params(self._params) - logger.debug("AIC VAD backend initialized in analyzer.") + logger.debug("AIC VAD context initialized in analyzer.") except Exception as e: # noqa: BLE001 # Filter may not be started yet; try again later - logger.debug(f"Deferring AIC VAD backend initialization: {e}") + logger.debug(f"Deferring AIC VAD context initialization: {e}") def set_sample_rate(self, sample_rate: int): """Set the sample rate for audio processing. @@ -116,10 +127,10 @@ class AICVADAnalyzer(VADAnalyzer): Args: sample_rate: Audio sample rate in Hz. """ - # Set rate and attempt backend initialization once we know SR + # Set rate and attempt VAD context initialization once we know SR self._sample_rate = self._init_sample_rate or sample_rate - self._ensure_backend_initialized() - # Ensure params are initialized even if backend not ready yet + self._ensure_vad_context_initialized() + # Ensure params are initialized even if VAD context not ready yet try: super().set_params(self._params) except Exception: @@ -135,23 +146,29 @@ class AICVADAnalyzer(VADAnalyzer): return int(self.sample_rate * 0.01) if self.sample_rate > 0 else 160 def voice_confidence(self, buffer: bytes) -> float: - """Calculate voice activity confidence for the given audio buffer. + """Return voice activity detection result for the given audio buffer. + + Note: + The AIC SDK provides binary speech detection (not a probability score). + This method returns 1.0 when speech is detected and 0.0 otherwise, + rather than a true confidence value. Args: - buffer: Audio buffer to analyze. + buffer: Audio buffer (unused - AIC VAD state is updated internally + by the enhancement pipeline). Returns: - Voice confidence score is 0.0 or 1.0. + 1.0 if speech is detected, 0.0 otherwise. """ - # Ensure backend exists (filter might have started since last call) - self._ensure_backend_initialized() - if self._backend_vad is None: + # Ensure VAD context exists (filter might have started since last call) + self._ensure_vad_context_initialized() + if self._vad_ctx is None: return 0.0 - # We do not need to analyze 'buffer' here since the model's VAD is updated + # We do not need to analyze 'buffer' here since the processor's VAD is updated # as part of the enhancement pipeline. Simply query the boolean and map it. try: - is_speech = self._backend_vad.is_speech_detected() + is_speech = self._vad_ctx.is_speech_detected() return 1.0 if is_speech else 0.0 except Exception as e: # noqa: BLE001 logger.error(f"AIC VAD inference error: {e}") diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 81dbfe844..3a8bbb542 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -464,9 +464,11 @@ class LLMUserAggregator(LLMContextAggregator): await s.setup(self.task_manager) async def _stop(self, frame: EndFrame): + await self._maybe_emit_user_turn_stopped(on_session_end=True) await self._cleanup() async def _cancel(self, frame: CancelFrame): + await self._maybe_emit_user_turn_stopped(on_session_end=True) await self._cleanup() async def _cleanup(self): @@ -602,14 +604,7 @@ class LLMUserAggregator(LLMContextAggregator): if params.enable_user_speaking_frames: await self.broadcast_frame(UserStoppedSpeakingFrame) - # Always push context frame. - aggregation = await self.push_aggregation() - - message = UserTurnStoppedMessage( - content=aggregation, timestamp=self._user_turn_start_timestamp - ) - await self._call_event_handler("on_user_turn_stopped", strategy, message) - self._user_turn_start_timestamp = "" + await self._maybe_emit_user_turn_stopped(strategy) async def _on_user_turn_stop_timeout(self, controller): await self._call_event_handler("on_user_turn_stop_timeout") @@ -617,6 +612,26 @@ class LLMUserAggregator(LLMContextAggregator): async def _on_user_turn_idle(self, controller): await self._call_event_handler("on_user_turn_idle") + async def _maybe_emit_user_turn_stopped( + self, + strategy: Optional[BaseUserTurnStopStrategy] = None, + on_session_end: bool = False, + ): + """Maybe emit user turn stopped event. + + Args: + strategy: The strategy that triggered the turn stop. + on_session_end: If True, only emit if there's unemitted content + (avoids duplicate events when session ends). + """ + aggregation = await self.push_aggregation() + if not on_session_end or aggregation: + message = UserTurnStoppedMessage( + content=aggregation, timestamp=self._user_turn_start_timestamp + ) + await self._call_event_handler("on_user_turn_stopped", strategy, message) + self._user_turn_start_timestamp = "" + class LLMAssistantAggregator(LLMContextAggregator): """Assistant LLM aggregator that processes bot responses and function calls. @@ -739,6 +754,9 @@ class LLMAssistantAggregator(LLMContextAggregator): if isinstance(frame, InterruptionFrame): await self._handle_interruptions(frame) await self.push_frame(frame, direction) + elif isinstance(frame, (EndFrame, CancelFrame)): + await self._handle_end_or_cancel(frame) + await self.push_frame(frame, direction) elif isinstance(frame, LLMFullResponseStartFrame): await self._handle_llm_start(frame) elif isinstance(frame, LLMFullResponseEndFrame): @@ -813,6 +831,10 @@ class LLMAssistantAggregator(LLMContextAggregator): self._started = 0 await self.reset() + async def _handle_end_or_cancel(self, frame: Frame): + await self._trigger_assistant_turn_stopped() + self._started = 0 + async def _handle_function_calls_started(self, frame: FunctionCallsStartedFrame): function_names = [f"{f.function_name}:{f.tool_call_id}" for f in frame.function_calls] logger.debug(f"{self} FunctionCallsStartedFrame: {function_names}") @@ -833,7 +855,7 @@ class LLMAssistantAggregator(LLMContextAggregator): "id": frame.tool_call_id, "function": { "name": frame.function_name, - "arguments": json.dumps(frame.arguments), + "arguments": json.dumps(frame.arguments, ensure_ascii=False), }, "type": "function", } @@ -866,7 +888,7 @@ class LLMAssistantAggregator(LLMContextAggregator): # Update context with the function call result if frame.result: - result = json.dumps(frame.result) + result = json.dumps(frame.result, ensure_ascii=False) self._update_function_call_result(frame.function_name, frame.tool_call_id, result) else: self._update_function_call_result(frame.function_name, frame.tool_call_id, "COMPLETED") diff --git a/src/pipecat/processors/audio/audio_buffer_processor.py b/src/pipecat/processors/audio/audio_buffer_processor.py index 0d3ed76d4..ceadddf95 100644 --- a/src/pipecat/processors/audio/audio_buffer_processor.py +++ b/src/pipecat/processors/audio/audio_buffer_processor.py @@ -11,7 +11,6 @@ of audio from both user input and bot output sources, with support for various a configurations and event-driven processing. """ -import time from typing import Optional from pipecat.audio.utils import create_stream_resampler, interleave_stereo_audio, mix_audio @@ -104,10 +103,6 @@ class AudioBufferProcessor(FrameProcessor): self._user_turn_audio_buffer = bytearray() self._bot_turn_audio_buffer = bytearray() - # Intermittent (non continous user stream variables) - self._last_user_frame_at = 0 - self._last_bot_frame_at = 0 - self._recording = False self._input_resampler = create_stream_resampler() @@ -211,23 +206,31 @@ class AudioBufferProcessor(FrameProcessor): """Process audio frames for recording.""" resampled = None if isinstance(frame, InputAudioRawFrame): - # Add silence if we need to. - silence = self._compute_silence(self._last_user_frame_at) - self._user_audio_buffer.extend(silence) - # Add user audio. resampled = await self._resample_input_audio(frame) - self._user_audio_buffer.extend(resampled) - # Save time of frame so we can compute silence. - self._last_user_frame_at = time.time() + # Ignoring in case we don't have audio + if len(resampled) > 0: + # Sync bot buffer to current user position before adding user audio. + # We sync BEFORE extending to align both buffers at the same starting timestamp. + # For example, user buffer is at 100 bytes, and you receive 20 bytes of new audio + # - Bot buffer sees User is at 100. Bot pads itself to 100. + # - User buffer adds 20. User is now at 120. + # - Outcome: At index 100-120, we have User Audio and (potentially) Bot Audio or silence. They are aligned + # This gives the opportunity to the bot to send audio. + # + # If we synced AFTER, we'd pad the bot buffer with silence for the same + # window we just gave to the user, effectively "overwriting" that time slot + # with silence and causing the bot's audio to flicker or cut out. + self._sync_buffer_to_position(self._bot_audio_buffer, len(self._user_audio_buffer)) + # Add user audio. + self._user_audio_buffer.extend(resampled) elif self._recording and isinstance(frame, OutputAudioRawFrame): - # Add silence if we need to. - silence = self._compute_silence(self._last_bot_frame_at) - self._bot_audio_buffer.extend(silence) - # Add bot audio. resampled = await self._resample_output_audio(frame) - self._bot_audio_buffer.extend(resampled) - # Save time of frame so we can compute silence. - self._last_bot_frame_at = time.time() + # Ignoring in case we don't have audio + if len(resampled) > 0: + # Sync user buffer to current bot position before adding bot audio + self._sync_buffer_to_position(self._user_audio_buffer, len(self._bot_audio_buffer)) + # Add bot audio. + self._bot_audio_buffer.extend(resampled) if self._buffer_size > 0 and ( len(self._user_audio_buffer) >= self._buffer_size @@ -240,6 +243,21 @@ class AudioBufferProcessor(FrameProcessor): if self._enable_turn_audio: await self._process_turn_recording(frame, resampled) + def _sync_buffer_to_position(self, buffer: bytearray, target_position: int): + """Pad buffer with silence if it's behind the target position. + + This ensures both buffers stay synchronized by padding the lagging + buffer before new audio is added to the other buffer. + + Args: + buffer: The buffer to potentially pad. + target_position: The position (in bytes) the buffer should reach. + """ + current_len = len(buffer) + if current_len < target_position: + silence_needed = target_position - current_len + buffer.extend(b"\x00" * silence_needed) + async def _process_turn_recording(self, frame: Frame, resampled_audio: Optional[bytes] = None): """Process frames for turn-based audio recording.""" if isinstance(frame, UserStartedSpeakingFrame): @@ -281,8 +299,8 @@ class AudioBufferProcessor(FrameProcessor): if len(self._user_audio_buffer) == 0 and len(self._bot_audio_buffer) == 0: return + # Final alignment before we send the audio self._align_track_buffers() - flush_time = time.time() # Call original handler with merged audio merged_audio = self.merge_audio_buffers() @@ -299,9 +317,6 @@ class AudioBufferProcessor(FrameProcessor): self._num_channels, ) - self._last_user_frame_at = flush_time - self._last_bot_frame_at = flush_time - def _buffer_has_audio(self, buffer: bytearray) -> bool: """Check if a buffer contains audio data.""" return buffer is not None and len(buffer) > 0 @@ -309,8 +324,6 @@ class AudioBufferProcessor(FrameProcessor): def _reset_recording(self): """Reset recording state and buffers.""" self._reset_all_audio_buffers() - self._last_user_frame_at = time.time() - self._last_bot_frame_at = time.time() def _reset_all_audio_buffers(self): """Reset all audio buffers to empty state.""" @@ -336,11 +349,9 @@ class AudioBufferProcessor(FrameProcessor): target_len = max(user_len, bot_len) if user_len < target_len: - self._user_audio_buffer.extend(b"\x00" * (target_len - user_len)) - self._last_user_frame_at = max(self._last_user_frame_at, self._last_bot_frame_at) + self._sync_buffer_to_position(self._user_audio_buffer, target_len) if bot_len < target_len: - self._bot_audio_buffer.extend(b"\x00" * (target_len - bot_len)) - self._last_bot_frame_at = max(self._last_bot_frame_at, self._last_user_frame_at) + self._sync_buffer_to_position(self._bot_audio_buffer, target_len) async def _resample_input_audio(self, frame: InputAudioRawFrame) -> bytes: """Resample audio frame to the target sample rate.""" @@ -353,14 +364,3 @@ class AudioBufferProcessor(FrameProcessor): return await self._output_resampler.resample( frame.audio, frame.sample_rate, self._sample_rate ) - - def _compute_silence(self, from_time: float) -> bytes: - """Compute silence to insert based on time gap.""" - quiet_time = time.time() - from_time - # We should get audio frames very frequently. We introduce silence only - # if there's a big enough gap of 1s. - if from_time == 0 or quiet_time < 1.0: - return b"" - num_bytes = int(quiet_time * self._sample_rate) * 2 - silence = b"\x00" * num_bytes - return silence diff --git a/src/pipecat/serializers/base_serializer.py b/src/pipecat/serializers/base_serializer.py index 490951a69..9cc38cdc6 100644 --- a/src/pipecat/serializers/base_serializer.py +++ b/src/pipecat/serializers/base_serializer.py @@ -9,9 +9,10 @@ from abc import ABC, abstractmethod from pipecat.frames.frames import Frame, StartFrame +from pipecat.utils.base_object import BaseObject -class FrameSerializer(ABC): +class FrameSerializer(BaseObject): """Abstract base class for frame serialization implementations. Defines the interface for converting frames to/from serialized formats diff --git a/src/pipecat/services/azure/tts.py b/src/pipecat/services/azure/tts.py index 93c421c1e..dc0c56e7b 100644 --- a/src/pipecat/services/azure/tts.py +++ b/src/pipecat/services/azure/tts.py @@ -90,7 +90,7 @@ class AzureBaseTTSService: emphasis: Emphasis level for speech ("strong", "moderate", "reduced"). language: Language for synthesis. Defaults to English (US). pitch: Voice pitch adjustment (e.g., "+10%", "-5Hz", "high"). - rate: Speech rate multiplier. Defaults to "1.05". + rate: Speech rate adjustment (e.g., "1.0", "1.25", "slow", "fast"). role: Voice role for expression (e.g., "YoungAdultFemale"). style: Speaking style (e.g., "cheerful", "sad", "excited"). style_degree: Intensity of the speaking style (0.01 to 2.0). @@ -100,7 +100,7 @@ class AzureBaseTTSService: emphasis: Optional[str] = None language: Optional[Language] = Language.EN_US pitch: Optional[str] = None - rate: Optional[str] = "1.05" + rate: Optional[str] = None role: Optional[str] = None style: Optional[str] = None style_degree: Optional[str] = None @@ -185,7 +185,9 @@ class AzureBaseTTSService: if self._settings["volume"]: prosody_attrs.append(f"volume='{self._settings['volume']}'") - ssml += f"" + # Only wrap in prosody tag if there are prosody attributes + if prosody_attrs: + ssml += f"" if self._settings["emphasis"]: ssml += f"" @@ -195,7 +197,8 @@ class AzureBaseTTSService: if self._settings["emphasis"]: ssml += "" - ssml += "" + if prosody_attrs: + ssml += "" if self._settings["style"]: ssml += "" @@ -277,6 +280,11 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): self._started = False self._first_chunk = True self._cumulative_audio_offset: float = 0.0 # Cumulative audio duration in seconds + self._current_sentence_base_offset: float = 0.0 # Base offset for current sentence + self._current_sentence_duration: float = 0.0 # Duration from Azure callback + self._current_sentence_max_word_offset: float = ( + 0.0 # Max word boundary offset seen in current sentence (for 8kHz workaround) + ) self._last_word: Optional[str] = None # Track last word for punctuation merging self._last_timestamp: Optional[float] = None # Track last timestamp @@ -386,8 +394,14 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): word = evt.text sentence_relative_seconds = evt.audio_offset / 10_000_000.0 - # Add cumulative offset to get absolute timestamp across sentences - absolute_seconds = self._cumulative_audio_offset + sentence_relative_seconds + # Use base offset captured at start of run_tts to avoid race conditions + # with callbacks from overlapping TTS requests + absolute_seconds = self._current_sentence_base_offset + sentence_relative_seconds + + # Track max word offset for accurate cumulative timing + # (audio_duration from Azure doesn't always match word boundary offsets at 8kHz) + if sentence_relative_seconds > self._current_sentence_max_word_offset: + self._current_sentence_max_word_offset = sentence_relative_seconds if not word: return @@ -492,9 +506,9 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): self._last_word = None self._last_timestamp = None - # Update cumulative audio offset for next sentence + # Store duration for cumulative offset calculation if evt.result and evt.result.audio_duration: - self._cumulative_audio_offset += evt.result.audio_duration.total_seconds() + self._current_sentence_duration = evt.result.audio_duration.total_seconds() self._audio_queue.put_nowait(None) # Signal completion @@ -530,6 +544,9 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): self._started = False self._first_chunk = True self._cumulative_audio_offset = 0.0 + self._current_sentence_base_offset = 0.0 + self._current_sentence_duration = 0.0 + self._current_sentence_max_word_offset = 0.0 self._last_word = None self._last_timestamp = None @@ -604,6 +621,12 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): self._started = True self._first_chunk = True + # Capture base offset BEFORE starting synthesis to avoid race conditions + # Word boundary callbacks will use this value + self._current_sentence_base_offset = self._cumulative_audio_offset + self._current_sentence_duration = 0.0 + self._current_sentence_max_word_offset = 0.0 + ssml = self._construct_ssml(text) self._speech_synthesizer.speak_ssml_async(ssml) await self.start_tts_usage_metrics(text) @@ -627,6 +650,16 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): ) yield frame + # Update cumulative offset for next sentence + # At 8kHz, Azure's audio_duration doesn't match word boundary offsets, + # so we use max_word_offset as a workaround. At other sample rates, + # audio_duration is accurate. + # TODO: Remove after Azure fixes word boundary timing at 8kHz + if self.sample_rate == 8000: + self._cumulative_audio_offset += self._current_sentence_max_word_offset + else: + self._cumulative_audio_offset += self._current_sentence_duration + except Exception as e: yield ErrorFrame(error=f"Unknown error occurred: {e}") yield TTSStoppedFrame() diff --git a/src/pipecat/services/cartesia/stt.py b/src/pipecat/services/cartesia/stt.py index 2fb08b981..c0bfb3e8d 100644 --- a/src/pipecat/services/cartesia/stt.py +++ b/src/pipecat/services/cartesia/stt.py @@ -221,13 +221,10 @@ class CartesiaSTTService(WebsocketSTTService): await super().process_frame(frame, direction) if isinstance(frame, VADUserStartedSpeakingFrame): - # Reset finalize state for new utterance - self.set_finalize_pending(False) await self._start_metrics() elif isinstance(frame, VADUserStoppedSpeakingFrame): # Send finalize command to flush the transcription session if self._websocket and self._websocket.state is State.OPEN: - self.set_finalize_pending(True) await self._websocket.send("finalize") async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: diff --git a/src/pipecat/services/elevenlabs/stt.py b/src/pipecat/services/elevenlabs/stt.py index a1be56a0d..469d2c4fa 100644 --- a/src/pipecat/services/elevenlabs/stt.py +++ b/src/pipecat/services/elevenlabs/stt.py @@ -551,8 +551,6 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): await super().process_frame(frame, direction) if isinstance(frame, VADUserStartedSpeakingFrame): - # Reset finalize state for new utterance - self.set_finalize_pending(False) # Start metrics when user starts speaking await self._start_metrics() elif isinstance(frame, VADUserStoppedSpeakingFrame): @@ -560,8 +558,6 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): if self._params.commit_strategy == CommitStrategy.MANUAL: if self._websocket and self._websocket.state is State.OPEN: try: - # Mark that the next committed transcript should be finalized - self.set_finalize_pending(True) commit_message = { "message_type": "input_audio_chunk", "audio_base_64": "", diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 128bcc93a..1199d8556 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -525,6 +525,14 @@ class OpenAIRealtimeBetaLLMService(LLMService): # note: ttfb is faster by 1/2 RTT than ttfb as measured for other services, since we're getting # this event from the server await self.stop_ttfb_metrics() + + if self._current_audio_response and self._current_audio_response.item_id != evt.item_id: + logger.warning( + f"Received a new audio delta for an already completed audio response before receiving the BotStoppedSpeakingFrame." + ) + logger.debug("Forcing previous audio response to None") + self._current_audio_response = None + if not self._current_audio_response: self._current_audio_response = CurrentAudioResponse( item_id=evt.item_id, diff --git a/src/pipecat/services/sarvam/stt.py b/src/pipecat/services/sarvam/stt.py index 96447dce9..164ad289e 100644 --- a/src/pipecat/services/sarvam/stt.py +++ b/src/pipecat/services/sarvam/stt.py @@ -193,11 +193,9 @@ class SarvamSTTService(STTService): # Only handle VAD frames when not using Sarvam's VAD signals if not self._vad_signals: if isinstance(frame, VADUserStartedSpeakingFrame): - self.set_finalize_pending(False) await self._start_metrics() elif isinstance(frame, VADUserStoppedSpeakingFrame): if self._socket_client: - self.set_finalize_pending(True) await self._socket_client.flush() async def set_language(self, language: Language): diff --git a/src/pipecat/services/speechmatics/stt.py b/src/pipecat/services/speechmatics/stt.py index 457388c31..41434addf 100644 --- a/src/pipecat/services/speechmatics/stt.py +++ b/src/pipecat/services/speechmatics/stt.py @@ -8,7 +8,6 @@ import asyncio import os -import time from enum import Enum from typing import Any, AsyncGenerator @@ -67,7 +66,7 @@ class TurnDetectionMode(str, Enum): """Endpoint and turn detection handling mode. How the STT engine handles the endpointing of speech. If using Pipecat's built-in endpointing, - then use `TurnDetectionMode.FIXED` (default). + then use `TurnDetectionMode.EXTERNAL` (default). To use the STT engine's built-in endpointing, then use `TurnDetectionMode.ADAPTIVE` for simple voice activity detection or `TurnDetectionMode.SMART_TURN` for more advanced ML-based @@ -107,7 +106,7 @@ class SpeechmaticsSTTService(STTService): turn_detection_mode: Endpoint handling, one of `TurnDetectionMode.FIXED`, `TurnDetectionMode.EXTERNAL`, `TurnDetectionMode.ADAPTIVE` and - `TurnDetectionMode.SMART_TURN`. Defaults to `TurnDetectionMode.FIXED`. + `TurnDetectionMode.SMART_TURN`. Defaults to `TurnDetectionMode.EXTERNAL`. speaker_active_format: Formatter for active speaker ID. This formatter is used to format the text output for individual speakers and ensures that the context is clear for @@ -201,6 +200,7 @@ class SpeechmaticsSTTService(STTService): extra_params: Extra parameters to pass to the STT engine. This is a dictionary of additional parameters that can be used to configure the STT engine. Default to None. + """ # Service configuration @@ -208,7 +208,7 @@ class SpeechmaticsSTTService(STTService): language: Language | str = Language.EN # Endpointing mode - turn_detection_mode: TurnDetectionMode = TurnDetectionMode.FIXED + turn_detection_mode: TurnDetectionMode = TurnDetectionMode.EXTERNAL # Output formatting speaker_active_format: str | None = None @@ -346,7 +346,7 @@ class SpeechmaticsSTTService(STTService): params.speaker_passive_format or params.speaker_active_format ) - # Metrics + # Model + metrics self.set_model_name(self._config.operating_point.value) # Message queue @@ -598,9 +598,6 @@ class SpeechmaticsSTTService(STTService): if segments: await self._send_frames(segments) - # Update metrics - await self._emit_metrics(message.get("metadata", {}).get("processing_time", 0.0)) - async def _handle_segment(self, message: dict[str, Any]) -> None: """Handle AddSegment events. @@ -695,6 +692,7 @@ class SpeechmaticsSTTService(STTService): f"{self} VADUserStoppedSpeakingFrame received but internal VAD is being used" ) elif not self._enable_vad and self._client is not None: + self.request_finalize() self._client.finalize() async def _send_frames(self, segments: list[dict[str, Any]], finalized: bool = False) -> None: @@ -738,16 +736,33 @@ class SpeechmaticsSTTService(STTService): # If final, then re-parse into TranscriptionFrame if finalized: + # Do any segments have `is_eou` set to True? + if ( + any(segment.get("is_eou", False) for segment in segments) + and self._finalize_requested + ): + self.confirm_finalize() + + # Add the finalized frames frames += [TranscriptionFrame(**attr_from_segment(segment)) for segment in segments] + + # Handle the text (for metrics reporting) finalized_text = "|".join([s["text"] for s in segments]) - await self._handle_transcription(finalized_text, True, segments[0]["language"]) + await self._handle_transcription( + finalized_text, is_final=True, language=segments[0]["language"] + ) + + # Log the frames logger.debug(f"{self} finalized transcript: {[f.text for f in frames]}") # Return as interim results (unformatted) else: + # Add the interim frames frames += [ InterimTranscriptionFrame(**attr_from_segment(segment)) for segment in segments ] + + # Log the frames logger.debug(f"{self} interim transcript: {[f.text for f in frames]}") # Send the frames @@ -804,28 +819,6 @@ class SpeechmaticsSTTService(STTService): yield ErrorFrame(f"Speechmatics error: {e}") await self._disconnect() - async def _emit_metrics(self, processing_time: float) -> None: - """Create TTFB metrics. - - The TTFB is the seconds between the person speaking and the STT - engine emitting the first partial. This is only calculated at the - start of an utterance. - """ - # Skip if metrics not available - if not self._metrics or processing_time == 0.0: - return - - # Calculate time as time.time() - ttfb (which is seconds) - start_time = time.time() - processing_time - - # Update internal metrics - self._metrics._start_ttfb_time = start_time - self._metrics._start_processing_time = start_time - - # Stop TTFB metrics - await self.stop_ttfb_metrics() - await self.stop_processing_metrics() - # ============================================================================ # HELPERS # ============================================================================ diff --git a/src/pipecat/services/stt_service.py b/src/pipecat/services/stt_service.py index 24b1aafcf..44c7af3b4 100644 --- a/src/pipecat/services/stt_service.py +++ b/src/pipecat/services/stt_service.py @@ -119,28 +119,15 @@ class STTService(AIService): """ return self._muted - def set_finalize_pending(self, value: bool): - """Set whether the next TranscriptionFrame should be marked as finalized. - - When True, the next TranscriptionFrame pushed will have its `finalized` - field set to True, and this flag will automatically reset to False. - This is used to signal that a transcript is the final result for an - utterance, enabling immediate TTFB reporting. - - Args: - value: True to mark the next transcription as finalized. - """ - self._finalize_pending = value - def request_finalize(self): """Mark that a finalize request has been sent, awaiting server confirmation. - For providers that require server confirmation before marking transcripts - as finalized (e.g., Deepgram's from_finalize field), call this when sending - the finalize request. Then call confirm_finalize() when the server confirms. + For providers that have explicit server confirmation of finalization + (e.g., Deepgram's from_finalize field), call this when sending the finalize + request. Then call confirm_finalize() when the server confirms. - This is an alternative to set_finalize_pending() for providers that need - two-step finalization. + For providers without server confirmation, don't call this method - just + send the finalize/flush/commit command and rely on the TTFB timeout. """ self._finalize_requested = True @@ -298,7 +285,7 @@ class STTService(AIService): """Push a frame downstream, tracking TranscriptionFrame timestamps for TTFB. Stores the timestamp of each TranscriptionFrame for TTFB calculation. - If the frame is marked as finalized (either directly or via set_finalize_pending), + If the frame is marked as finalized (via request_finalize/confirm_finalize), reports TTFB immediately and cancels any pending timeout. Otherwise, TTFB is reported after a timeout. @@ -361,6 +348,7 @@ class STTService(AIService): """Handle VAD user started speaking frame to start tracking transcriptions. Cancels any pending TTFB timeout, resets TTFB tracking state, and marks user as speaking. + Also resets finalization state to prevent stale finalization from a previous utterance. Args: frame: The VAD user started speaking frame. @@ -368,6 +356,7 @@ class STTService(AIService): await self._reset_stt_ttfb_state() self._user_speaking = True self._finalize_requested = False + self._finalize_pending = False async def _handle_vad_user_stopped_speaking(self, frame: VADUserStoppedSpeakingFrame): """Handle VAD user stopped speaking frame. diff --git a/src/pipecat/services/websocket_service.py b/src/pipecat/services/websocket_service.py index e86dee73f..85e5b2db7 100644 --- a/src/pipecat/services/websocket_service.py +++ b/src/pipecat/services/websocket_service.py @@ -123,26 +123,29 @@ class WebsocketService(ABC): async def _maybe_try_reconnect( self, - error: Exception, error_message: str, report_error: Callable[[ErrorFrame], Awaitable[None]], + error: Optional[Exception] = None, ) -> bool: """Check if reconnection should be attempted and try if appropriate. Args: - error: The exception that occurred. error_message: Human-readable error message for logging. report_error: Callback function to report connection errors. + error: The exception that occurred (optional, may be None for graceful closes). Returns: True if should continue the receive loop, False if should break. """ # Don't reconnect if we're intentionally disconnecting if self._disconnecting: - logger.warning(f"{self} error during disconnect: {error}") + if error: + logger.warning(f"{self} error during disconnect: {error}") + else: + logger.debug(f"{self} receive loop ended during disconnect") return False - # Log the error + # Log the message logger.warning(error_message) # Try to reconnect if enabled @@ -167,6 +170,14 @@ class WebsocketService(ABC): while True: try: await self._receive_messages() + # _receive_messages() returned normally. This happens when the websocket + # closes gracefully (server sent close frame). The async for loop over + # the websocket exits without raising an exception in this case. + # We must handle this to avoid an infinite loop. + message = f"{self} connection closed by server" + should_continue = await self._maybe_try_reconnect(message, report_error) + if not should_continue: + break except ConnectionClosedOK as e: # Normal closure, don't retry logger.debug(f"{self} connection closed normally: {e}") @@ -175,13 +186,13 @@ class WebsocketService(ABC): # Connection closed with error (e.g., no close frame received/sent) # This often indicates network issues, server problems, or abrupt disconnection message = f"{self} connection closed, but with an error: {e}" - should_continue = await self._maybe_try_reconnect(e, message, report_error) + should_continue = await self._maybe_try_reconnect(message, report_error, e) if not should_continue: break except Exception as e: # General error during message receiving message = f"{self} error receiving messages: {e}" - should_continue = await self._maybe_try_reconnect(e, message, report_error) + should_continue = await self._maybe_try_reconnect(message, report_error, e) if not should_continue: break diff --git a/src/pipecat/transports/daily/transport.py b/src/pipecat/transports/daily/transport.py index fe106ebeb..20a0be29f 100644 --- a/src/pipecat/transports/daily/transport.py +++ b/src/pipecat/transports/daily/transport.py @@ -1733,7 +1733,7 @@ class DailyInputTransport(BaseInputTransport): message: The message data to send. sender: ID of the message sender. """ - await self.broadcast_frame_class( + await self.broadcast_frame( DailyInputTransportMessageFrame, message=message, participant_id=sender ) diff --git a/src/pipecat/transports/smallwebrtc/transport.py b/src/pipecat/transports/smallwebrtc/transport.py index bcc3c8b79..7e76771c4 100644 --- a/src/pipecat/transports/smallwebrtc/transport.py +++ b/src/pipecat/transports/smallwebrtc/transport.py @@ -698,7 +698,7 @@ class SmallWebRTCInputTransport(BaseInputTransport): message: The application message to process. """ logger.debug(f"Received app message inside SmallWebRTCInputTransport {message}") - await self.broadcast_frame_class(InputTransportMessageFrame, message=message) + await self.broadcast_frame(InputTransportMessageFrame, message=message) # Add this method similar to DailyInputTransport.request_participant_image async def request_participant_image(self, frame: UserImageRequestFrame): diff --git a/tests/test_aic_filter.py b/tests/test_aic_filter.py new file mode 100644 index 000000000..6499084af --- /dev/null +++ b/tests/test_aic_filter.py @@ -0,0 +1,471 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import unittest +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import numpy as np + +# Check if aic_sdk is available +try: + import aic_sdk + + HAS_AIC_SDK = True +except ImportError: + HAS_AIC_SDK = False + +# Module path for patching +AIC_FILTER_MODULE = "pipecat.audio.filters.aic_filter" + + +class MockProcessor: + """A lightweight mock for AIC ProcessorAsync that mimics real behavior.""" + + def __init__(self): + self.processor_ctx = MockProcessorContext() + self.vad_ctx = MockVadContext() + + def get_processor_context(self): + return self.processor_ctx + + def get_vad_context(self): + return self.vad_ctx + + async def process_async(self, audio_array): + # Return a copy of the input (simulating passthrough) + return audio_array.copy() + + +class MockProcessorContext: + """A lightweight mock for AIC ProcessorContext.""" + + def __init__(self): + self.parameters_set: list[tuple] = [] + self.reset_called = False + self._output_delay = 0 + + def get_output_delay(self): + return self._output_delay + + def set_parameter(self, param, value): + self.parameters_set.append((param, value)) + + def reset(self): + self.reset_called = True + + +class MockVadContext: + """A lightweight mock for AIC VadContext.""" + + def __init__(self, speech_detected: bool = False): + self.speech_detected = speech_detected + self.parameters_set: list[tuple] = [] + + def is_speech_detected(self) -> bool: + return self.speech_detected + + def set_parameter(self, param, value): + self.parameters_set.append((param, value)) + + +class MockModel: + """A lightweight mock for AIC Model.""" + + def __init__(self, model_id: str = "test-model"): + self._model_id = model_id + self._optimal_num_frames = 160 + self._optimal_sample_rate = 16000 + + def get_optimal_num_frames(self, sample_rate: int): + """Return optimal number of frames for the given sample rate.""" + return self._optimal_num_frames + + def get_id(self): + return self._model_id + + def get_optimal_sample_rate(self): + return self._optimal_sample_rate + + +@unittest.skipUnless(HAS_AIC_SDK, "aic-sdk not installed") +class TestAICFilter(unittest.IsolatedAsyncioTestCase): + """Test suite for AICFilter audio filter using real aic_sdk types.""" + + @classmethod + def setUpClass(cls): + """Import AICFilter after confirming aic_sdk is available.""" + from pipecat.audio.filters.aic_filter import AICFilter + from pipecat.frames.frames import FilterEnableFrame + + cls.AICFilter = AICFilter + cls.FilterEnableFrame = FilterEnableFrame + + def setUp(self): + """Set up test fixtures before each test method.""" + self.mock_model = MockModel() + self.mock_processor = MockProcessor() + + def _create_filter_with_mocks(self, **kwargs): + """Create an AICFilter with mocked SDK components.""" + filter_kwargs = { + "license_key": "test-key", + "model_id": "test-model", + } + filter_kwargs.update(kwargs) + with patch(f"{AIC_FILTER_MODULE}.set_sdk_id"): + return self.AICFilter(**filter_kwargs) + + async def _start_filter_with_mocks(self, filter_instance, sample_rate=16000): + """Start a filter with mocked SDK components.""" + with ( + patch(f"{AIC_FILTER_MODULE}.Model") as mock_model_cls, + patch(f"{AIC_FILTER_MODULE}.ProcessorConfig") as mock_config_cls, + patch(f"{AIC_FILTER_MODULE}.ProcessorAsync", return_value=self.mock_processor), + ): + mock_model_cls.from_file.return_value = self.mock_model + mock_model_cls.download_async = AsyncMock(return_value="/tmp/model") + mock_config_cls.optimal.return_value = MagicMock() + await filter_instance.start(sample_rate) + + async def test_initialization_requires_model_id_or_path(self): + """Test filter initialization fails without model_id or model_path.""" + with patch(f"{AIC_FILTER_MODULE}.set_sdk_id"): + with self.assertRaises(ValueError) as context: + self.AICFilter(license_key="test-key") + + self.assertIn("model_id", str(context.exception)) + self.assertIn("model_path", str(context.exception)) + + async def test_initialization_with_model_id(self): + """Test filter initialization with model_id.""" + filter_instance = self._create_filter_with_mocks() + + self.assertEqual(filter_instance._license_key, "test-key") + self.assertEqual(filter_instance._model_id, "test-model") + self.assertIsNone(filter_instance._model_path) + self.assertFalse(filter_instance._bypass) + + async def test_initialization_with_model_path(self): + """Test filter initialization with model_path.""" + model_path = Path("/tmp/test.aicmodel") + filter_instance = self._create_filter_with_mocks(model_id=None, model_path=model_path) + + self.assertEqual(filter_instance._model_path, model_path) + self.assertIsNone(filter_instance._model_id) + + async def test_initialization_with_custom_download_dir(self): + """Test filter initialization with custom model_download_dir.""" + download_dir = Path("/custom/cache") + filter_instance = self._create_filter_with_mocks(model_download_dir=download_dir) + + self.assertEqual(filter_instance._model_download_dir, download_dir) + + async def test_start_with_model_path(self): + """Test starting filter with a local model path.""" + model_path = Path("/tmp/test.aicmodel") + filter_instance = self._create_filter_with_mocks(model_id=None, model_path=model_path) + + with ( + patch(f"{AIC_FILTER_MODULE}.Model") as mock_model_cls, + patch(f"{AIC_FILTER_MODULE}.ProcessorConfig") as mock_config_cls, + patch(f"{AIC_FILTER_MODULE}.ProcessorAsync", return_value=self.mock_processor), + ): + mock_model_cls.from_file.return_value = self.mock_model + mock_config_cls.optimal.return_value = MagicMock() + + await filter_instance.start(16000) + + mock_model_cls.from_file.assert_called_once_with(str(model_path)) + self.assertTrue(filter_instance._aic_ready) + self.assertEqual(filter_instance._sample_rate, 16000) + self.assertEqual(filter_instance._frames_per_block, 160) + + async def test_start_with_model_id_downloads(self): + """Test starting filter with model_id triggers download.""" + filter_instance = self._create_filter_with_mocks() + + with ( + patch(f"{AIC_FILTER_MODULE}.Model") as mock_model_cls, + patch(f"{AIC_FILTER_MODULE}.ProcessorConfig") as mock_config_cls, + patch(f"{AIC_FILTER_MODULE}.ProcessorAsync", return_value=self.mock_processor), + ): + mock_model_cls.from_file.return_value = self.mock_model + mock_model_cls.download_async = AsyncMock(return_value="/tmp/model") + mock_config_cls.optimal.return_value = MagicMock() + + await filter_instance.start(16000) + + mock_model_cls.download_async.assert_called_once() + mock_model_cls.from_file.assert_called_once() + self.assertTrue(filter_instance._aic_ready) + + async def test_start_creates_processor(self): + """Test that start creates processor with correct config.""" + filter_instance = self._create_filter_with_mocks() + + with ( + patch(f"{AIC_FILTER_MODULE}.Model") as mock_model_cls, + patch(f"{AIC_FILTER_MODULE}.ProcessorConfig") as mock_config_cls, + patch( + f"{AIC_FILTER_MODULE}.ProcessorAsync", return_value=self.mock_processor + ) as mock_processor_cls, + ): + mock_model_cls.from_file.return_value = self.mock_model + mock_model_cls.download_async = AsyncMock(return_value="/tmp/model") + mock_config_cls.optimal.return_value = MagicMock() + + await filter_instance.start(16000) + + mock_config_cls.optimal.assert_called_once() + mock_processor_cls.assert_called_once() + self.assertIsNotNone(filter_instance._processor_ctx) + self.assertIsNotNone(filter_instance._vad_ctx) + + async def test_start_applies_initial_bypass_parameter(self): + """Test that start applies bypass parameter.""" + filter_instance = self._create_filter_with_mocks() + await self._start_filter_with_mocks(filter_instance) + + # Check that bypass was set to 0.0 (enabled) + bypass_params = [ + (p, v) + for p, v in self.mock_processor.processor_ctx.parameters_set + if p == aic_sdk.ProcessorParameter.Bypass + ] + self.assertTrue(len(bypass_params) > 0) + self.assertEqual(bypass_params[-1][1], 0.0) + + async def test_stop_cleans_up_resources(self): + """Test that stop properly cleans up resources.""" + filter_instance = self._create_filter_with_mocks() + await self._start_filter_with_mocks(filter_instance) + + await filter_instance.stop() + + self.assertTrue(self.mock_processor.processor_ctx.reset_called) + self.assertIsNone(filter_instance._processor) + self.assertIsNone(filter_instance._processor_ctx) + self.assertIsNone(filter_instance._vad_ctx) + self.assertIsNone(filter_instance._model) + self.assertFalse(filter_instance._aic_ready) + + async def test_stop_without_start(self): + """Test that stop can be called safely without start.""" + filter_instance = self._create_filter_with_mocks() + + # Should not raise + await filter_instance.stop() + + async def test_process_frame_enable(self): + """Test processing FilterEnableFrame to enable filtering.""" + filter_instance = self._create_filter_with_mocks() + await self._start_filter_with_mocks(filter_instance) + filter_instance._bypass = True + + enable_frame = self.FilterEnableFrame(enable=True) + await filter_instance.process_frame(enable_frame) + + self.assertFalse(filter_instance._bypass) + + async def test_process_frame_disable(self): + """Test processing FilterEnableFrame to disable filtering.""" + filter_instance = self._create_filter_with_mocks() + await self._start_filter_with_mocks(filter_instance) + + disable_frame = self.FilterEnableFrame(enable=False) + await filter_instance.process_frame(disable_frame) + + self.assertTrue(filter_instance._bypass) + + async def test_filter_when_not_ready(self): + """Test that filter returns audio unchanged when not ready.""" + filter_instance = self._create_filter_with_mocks() + # Don't call start() + + input_audio = b"\x00\x01\x02\x03" + output_audio = await filter_instance.filter(input_audio) + + self.assertEqual(output_audio, input_audio) + + async def test_filter_with_incomplete_frame(self): + """Test filtering audio with incomplete frame data.""" + filter_instance = self._create_filter_with_mocks() + await self._start_filter_with_mocks(filter_instance) + + # Create audio data for less than one frame (100 samples = 200 bytes) + samples = np.random.randint(-32768, 32767, size=100, dtype=np.int16) + input_audio = samples.tobytes() + + output_audio = await filter_instance.filter(input_audio) + + # Should return empty bytes since no complete frame + self.assertEqual(output_audio, b"") + + async def test_filter_with_complete_frame(self): + """Test filtering audio with exactly one complete frame.""" + filter_instance = self._create_filter_with_mocks() + await self._start_filter_with_mocks(filter_instance) + + # Create audio data for exactly one frame (160 samples = 320 bytes) + samples = np.random.randint(-32768, 32767, size=160, dtype=np.int16) + input_audio = samples.tobytes() + + output_audio = await filter_instance.filter(input_audio) + + self.assertIsInstance(output_audio, bytes) + self.assertEqual(len(output_audio), len(input_audio)) + + async def test_filter_with_multiple_frames(self): + """Test filtering audio with multiple complete frames.""" + filter_instance = self._create_filter_with_mocks() + await self._start_filter_with_mocks(filter_instance) + + # Create audio data for 3 complete frames (480 samples = 960 bytes) + samples = np.random.randint(-32768, 32767, size=480, dtype=np.int16) + input_audio = samples.tobytes() + + output_audio = await filter_instance.filter(input_audio) + + self.assertEqual(len(output_audio), len(input_audio)) + + async def test_filter_with_buffering(self): + """Test that filter properly buffers incomplete frames.""" + filter_instance = self._create_filter_with_mocks() + await self._start_filter_with_mocks(filter_instance) + + # First call: Send 100 samples (incomplete frame) + samples1 = np.random.randint(-32768, 32767, size=100, dtype=np.int16) + input_audio1 = samples1.tobytes() + output_audio1 = await filter_instance.filter(input_audio1) + + self.assertEqual(output_audio1, b"") + self.assertEqual(len(filter_instance._audio_buffer), 200) + + # Second call: Send 60 more samples (now we have 160 total = 1 complete frame) + samples2 = np.random.randint(-32768, 32767, size=60, dtype=np.int16) + input_audio2 = samples2.tobytes() + output_audio2 = await filter_instance.filter(input_audio2) + + self.assertEqual(len(output_audio2), 320) + self.assertEqual(len(filter_instance._audio_buffer), 0) + + async def test_filter_with_partial_buffering(self): + """Test that filter keeps remainder in buffer after processing.""" + filter_instance = self._create_filter_with_mocks() + await self._start_filter_with_mocks(filter_instance) + + # Send 250 samples (1 complete frame + 90 samples remainder) + samples = np.random.randint(-32768, 32767, size=250, dtype=np.int16) + input_audio = samples.tobytes() + + output_audio = await filter_instance.filter(input_audio) + + self.assertEqual(len(output_audio), 320) # 1 frame + self.assertEqual(len(filter_instance._audio_buffer), 180) # 90 samples * 2 bytes + + async def test_get_vad_context_before_start(self): + """Test that get_vad_context raises before start.""" + filter_instance = self._create_filter_with_mocks() + + with self.assertRaises(RuntimeError) as context: + filter_instance.get_vad_context() + + self.assertIn("not initialized", str(context.exception)) + + async def test_get_vad_context_after_start(self): + """Test that get_vad_context returns context after start.""" + filter_instance = self._create_filter_with_mocks() + await self._start_filter_with_mocks(filter_instance) + + vad_ctx = filter_instance.get_vad_context() + + self.assertEqual(vad_ctx, self.mock_processor.vad_ctx) + + async def test_create_vad_analyzer(self): + """Test create_vad_analyzer returns analyzer with factory.""" + filter_instance = self._create_filter_with_mocks() + + analyzer = filter_instance.create_vad_analyzer() + + self.assertIsNotNone(analyzer) + # Factory should be set + self.assertIsNotNone(analyzer._vad_context_factory) + + async def test_create_vad_analyzer_with_params(self): + """Test create_vad_analyzer with custom parameters.""" + filter_instance = self._create_filter_with_mocks() + + analyzer = filter_instance.create_vad_analyzer( + speech_hold_duration=0.1, + minimum_speech_duration=0.05, + sensitivity=8.0, + ) + + self.assertEqual(analyzer._pending_speech_hold_duration, 0.1) + self.assertEqual(analyzer._pending_minimum_speech_duration, 0.05) + self.assertEqual(analyzer._pending_sensitivity, 8.0) + + async def test_multiple_start_stop_cycles(self): + """Test multiple start/stop cycles.""" + filter_instance = self._create_filter_with_mocks() + + for sample_rate in [16000, 24000, 48000]: + # Create fresh mock processor for each cycle + self.mock_processor = MockProcessor() + await self._start_filter_with_mocks(filter_instance, sample_rate) + self.assertTrue(filter_instance._aic_ready) + self.assertEqual(filter_instance._sample_rate, sample_rate) + + await filter_instance.stop() + self.assertFalse(filter_instance._aic_ready) + + async def test_concurrent_filter_calls(self): + """Test that concurrent filter calls are handled safely.""" + filter_instance = self._create_filter_with_mocks() + await self._start_filter_with_mocks(filter_instance) + + samples = np.random.randint(-32768, 32767, size=160, dtype=np.int16) + input_audio = samples.tobytes() + + async def filter_audio(): + return await filter_instance.filter(input_audio) + + tasks = [filter_audio() for _ in range(10)] + results = await asyncio.gather(*tasks) + + self.assertEqual(len(results), 10) + for result in results: + self.assertIsInstance(result, bytes) + + async def test_buffer_cleared_on_stop(self): + """Test that audio buffer is cleared when stopping.""" + filter_instance = self._create_filter_with_mocks() + await self._start_filter_with_mocks(filter_instance) + + # Add incomplete frame to buffer + samples = np.random.randint(-32768, 32767, size=100, dtype=np.int16) + input_audio = samples.tobytes() + await filter_instance.filter(input_audio) + + # Verify buffer has data + self.assertGreater(len(filter_instance._audio_buffer), 0) + + # Stop should clear buffer + await filter_instance.stop() + self.assertEqual(len(filter_instance._audio_buffer), 0) + + async def test_set_sdk_id_called_on_init(self): + """Test that set_sdk_id is called during initialization.""" + with patch(f"{AIC_FILTER_MODULE}.set_sdk_id") as mock_set_sdk_id: + self.AICFilter(license_key="test-key", model_id="test-model") + + mock_set_sdk_id.assert_called_once_with(6) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_aic_vad.py b/tests/test_aic_vad.py new file mode 100644 index 000000000..5028da46b --- /dev/null +++ b/tests/test_aic_vad.py @@ -0,0 +1,322 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import unittest + +# Check if aic_sdk is available +try: + import aic_sdk + + HAS_AIC_SDK = True +except ImportError: + HAS_AIC_SDK = False + + +@unittest.skipUnless(HAS_AIC_SDK, "aic-sdk not installed") +class TestAICVADAnalyzer(unittest.IsolatedAsyncioTestCase): + """Test suite for AICVADAnalyzer using real aic_sdk.""" + + @classmethod + def setUpClass(cls): + """Import AICVADAnalyzer after confirming aic_sdk is available.""" + from pipecat.audio.vad.aic_vad import AICVADAnalyzer + + cls.AICVADAnalyzer = AICVADAnalyzer + + def test_initialization_without_factory(self): + """Test analyzer initialization without a factory.""" + analyzer = self.AICVADAnalyzer() + + self.assertIsNone(analyzer._vad_context_factory) + self.assertIsNone(analyzer._vad_ctx) + # Fixed params should be set + self.assertEqual(analyzer._params.confidence, 0.5) + self.assertEqual(analyzer._params.start_secs, 0.0) + self.assertEqual(analyzer._params.stop_secs, 0.0) + self.assertEqual(analyzer._params.min_volume, 0.0) + + def test_initialization_with_factory(self): + """Test analyzer initialization with a factory.""" + # Create a mock VAD context for testing + mock_vad_ctx = MockVadContext() + factory = lambda: mock_vad_ctx + analyzer = self.AICVADAnalyzer(vad_context_factory=factory) + + self.assertIsNotNone(analyzer._vad_context_factory) + + def test_initialization_with_vad_params(self): + """Test analyzer initialization with VAD parameters.""" + analyzer = self.AICVADAnalyzer( + speech_hold_duration=0.1, + minimum_speech_duration=0.05, + sensitivity=8.0, + ) + + self.assertEqual(analyzer._pending_speech_hold_duration, 0.1) + self.assertEqual(analyzer._pending_minimum_speech_duration, 0.05) + self.assertEqual(analyzer._pending_sensitivity, 8.0) + + def test_bind_vad_context_factory(self): + """Test binding a factory post-construction.""" + mock_vad_ctx = MockVadContext() + analyzer = self.AICVADAnalyzer() + factory = lambda: mock_vad_ctx + + analyzer.bind_vad_context_factory(factory) + + self.assertEqual(analyzer._vad_context_factory, factory) + # Should have attempted to initialize + self.assertEqual(analyzer._vad_ctx, mock_vad_ctx) + + def test_bind_vad_context_factory_applies_params(self): + """Test that binding factory applies pending VAD params.""" + mock_vad_ctx = MockVadContext() + analyzer = self.AICVADAnalyzer( + speech_hold_duration=0.1, + minimum_speech_duration=0.05, + sensitivity=8.0, + ) + factory = lambda: mock_vad_ctx + + analyzer.bind_vad_context_factory(factory) + + # Verify parameters were applied + self.assertIn( + (aic_sdk.VadParameter.SpeechHoldDuration, 0.1), + mock_vad_ctx.parameters_set, + ) + self.assertIn( + (aic_sdk.VadParameter.MinimumSpeechDuration, 0.05), + mock_vad_ctx.parameters_set, + ) + self.assertIn( + (aic_sdk.VadParameter.Sensitivity, 8.0), + mock_vad_ctx.parameters_set, + ) + + def test_set_sample_rate(self): + """Test setting sample rate.""" + analyzer = self.AICVADAnalyzer() + + analyzer.set_sample_rate(16000) + + self.assertEqual(analyzer._sample_rate, 16000) + + def test_set_sample_rate_with_init_sample_rate(self): + """Test that init_sample_rate takes precedence.""" + # Create analyzer and manually set _init_sample_rate + analyzer = self.AICVADAnalyzer() + analyzer._init_sample_rate = 48000 + + analyzer.set_sample_rate(16000) + + # init_sample_rate should take precedence + self.assertEqual(analyzer._sample_rate, 48000) + + def test_set_sample_rate_triggers_context_init(self): + """Test that set_sample_rate attempts context initialization.""" + mock_vad_ctx = MockVadContext() + factory = lambda: mock_vad_ctx + analyzer = self.AICVADAnalyzer(vad_context_factory=factory) + + analyzer.set_sample_rate(16000) + + self.assertEqual(analyzer._vad_ctx, mock_vad_ctx) + + def test_num_frames_required_with_sample_rate(self): + """Test num_frames_required returns correct value.""" + analyzer = self.AICVADAnalyzer() + analyzer.set_sample_rate(16000) + + frames = analyzer.num_frames_required() + + # 10ms at 16kHz = 160 frames + self.assertEqual(frames, 160) + + def test_num_frames_required_different_sample_rates(self): + """Test num_frames_required for different sample rates.""" + analyzer = self.AICVADAnalyzer() + + test_cases = [ + (8000, 80), # 10ms at 8kHz + (16000, 160), # 10ms at 16kHz + (24000, 240), # 10ms at 24kHz + (48000, 480), # 10ms at 48kHz + ] + + for sample_rate, expected_frames in test_cases: + analyzer.set_sample_rate(sample_rate) + frames = analyzer.num_frames_required() + self.assertEqual(frames, expected_frames, f"Failed for {sample_rate}Hz") + + def test_num_frames_required_no_sample_rate(self): + """Test num_frames_required returns default when no sample rate.""" + analyzer = self.AICVADAnalyzer() + + frames = analyzer.num_frames_required() + + # Default is 160 + self.assertEqual(frames, 160) + + def test_voice_confidence_no_context(self): + """Test voice_confidence returns 0.0 when no context.""" + analyzer = self.AICVADAnalyzer() + + confidence = analyzer.voice_confidence(b"\x00" * 320) + + self.assertEqual(confidence, 0.0) + + def test_voice_confidence_speech_detected(self): + """Test voice_confidence returns 1.0 when speech detected.""" + mock_vad_ctx = MockVadContext(speech_detected=True) + factory = lambda: mock_vad_ctx + analyzer = self.AICVADAnalyzer(vad_context_factory=factory) + analyzer.set_sample_rate(16000) + + confidence = analyzer.voice_confidence(b"\x00" * 320) + + self.assertEqual(confidence, 1.0) + + def test_voice_confidence_no_speech(self): + """Test voice_confidence returns 0.0 when no speech.""" + mock_vad_ctx = MockVadContext(speech_detected=False) + factory = lambda: mock_vad_ctx + analyzer = self.AICVADAnalyzer(vad_context_factory=factory) + analyzer.set_sample_rate(16000) + + confidence = analyzer.voice_confidence(b"\x00" * 320) + + self.assertEqual(confidence, 0.0) + + def test_voice_confidence_handles_exception(self): + """Test voice_confidence handles exceptions gracefully.""" + mock_vad_ctx = MockVadContext(raise_on_detect=True) + factory = lambda: mock_vad_ctx + analyzer = self.AICVADAnalyzer(vad_context_factory=factory) + analyzer.set_sample_rate(16000) + + confidence = analyzer.voice_confidence(b"\x00" * 320) + + self.assertEqual(confidence, 0.0) + + def test_lazy_initialization(self): + """Test that VAD context is lazily initialized.""" + call_count = 0 + mock_vad_ctx = MockVadContext() + + def counting_factory(): + nonlocal call_count + call_count += 1 + return mock_vad_ctx + + analyzer = self.AICVADAnalyzer(vad_context_factory=counting_factory) + + # Factory not called yet + self.assertEqual(call_count, 0) + + # First call to voice_confidence triggers initialization + analyzer.voice_confidence(b"\x00" * 320) + self.assertEqual(call_count, 1) + + # Subsequent calls don't re-initialize + analyzer.voice_confidence(b"\x00" * 320) + analyzer.voice_confidence(b"\x00" * 320) + self.assertEqual(call_count, 1) + + def test_deferred_initialization_on_factory_failure(self): + """Test that initialization is deferred when factory fails.""" + call_count = 0 + mock_vad_ctx = MockVadContext(speech_detected=True) + + def failing_then_succeeding_factory(): + nonlocal call_count + call_count += 1 + if call_count < 3: + raise RuntimeError("Not ready yet") + return mock_vad_ctx + + analyzer = self.AICVADAnalyzer(vad_context_factory=failing_then_succeeding_factory) + + # First two calls fail, should return 0.0 + self.assertEqual(analyzer.voice_confidence(b"\x00" * 320), 0.0) + self.assertEqual(analyzer.voice_confidence(b"\x00" * 320), 0.0) + + # Third call succeeds + self.assertEqual(analyzer.voice_confidence(b"\x00" * 320), 1.0) + + def test_apply_vad_params_deferred_on_failure(self): + """Test that VAD param application handles exceptions.""" + mock_vad_ctx = MockVadContext(raise_on_set_param=True) + factory = lambda: mock_vad_ctx + + analyzer = self.AICVADAnalyzer( + vad_context_factory=factory, + speech_hold_duration=0.1, + ) + + # Should not raise, just log debug message + analyzer.bind_vad_context_factory(factory) + + # Context should still be set despite param failure + self.assertEqual(analyzer._vad_ctx, mock_vad_ctx) + + def test_apply_vad_params_only_set_values(self): + """Test that only specified VAD params are applied.""" + mock_vad_ctx = MockVadContext() + factory = lambda: mock_vad_ctx + analyzer = self.AICVADAnalyzer( + vad_context_factory=factory, + speech_hold_duration=0.1, + # minimum_speech_duration and sensitivity not set + ) + + analyzer.bind_vad_context_factory(factory) + + # Only SpeechHoldDuration should be set + self.assertEqual(len(mock_vad_ctx.parameters_set), 1) + self.assertIn( + (aic_sdk.VadParameter.SpeechHoldDuration, 0.1), + mock_vad_ctx.parameters_set, + ) + + def test_fixed_vad_params(self): + """Test that VAD uses fixed parameters.""" + analyzer = self.AICVADAnalyzer() + + # These are the fixed params for AIC VAD + self.assertEqual(analyzer._params.confidence, 0.5) + self.assertEqual(analyzer._params.start_secs, 0.0) + self.assertEqual(analyzer._params.stop_secs, 0.0) + self.assertEqual(analyzer._params.min_volume, 0.0) + + +class MockVadContext: + """A lightweight mock for AIC VadContext that mimics real behavior.""" + + def __init__( + self, + speech_detected: bool = False, + raise_on_detect: bool = False, + raise_on_set_param: bool = False, + ): + self.speech_detected = speech_detected + self.raise_on_detect = raise_on_detect + self.raise_on_set_param = raise_on_set_param + self.parameters_set: list[tuple] = [] + + def is_speech_detected(self) -> bool: + if self.raise_on_detect: + raise RuntimeError("VAD error") + return self.speech_detected + + def set_parameter(self, param, value): + if self.raise_on_set_param: + raise RuntimeError("Param error") + self.parameters_set.append((param, value)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_context_aggregators_universal.py b/tests/test_context_aggregators_universal.py index 3cb1407f1..63f1fef9a 100644 --- a/tests/test_context_aggregators_universal.py +++ b/tests/test_context_aggregators_universal.py @@ -344,6 +344,35 @@ class TestLLMUserAggregator(unittest.IsolatedAsyncioTestCase): # The user mute strategies should have muted the user. self.assertFalse(user_turn) + async def test_pending_transcription_emitted_on_end_frame(self): + """Pending user transcription should be emitted when EndFrame arrives.""" + context = LLMContext() + + user_aggregator = LLMUserAggregator(context) + + stop_messages = [] + + @user_aggregator.event_handler("on_user_turn_stopped") + async def on_user_turn_stopped(aggregator, strategy, message): + stop_messages.append((strategy, message)) + + pipeline = Pipeline([user_aggregator]) + + # Start turn and send transcription, but don't trigger normal turn stop + frames_to_send = [ + VADUserStartedSpeakingFrame(), + TranscriptionFrame(text="Hello!", user_id="", timestamp="now"), + # No VADUserStoppedSpeakingFrame - turn doesn't stop normally + # EndFrame will be sent by run_test, triggering emission + ] + await run_test(pipeline, frames_to_send=frames_to_send) + + # The pending transcription should be emitted on EndFrame + self.assertEqual(len(stop_messages), 1) + strategy, message = stop_messages[0] + self.assertIsNone(strategy) # strategy is None for end/cancel + self.assertEqual(message.content, "Hello!") + class TestLLMAssistantAggregator(unittest.IsolatedAsyncioTestCase): async def test_empty(self): @@ -512,3 +541,28 @@ class TestLLMAssistantAggregator(unittest.IsolatedAsyncioTestCase): ] await run_test(aggregator, frames_to_send=frames_to_send) self.assertEqual(thought_message.content, "I'm thinking!") + + async def test_pending_text_emitted_on_end_frame(self): + """Pending assistant text should be emitted when EndFrame arrives.""" + context = LLMContext() + + aggregator = LLMAssistantAggregator(context) + + stop_messages = [] + + @aggregator.event_handler("on_assistant_turn_stopped") + async def on_assistant_turn_stopped(aggregator, message: AssistantTurnStoppedMessage): + stop_messages.append(message) + + # Start response and send text, but don't send LLMFullResponseEndFrame + frames_to_send = [ + LLMFullResponseStartFrame(), + LLMTextFrame("Hello from Pipecat!"), + # No LLMFullResponseEndFrame - response doesn't end normally + # EndFrame will be sent by run_test, triggering emission + ] + await run_test(aggregator, frames_to_send=frames_to_send) + + # The pending text should be emitted on EndFrame + self.assertEqual(len(stop_messages), 1) + self.assertEqual(stop_messages[0].content, "Hello from Pipecat!") diff --git a/uv.lock b/uv.lock index 45c1f892f..b35c9f25d 100644 --- a/uv.lock +++ b/uv.lock @@ -38,12 +38,44 @@ wheels = [ [[package]] name = "aic-sdk" -version = "1.2.0" +version = "2.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f9/ba/3ebe31b91e03d42437ec864e9d2af3a52b7ccc73a1a0c1026275956270b0/aic_sdk-1.2.0.tar.gz", hash = "sha256:eeda9a181c679f175dbe6f0efc0c67ec98ff3d84cfe01541fef7fa12ecd505ca", size = 35606, upload-time = "2025-11-20T14:42:14.333Z" } +sdist = { url = "https://files.pythonhosted.org/packages/68/c6/1f0b3d3d226c6d19ec654fdaea7859ee9931e0286735385b1f9ea4bcfba1/aic_sdk-2.0.1.tar.gz", hash = "sha256:2480d8398a26639ed7fb5175c37da82cf5e6b1138a1a301938cd8491fe461c20", size = 73091, upload-time = "2026-01-23T23:38:15.77Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/cf/b2f56f3129b8e393362487b6828a6811cc2f252d438bbf53dc917fd53f23/aic_sdk-2.0.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:583e0b51d236d02396b9d13fce112bb63aa2b6953e42c925af093beea2b82edb", size = 4892239, upload-time = "2026-01-23T23:36:15.832Z" }, + { url = "https://files.pythonhosted.org/packages/92/bc/300366b9a64c97ca40db4d54a0ab8390f4c6860bf6cb5e1e0c55988aca1f/aic_sdk-2.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ef80b2ef5d1f43ef28e117c7db3503e4877d532e12ebac79dd0c0a1944bc6a0a", size = 4449896, upload-time = "2026-01-23T23:36:20.784Z" }, + { url = "https://files.pythonhosted.org/packages/52/76/57e365ede8d4f88dbdce119ec6d8910d76c5e85e506ee3062a4a1222ea97/aic_sdk-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ee7b00bcf7eb870ef05bdefcb65eaf4894285155d454e85187c49f313978152", size = 3595181, upload-time = "2026-01-23T23:36:25.641Z" }, + { url = "https://files.pythonhosted.org/packages/c4/59/f6d92c34469ab54c74cbd59590d2f0f8247d2e576f0f97723e11004708ff/aic_sdk-2.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb0b74a457f3749a90414304e1291fcb6ffb8019f3c59f39c2f395eabf902b", size = 4111674, upload-time = "2026-01-23T23:36:31.985Z" }, + { url = "https://files.pythonhosted.org/packages/ad/78/2fe743d9194f4a187ca72dd9e24d96c9f3687e11990f2ebb2f900719303e/aic_sdk-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:559307bded02c0b64a00595ec8e5383bb7fe5e9b0865cd9b49e2b15411057f1a", size = 3663836, upload-time = "2026-01-23T23:36:36.322Z" }, + { url = "https://files.pythonhosted.org/packages/8b/e4/2f6bdd665b4d4da43e890f8849daf9661ef36c7304a4c675f3cbf617cb14/aic_sdk-2.0.1-cp310-cp310-win_arm64.whl", hash = "sha256:e4b64f289416779711cd083905abdd80fdb4f8a6802480b958951ded1517c6a5", size = 3275160, upload-time = "2026-01-23T23:36:39.014Z" }, + { url = "https://files.pythonhosted.org/packages/57/6f/2a065d61ed333e46a704f6592b33a88ffd0848b2efa99b039c8e427b21a3/aic_sdk-2.0.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:06aa50a7f014c8b06387cdea6fb37c53c9697490eab98959039aeccc8d51e360", size = 4892089, upload-time = "2026-01-23T23:36:41.249Z" }, + { url = "https://files.pythonhosted.org/packages/de/f7/cd0c82cec01a94d7e121d411780f43cb8e6611bd797a10c02fd02c858f49/aic_sdk-2.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1f1ade29783354f09f270ce38649dd6aed57c237c1b090b2ddc0fb61bc651d47", size = 4449813, upload-time = "2026-01-23T23:36:43.845Z" }, + { url = "https://files.pythonhosted.org/packages/44/16/d90d39716cf487f0a41fd5bd01670884f9d0901902d6616595ad3ea17464/aic_sdk-2.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17040ea4d6a686429a214a5673c362890ad10cefb265b6f878a240763e6f39ef", size = 3594996, upload-time = "2026-01-23T23:36:46.334Z" }, + { url = "https://files.pythonhosted.org/packages/21/5d/8852484f85fa60a8ec2e696f6de8363301cd6100b2e5a68289ccc36d02ca/aic_sdk-2.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f13f9b2211136dd6f46fa2f148a55aabf5b9c3cb40fc0beed9e435b0df60d34c", size = 4111589, upload-time = "2026-01-23T23:36:50.448Z" }, + { url = "https://files.pythonhosted.org/packages/71/ca/22c99be2aca92f77d4f0fe742827cd2db5f0c761797ebe0e5bd43872259a/aic_sdk-2.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:f4c3556bad0b74f2c0a5c2a253f14ca58b7129ce5b17848b8b0948f68286639d", size = 3663706, upload-time = "2026-01-23T23:36:54.581Z" }, + { url = "https://files.pythonhosted.org/packages/19/fc/fbd7ee793cf15ef3319d12399ee9300c21c09acf654e1d8d1f64f682d750/aic_sdk-2.0.1-cp311-cp311-win_arm64.whl", hash = "sha256:11de01064d028adeb2d2edda4546e86002d5b43710fcdc00a33ee2403a1676d4", size = 3274994, upload-time = "2026-01-23T23:36:58.177Z" }, + { url = "https://files.pythonhosted.org/packages/8b/04/07ed2ae4b4dc9f31522fa971791fd7d7e38feac8ce2b9d3316394b2e5fe5/aic_sdk-2.0.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f48dde209a704a51e65a44c7846c033dc860003467cef0fc2d15d7f8aa137dbc", size = 4893276, upload-time = "2026-01-23T23:37:03.097Z" }, + { url = "https://files.pythonhosted.org/packages/58/87/6328bcf58e633acdf65fd72c4dee61f468fef399c0868e5c446b99166bf5/aic_sdk-2.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2017ea843fc9e38612a13f1b0a668428a3f6862792baf230ac79a65d9c0633d9", size = 4450341, upload-time = "2026-01-23T23:37:08.648Z" }, + { url = "https://files.pythonhosted.org/packages/fc/59/da5138346944ac7dc61ed70e66c1fb2fddef815dc2bab561316db5aef252/aic_sdk-2.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:065954c17116b96408ebfbff29152ca458bb083a9e56178c70adbafdda08218a", size = 3594974, upload-time = "2026-01-23T23:37:13.83Z" }, + { url = "https://files.pythonhosted.org/packages/06/d8/17e1a77820a6848efb7c97751bc6022f65c5ca6436dc3caf3a9da356def1/aic_sdk-2.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa7a196160d6eaf2b856c542bc967c2e08e11a5d93ac4e632f843a01b2872274", size = 4113591, upload-time = "2026-01-23T23:37:19.067Z" }, + { url = "https://files.pythonhosted.org/packages/38/3b/04b70a75364c2ef1717018a81963a8e16bffc3f9f064f125cb111870b6a4/aic_sdk-2.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:dbd007a683ebff4def95fa5a7ace1602aa2d150fa80761231b044edd57e98bbb", size = 3661883, upload-time = "2026-01-23T23:37:25.242Z" }, + { url = "https://files.pythonhosted.org/packages/2a/bd/64bc3ce090cc110f3721c5e54f97f9fcb67dc50bd8dc6408896650d1d68e/aic_sdk-2.0.1-cp312-cp312-win_arm64.whl", hash = "sha256:f776c5f0425b39073d4caca2f0bdea036647c4162d4673ef498e1306d41bb39e", size = 3271232, upload-time = "2026-01-23T23:37:30.005Z" }, + { url = "https://files.pythonhosted.org/packages/6e/72/8445a7201aa5969216b5d4ab60bb2ebefa2ac07f557e9ebca27172be2f00/aic_sdk-2.0.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:a7ff35422ffb813e8a5b4afed6eb56d4e8abc1ecabf464084d4c7b5b8aff0e43", size = 4892624, upload-time = "2026-01-23T23:37:33.229Z" }, + { url = "https://files.pythonhosted.org/packages/c7/9c/5b060cbd9e9bcea5e62df13cf3e722f4355286e2174c298ffcfe337c680d/aic_sdk-2.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6cda0b6db664712099da483f803128e4e5256625aed2d85e65e5cc823a0e873b", size = 4449490, upload-time = "2026-01-23T23:37:35.938Z" }, + { url = "https://files.pythonhosted.org/packages/e4/f8/ac61d007dc8d158a8f516327db74b6f3b1cf78b16be43acd29775197533d/aic_sdk-2.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ade8754bd878da0509e70636a9b4eaba0280741db5afabf752102ef605ffaac", size = 3594360, upload-time = "2026-01-23T23:37:38.943Z" }, + { url = "https://files.pythonhosted.org/packages/1b/c4/af0c00055450b060b23e8dd5f3c1a208ea1444c6b497eaf29d3de6e215fa/aic_sdk-2.0.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b2f44c660aa29613be05c576da25092b3570c8fb3dddc09b70625789d066202", size = 4112325, upload-time = "2026-01-23T23:37:41.613Z" }, + { url = "https://files.pythonhosted.org/packages/86/8e/62a7c53cc1bf2345ea20b554d1d8a61058301cd8088ff94ba95809b04a02/aic_sdk-2.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:ce1656991fc4dbbb40257c72a4b8fc4d4839363ca4b7b25a84ae5a83914ae90c", size = 3661441, upload-time = "2026-01-23T23:37:45.025Z" }, + { url = "https://files.pythonhosted.org/packages/39/98/aa9d6ccba0a1902f8480544fcf468dd3696ecb5392f02c2770f9020e6f9a/aic_sdk-2.0.1-cp313-cp313-win_arm64.whl", hash = "sha256:0138e964feb15d9fb5d2c9c64a8d45a807171900f53351e5525c26869237bd1c", size = 3270753, upload-time = "2026-01-23T23:37:47.882Z" }, + { url = "https://files.pythonhosted.org/packages/c0/3e/6a693ba223e2e55e142983c6243968222070405c6a90ec4c5a61b46652c1/aic_sdk-2.0.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:11eb0c3686ff83f340c875b864840fad19e3a98cd6e59815f83e9248a3ffb397", size = 4893527, upload-time = "2026-01-23T23:37:52.021Z" }, + { url = "https://files.pythonhosted.org/packages/35/9c/f149870d75f28c851de439d4039f85aa590f47499272f932841e4dc0a9a5/aic_sdk-2.0.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:27844521dc1ae3e1226e7371ec3e68fc4726515f14c5e82a6030f276b612c1a8", size = 4450169, upload-time = "2026-01-23T23:37:55.964Z" }, + { url = "https://files.pythonhosted.org/packages/92/87/8ee4e1763b603ad3d6d535d7ecfa7a2943145bcc18f2db4600279aa37af3/aic_sdk-2.0.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:adf617d4e4e8910764118d1baf1521c752218fd304c75d7e22352d4755cabd50", size = 3595300, upload-time = "2026-01-23T23:37:59.494Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b4/06d6f5c1b45d839d4d8ad4fbcb45dc224e980c69976c48e39d5e32850c51/aic_sdk-2.0.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2bc9071599b9703783b6b100758cd7621b30a64abddc8072f6872932b74c21", size = 4113837, upload-time = "2026-01-23T23:38:03.565Z" }, + { url = "https://files.pythonhosted.org/packages/1e/35/d7a2f7b37183b08b2e8969c3d6c6d1824253cd32894d72f250075edee654/aic_sdk-2.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:9db2bfb4f1ab40a4b130d8d0e158277461b25c7b78bffffba90f816766cb28e9", size = 3663055, upload-time = "2026-01-23T23:38:07.741Z" }, + { url = "https://files.pythonhosted.org/packages/90/97/9ed859e70b1d0c68edc9748c4e69d251e89a3faa462f36ce26c1f8aa7844/aic_sdk-2.0.1-cp314-cp314-win_arm64.whl", hash = "sha256:3c6ed1bfda589970e6c6b96ae29f112baa430ad91e149e76004825870198a5c7", size = 3272737, upload-time = "2026-01-23T23:38:13.966Z" }, +] [[package]] name = "aioboto3" @@ -531,18 +563,18 @@ wheels = [ [[package]] name = "azure-cognitiveservices-speech" -version = "1.44.0" +version = "1.47.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "azure-core" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/0d/0752835f079e8d2cc42bb634f3ccd761c8d6e9d0d46a2d6cf7b3ed8e714c/azure_cognitiveservices_speech-1.44.0-py3-none-macosx_10_14_x86_64.whl", hash = "sha256:78037a147ba72abb57e8c10b693d43a1bb029986fae0918f1f9b7d6342737bfe", size = 7492396, upload-time = "2025-05-19T15:46:11.318Z" }, - { url = "https://files.pythonhosted.org/packages/76/1d/d0ed4ec0f51303a2a532dc845eeb72c7729a3c8639b08050f3c1cd96db79/azure_cognitiveservices_speech-1.44.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:2c9b436326cd8dd82dfa88454b7b68359dfc7149e2ac9029f9bcff155ebd5c95", size = 7347577, upload-time = "2025-05-19T15:46:13.644Z" }, - { url = "https://files.pythonhosted.org/packages/89/c8/f0a4ea8bea014b912046f737e429378ceadad68258395454d62acf7f65bb/azure_cognitiveservices_speech-1.44.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:e5f07fc0587067850288c17aebf33d307d2c1ef9e0b2d11d9f44bff2af400568", size = 40977193, upload-time = "2025-05-19T15:46:15.878Z" }, - { url = "https://files.pythonhosted.org/packages/6a/0d/0a0394e8102d6660afeec6b780c451401f6074b1e19f00e90785529e459e/azure_cognitiveservices_speech-1.44.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:3461e22cf04816f69a964d936218d920240f987c0656fdaaf46571529ff0f7e6", size = 40747860, upload-time = "2025-05-19T15:46:19.316Z" }, - { url = "https://files.pythonhosted.org/packages/55/ad/3b7f6eca73040821358ce01f22067446a03d876bfed41cd784291706db4c/azure_cognitiveservices_speech-1.44.0-py3-none-win32.whl", hash = "sha256:a3fe7fd67ba7db281ae490de3d71b5a22648454ec2630eb6a70797f666330586", size = 2164045, upload-time = "2025-05-19T15:46:22.373Z" }, - { url = "https://files.pythonhosted.org/packages/83/ac/f491487d7d0e25ae2929b4f07e7f9b7456feb38e65b36fb605b2c9685b10/azure_cognitiveservices_speech-1.44.0-py3-none-win_amd64.whl", hash = "sha256:77cfb5dd40733b7ccc21edc427e9fb4720997832ea8a1ba460dc94345f3588ae", size = 2422937, upload-time = "2025-05-19T15:46:23.657Z" }, + { url = "https://files.pythonhosted.org/packages/cc/e3/b6a3d1ef4f135f8ef00ed084b9284e65409e9cd52bc96cd0453a5c6637c6/azure_cognitiveservices_speech-1.47.0-py3-none-macosx_10_14_x86_64.whl", hash = "sha256:656577ed01ed4b8cd7c70fab2c921b300181b906f101758a16406bc99b133681", size = 3574346, upload-time = "2025-11-11T21:13:37.717Z" }, + { url = "https://files.pythonhosted.org/packages/82/fa/9cc0c5400e9d433bd98a1239bedf97b34abf410dbc8932a50886ae43e115/azure_cognitiveservices_speech-1.47.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:afd91653ceca482ccea5459eedda1ec9aa95ee07df12a15fc588c42d4f90f0a9", size = 3506219, upload-time = "2025-11-11T21:13:39.702Z" }, + { url = "https://files.pythonhosted.org/packages/6b/d6/b8f55421b8cb40b478f4fb793c52b1bb0ed794263a5475ae2a6490a4cd53/azure_cognitiveservices_speech-1.47.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:577b702ee30d35ecc581e7e2ac23f4387782f93c241d7f8f3c86f72bb883d02d", size = 35399363, upload-time = "2025-11-11T21:13:41.915Z" }, + { url = "https://files.pythonhosted.org/packages/98/91/c36be146824797f57b194128a173baf289a260c2540c86c166f8c7fbebe3/azure_cognitiveservices_speech-1.47.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:ff72c74abe4b4c0f5a527eabf8511a8c0e689d884a95c54a46495b293e302e73", size = 35196906, upload-time = "2025-11-11T21:13:45.31Z" }, + { url = "https://files.pythonhosted.org/packages/fb/19/dd6f08dc623f2b336cc9cd5cf765712df5262fd675583e701922491e455d/azure_cognitiveservices_speech-1.47.0-py3-none-win_amd64.whl", hash = "sha256:ecfce57d66907afe305fb2950cc781ea8f327274facd2db66950e701b6cfd715", size = 2182376, upload-time = "2025-11-11T21:13:47.753Z" }, + { url = "https://files.pythonhosted.org/packages/1b/16/a6d1f7ab7eae21b00da2eee7186a7db9c9a2434e0ef833f071ff686b833f/azure_cognitiveservices_speech-1.47.0-py3-none-win_arm64.whl", hash = "sha256:4351734cf240d11340a057ecb388397e5ecf40e97e4b67a6a990fffe2791b56c", size = 1978493, upload-time = "2025-11-11T21:13:49.445Z" }, ] [[package]] @@ -4495,7 +4527,7 @@ docs = [ [package.metadata] requires-dist = [ { name = "accelerate", marker = "extra == 'moondream'", specifier = "~=1.10.0" }, - { name = "aic-sdk", marker = "extra == 'aic'", specifier = "~=1.2.0" }, + { name = "aic-sdk", marker = "extra == 'aic'", specifier = "~=2.0.1" }, { name = "aioboto3", marker = "extra == 'aws'", specifier = "~=15.5.0" }, { name = "aiofiles", specifier = ">=24.1.0,<25" }, { name = "aiohttp", specifier = ">=3.11.12,<4" }, @@ -4504,7 +4536,7 @@ requires-dist = [ { name = "audioop-lts", marker = "python_full_version >= '3.13'", specifier = "~=0.2.1" }, { name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12' and extra == 'aws-nova-sonic'", specifier = "~=0.2.0" }, { name = "aws-sdk-sagemaker-runtime-http2", marker = "python_full_version >= '3.12' and extra == 'sagemaker'" }, - { name = "azure-cognitiveservices-speech", marker = "extra == 'azure'", specifier = "~=1.44.0" }, + { name = "azure-cognitiveservices-speech", marker = "extra == 'azure'", specifier = "~=1.47.0" }, { name = "camb-sdk", marker = "extra == 'camb'", specifier = ">=1.5.4" }, { name = "cartesia", marker = "extra == 'cartesia'", specifier = "~=2.0.3" }, { name = "coremltools", marker = "extra == 'local-smart-turn'", specifier = ">=8.0" }, @@ -4586,7 +4618,7 @@ requires-dist = [ { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=1.0.3" }, { name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.1" }, { name = "soxr", specifier = "~=0.5.0" }, - { name = "speechmatics-voice", extras = ["smart"], marker = "extra == 'speechmatics'", specifier = ">=0.2.6" }, + { name = "speechmatics-voice", extras = ["smart"], marker = "extra == 'speechmatics'", specifier = ">=0.2.8" }, { name = "strands-agents", marker = "extra == 'strands'", specifier = ">=1.9.1,<2" }, { name = "tenacity", marker = "extra == 'livekit'", specifier = ">=8.2.3,<10.0.0" }, { name = "timm", marker = "extra == 'moondream'", specifier = "~=1.0.13" }, @@ -6420,16 +6452,16 @@ wheels = [ [[package]] name = "speechmatics-voice" -version = "0.2.7" +version = "0.2.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "pydantic" }, { name = "speechmatics-rt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4a/94/47280c7fa5264676bfd6a2373c5cbfa562d5f1aefd77d7f241641a4889a6/speechmatics_voice-0.2.7.tar.gz", hash = "sha256:392b5129d2cbc0059f122fdf960d88dc59df5f26808992ef031f2eb40713c936", size = 61137, upload-time = "2026-01-12T14:21:17.672Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/b2/72b5b2203bbefbd22e7692adaca0dd7c2feebed1aaea5599ec579f74fbbf/speechmatics_voice-0.2.8.tar.gz", hash = "sha256:b2d9cbf773fd94400c744734662e2b16b5bdc4271d0dafde46ac032c438fe000", size = 61419, upload-time = "2026-01-26T16:26:09.082Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/72/e74dbcd42935b31b1d188b8f9d932d9d4078ea5edf303bb0ba0af4203ba2/speechmatics_voice-0.2.7-py3-none-any.whl", hash = "sha256:79c6072a5bf21cfa75770b5e3855cff5747222b024c417a276d0b9c2ae83cd0c", size = 57323, upload-time = "2026-01-12T14:21:16.679Z" }, + { url = "https://files.pythonhosted.org/packages/89/2d/a2ab215a7a31fad5ef9267420dc9ced96d6d52e5b80b131ef41424607849/speechmatics_voice-0.2.8-py3-none-any.whl", hash = "sha256:423ac7620ae8c98f175faace2184ac4ab1fe448ffb41af57aae05ec655326f79", size = 57629, upload-time = "2026-01-26T16:26:07.59Z" }, ] [package.optional-dependencies]