From 3c76917c1e4a0c1c5b71467fb9813e6bfd44ad9e Mon Sep 17 00:00:00 2001 From: Corvin Jaedicke Date: Wed, 12 Nov 2025 13:48:22 +0100 Subject: [PATCH 1/4] use async process function --- pyproject.toml | 80 ++++++++++++------------- src/pipecat/audio/filters/aic_filter.py | 2 +- uv.lock | 6 +- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a203b47b0..7b382f112 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,23 +1,23 @@ [build-system] -requires = ["setuptools>=64", "setuptools_scm>=8"] +requires = [ "setuptools>=64", "setuptools_scm>=8" ] build-backend = "setuptools.build_meta" [project] name = "pipecat-ai" -dynamic = ["version"] +dynamic = [ "version" ] description = "An open source framework for voice (and multimodal) assistants" license = "BSD-2-Clause" -license-files = ["LICENSE"] +license-files = [ "LICENSE" ] readme = "README.md" requires-python = ">=3.10" -keywords = ["webrtc", "audio", "video", "ai"] +keywords = [ "webrtc", "audio", "video", "ai" ] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Topic :: Communications :: Conferencing", "Topic :: Multimedia :: Sound/Audio", "Topic :: Multimedia :: Video", - "Topic :: Scientific/Engineering :: Artificial Intelligence" + "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ "aiofiles>=24.1.0,<25", @@ -45,30 +45,30 @@ Source = "https://github.com/pipecat-ai/pipecat" Website = "https://pipecat.ai" [project.optional-dependencies] -aic = [ "aic-sdk~=1.0.1" ] +aic = [ "aic-sdk~=1.1.0" ] anthropic = [ "anthropic~=0.49.0" ] assemblyai = [ "pipecat-ai[websockets-base]" ] asyncai = [ "pipecat-ai[websockets-base]" ] aws = [ "aioboto3~=15.0.0", "pipecat-ai[websockets-base]" ] aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.1.1; python_version>='3.12'" ] -azure = [ "azure-cognitiveservices-speech~=1.42.0"] +azure = [ "azure-cognitiveservices-speech~=1.42.0" ] cartesia = [ "cartesia~=2.0.3", "pipecat-ai[websockets-base]" ] -cerebras = [] -deepseek = [] +cerebras = [ ] +deepseek = [ ] daily = [ "daily-python~=0.21.0" ] deepgram = [ "deepgram-sdk~=4.7.0" ] elevenlabs = [ "pipecat-ai[websockets-base]" ] fal = [ "fal-client~=0.5.9" ] -fireworks = [] +fireworks = [ ] fish = [ "ormsgpack~=1.7.0", "pipecat-ai[websockets-base]" ] gladia = [ "pipecat-ai[websockets-base]" ] google = [ "google-cloud-speech>=2.33.0,<3", "google-cloud-texttospeech>=2.31.0,<3", "google-genai>=1.41.0,<2", "pipecat-ai[websockets-base]" ] -grok = [] +grok = [ ] groq = [ "groq~=0.23.0" ] gstreamer = [ "pygobject~=3.50.0" ] heygen = [ "livekit>=1.0.13", "pipecat-ai[websockets-base]" ] hume = [ "hume>=0.11.2" ] -inworld = [] +inworld = [ ] krisp = [ "pipecat-ai-krisp~=0.4.0" ] koala = [ "pvkoala~=2.0.3" ] langchain = [ "langchain~=0.3.20", "langchain-community~=0.3.20", "langchain-openai~=0.3.9" ] @@ -77,35 +77,35 @@ lmnt = [ "pipecat-ai[websockets-base]" ] local = [ "pyaudio~=0.2.14" ] mcp = [ "mcp[cli]>=1.11.0,<2" ] mem0 = [ "mem0ai~=0.1.94" ] -mistral = [] +mistral = [ ] mlx-whisper = [ "mlx-whisper~=0.4.2" ] moondream = [ "accelerate~=1.10.0", "einops~=0.8.0", "pyvips[binary]~=3.0.0", "timm~=1.0.13", "transformers>=4.48.0" ] -nim = [] +nim = [ ] neuphonic = [ "pipecat-ai[websockets-base]" ] noisereduce = [ "noisereduce~=3.0.3" ] openai = [ "pipecat-ai[websockets-base]" ] openpipe = [ "openpipe>=4.50.0,<6" ] -openrouter = [] -perplexity = [] +openrouter = [ ] +perplexity = [ ] playht = [ "pipecat-ai[websockets-base]" ] -qwen = [] +qwen = [ ] rime = [ "pipecat-ai[websockets-base]" ] riva = [ "nvidia-riva-client~=2.21.1" ] -runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<0.122.0", "pipecat-ai-small-webrtc-prebuilt>=1.0.0"] -sambanova = [] +runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<0.122.0", "pipecat-ai-small-webrtc-prebuilt>=1.0.0" ] +sambanova = [ ] sarvam = [ "sarvamai==0.1.21", "pipecat-ai[websockets-base]" ] sentry = [ "sentry-sdk>=2.28.0,<3" ] local-smart-turn = [ "coremltools>=8.0", "transformers", "torch>=2.5.0,<3", "torchaudio>=2.5.0,<3" ] local-smart-turn-v3 = [ "transformers", "onnxruntime>=1.20.1,<2" ] -remote-smart-turn = [] +remote-smart-turn = [ ] silero = [ "onnxruntime>=1.20.1,<2" ] -simli = [ "simli-ai~=0.1.25"] +simli = [ "simli-ai~=0.1.25" ] soniox = [ "pipecat-ai[websockets-base]" ] soundfile = [ "soundfile~=0.13.1" ] speechmatics = [ "speechmatics-rt>=0.5.0" ] strands = [ "strands-agents>=1.9.1,<2" ] -tavus=[] -together = [] +tavus = [ ] +together = [ ] tracing = [ "opentelemetry-sdk>=1.33.0", "opentelemetry-api>=1.33.0", "opentelemetry-instrumentation>=0.54b0" ] ultravox = [ "transformers>=4.48.0", "vllm>=0.9.0" ] webrtc = [ "aiortc>=1.13.0,<2", "opencv-python>=4.11.0.86,<5" ] @@ -139,10 +139,10 @@ docs = [ ] [tool.setuptools.packages.find] -where = ["src"] +where = [ "src" ] [tool.setuptools.package-data] -"pipecat" = ["py.typed"] +"pipecat" = [ "py.typed" ] "pipecat.audio.dtmf" = [ "src/pipecat/audio/dtmf/dtmf-0.wav", "src/pipecat/audio/dtmf/dtmf-1.wav", @@ -157,13 +157,13 @@ where = ["src"] "src/pipecat/audio/dtmf/dtmf-pound.wav", "src/pipecat/audio/dtmf/dtmf-star.wav", ] -"pipecat.services.aws_nova_sonic" = ["src/pipecat/services/aws_nova_sonic/ready.wav"] -"pipecat.audio.turn.smart_turn.data" = ["src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx"] +"pipecat.services.aws_nova_sonic" = [ "src/pipecat/services/aws_nova_sonic/ready.wav" ] +"pipecat.audio.turn.smart_turn.data" = [ "src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx" ] [tool.pytest.ini_options] addopts = "--verbose" -testpaths = ["tests"] -pythonpath = ["src"] +testpaths = [ "tests" ] +pythonpath = [ "src" ] asyncio_default_fixture_loop_scope = "function" filterwarnings = [ "ignore:'audioop' is deprecated:DeprecationWarning", @@ -174,7 +174,7 @@ local_scheme = "no-local-version" fallback_version = "0.0.0-dev" [tool.ruff] -exclude = [".git", "*_pb2.py"] +exclude = [ ".git", "*_pb2.py" ] line-length = 100 [tool.ruff.lint] @@ -183,25 +183,25 @@ select = [ "I", # Import rules ] ignore = [ - "D105", # Missing docstring in magic methods (__str__, __repr__, etc.) + "D105", # Missing docstring in magic methods (__str__, __repr__, etc.) ] [tool.ruff.lint.per-file-ignores] # Skip docstring checks for non-source code -"examples/**/*.py" = ["D"] -"tests/**/*.py" = ["D"] -"scripts/**/*.py" = ["D"] -"docs/**/*.py" = ["D"] +"examples/**/*.py" = [ "D" ] +"tests/**/*.py" = [ "D" ] +"scripts/**/*.py" = [ "D" ] +"docs/**/*.py" = [ "D" ] # Skip D104 (missing docstring in public package) for __init__.py files -"**/__init__.py" = ["D104"] +"**/__init__.py" = [ "D104" ] # Skip specific rules for generated protobuf files -"**/*_pb2.py" = ["D"] -"src/pipecat/services/__init__.py" = ["D"] +"**/*_pb2.py" = [ "D" ] +"src/pipecat/services/__init__.py" = [ "D" ] [tool.ruff.lint.pydocstyle] convention = "google" [tool.coverage.run] command_line = "--module pytest" -source = ["src"] -omit = ["*/tests/*"] +source = [ "src" ] +omit = [ "*/tests/*" ] diff --git a/src/pipecat/audio/filters/aic_filter.py b/src/pipecat/audio/filters/aic_filter.py index 8c2f3e7f4..3f1af2f57 100644 --- a/src/pipecat/audio/filters/aic_filter.py +++ b/src/pipecat/audio/filters/aic_filter.py @@ -185,7 +185,7 @@ class AICFilter(BaseAudioFilter): ) # Process planar in-place; returns ndarray (same shape) - out_f32 = self._aic.process(block_f32) + out_f32 = await self._aic.process_async(block_f32) # Convert back to int16 bytes, planar layout out_i16 = np.clip(out_f32 * 32768.0, -32768, 32767).astype(np.int16) diff --git a/uv.lock b/uv.lock index 0dc6c74f8..eabe03714 100644 --- a/uv.lock +++ b/uv.lock @@ -36,12 +36,12 @@ wheels = [ [[package]] name = "aic-sdk" -version = "1.0.2" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/51/90/b02e853e863c303f8456c689b42ac24ad403b781adc9642d0a91ed4bed7e/aic_sdk-1.0.2.tar.gz", hash = "sha256:239097dd3aaa8a8a0fd7542b75d2510cb34144caec796370639b7c636acbc56e", size = 32059, upload-time = "2025-08-24T09:20:03.9Z" } +sdist = { url = "https://files.pythonhosted.org/packages/99/83/bf38b95d98c67b8ebc574fb4a4f23c07a3740b51992d7522976173d30b98/aic_sdk-1.1.0.tar.gz", hash = "sha256:04e08df695581c8cb4db8acca20e73815e9f449e7bd08e0162fd55518c727963", size = 34954, upload-time = "2025-11-11T20:45:24.25Z" } [[package]] name = "aioboto3" @@ -4647,7 +4647,7 @@ docs = [ [package.metadata] requires-dist = [ { name = "accelerate", marker = "extra == 'moondream'", specifier = "~=1.10.0" }, - { name = "aic-sdk", marker = "extra == 'aic'", specifier = "~=1.0.1" }, + { name = "aic-sdk", marker = "extra == 'aic'", specifier = "~=1.1.0" }, { name = "aioboto3", marker = "extra == 'aws'", specifier = "~=15.0.0" }, { name = "aiofiles", specifier = ">=24.1.0,<25" }, { name = "aiohttp", specifier = ">=3.11.12,<4" }, From a7b2052b38f054af4c30e6fb2c00f5b2146f71bc Mon Sep 17 00:00:00 2001 From: Corvin Jaedicke Date: Thu, 13 Nov 2025 14:20:35 +0100 Subject: [PATCH 2/4] add ai-coustics VAD --- .../07ad-interruptible-aicoustics.py | 51 +++--- pyproject.toml | 4 + src/pipecat/audio/filters/aic_filter.py | 52 ++++++ src/pipecat/audio/vad/aic_vad.py | 158 ++++++++++++++++++ uv.lock | 36 ++-- 5 files changed, 261 insertions(+), 40 deletions(-) create mode 100644 src/pipecat/audio/vad/aic_vad.py diff --git a/examples/foundational/07ad-interruptible-aicoustics.py b/examples/foundational/07ad-interruptible-aicoustics.py index aa647ff33..0f6832a28 100644 --- a/examples/foundational/07ad-interruptible-aicoustics.py +++ b/examples/foundational/07ad-interruptible-aicoustics.py @@ -15,7 +15,6 @@ from loguru import logger from pipecat.audio.filters.aic_filter import AICFilter from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline @@ -48,7 +47,7 @@ def _create_aic_filter() -> AICFilter: return AICFilter( license_key=license_key, - enhancement_level=1.0, + enhancement_level=0.5, ) @@ -56,27 +55,33 @@ def _create_aic_filter() -> AICFilter: # instantiated. The function will be called when the desired transport gets # selected. transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - audio_in_filter=_create_aic_filter(), - ), - "twilio": lambda: FastAPIWebsocketParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - audio_in_filter=_create_aic_filter(), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - audio_in_filter=_create_aic_filter(), - ), + "daily": lambda: ( + lambda aic: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=aic, + ) + )(_create_aic_filter()), + "twilio": lambda: ( + lambda aic: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=aic, + ) + )(_create_aic_filter()), + "webrtc": lambda: ( + lambda aic: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=aic.create_vad_analyzer(lookback_buffer_size=6.0, sensitivity=6.0), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + audio_in_filter=aic, + ) + )(_create_aic_filter()), } diff --git a/pyproject.toml b/pyproject.toml index 7b382f112..4e7a6130d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -128,6 +128,7 @@ dev = [ "setuptools~=78.1.1", "setuptools_scm~=8.3.1", "python-dotenv>=1.0.1,<2.0.0", + "pipecat-ai[aic,daily,deepgram,local-smart-turn-v3,openai,runner,silero,webrtc]", ] docs = [ @@ -205,3 +206,6 @@ convention = "google" command_line = "--module pytest" source = [ "src" ] omit = [ "*/tests/*" ] + +[tool.uv.sources] +pipecat-ai = { workspace = true } diff --git a/src/pipecat/audio/filters/aic_filter.py b/src/pipecat/audio/filters/aic_filter.py index 3f1af2f57..2f4699912 100644 --- a/src/pipecat/audio/filters/aic_filter.py +++ b/src/pipecat/audio/filters/aic_filter.py @@ -68,6 +68,58 @@ class AICFilter(BaseAudioFilter): # Model will be created in start() since the API now requires sample_rate self._aic = None + def get_vad_factory(self): + """Return a zero-arg factory that will create the VAD once the model exists. + + Returns: + A zero-argument callable that, when invoked, returns an initialized + VoiceActivityDetector bound to the underlying AIC model. Raises a + RuntimeError if the model has not been initialized (i.e. start() + has not been called successfully). + """ + + def _factory(): + if self._aic is None: + raise RuntimeError("AIC model not initialized yet. Call start(sample_rate) first.") + return self._aic.create_vad() + + return _factory + + def create_vad_analyzer( + self, + *, + lookback_buffer_size: Optional[float] = None, + sensitivity: Optional[float] = None, + ): + """Return an analyzer that will lazily instantiate the AIC VAD when ready. + + AIC VAD parameters: + - lookback_buffer_size: + Number of window-length audio buffers used as a lookback buffer. + Higher values increase prediction stability but add latency. + Range: 1.0 .. 20.0, Default (SDK): 6.0 + - sensitivity: + Energy threshold sensitivity. Energy threshold = 10 ** (-sensitivity). + Range: 1.0 .. 15.0, Default (SDK): 6.0 + + Args: + lookback_buffer_size: Optional lookback buffer size to configure on the VAD. + Range: 1.0 .. 20.0. If None, SDK default is used. + sensitivity: Optional sensitivity (energy threshold) to configure on the VAD. + Range: 1.0 .. 15.0. If None, SDK default is used. + + Returns: + A lazily-initialized AICVADAnalyzer that will bind to the VAD backend + once the filter's model has been created (after start(sample_rate)). + """ + from pipecat.audio.vad.aic_vad import AICVADAnalyzer + + return AICVADAnalyzer( + vad_factory=self.get_vad_factory(), + lookback_buffer_size=lookback_buffer_size, + sensitivity=sensitivity, + ) + async def start(self, sample_rate: int): """Initialize the filter with the transport's sample rate. diff --git a/src/pipecat/audio/vad/aic_vad.py b/src/pipecat/audio/vad/aic_vad.py new file mode 100644 index 000000000..4907e4f55 --- /dev/null +++ b/src/pipecat/audio/vad/aic_vad.py @@ -0,0 +1,158 @@ +"""AIC-integrated VAD analyzer that lazily binds to the AIC SDK backend. + +This analyzer queries the backend's is_speech_detected() and maps it to a float +confidence (1.0/0.0). It uses 10 ms windows based on the sample rate and applies +optional AIC VAD parameters (lookback_buffer_size, sensitivity) when available. +""" + +from typing import Any, Callable, Optional + +from loguru import logger + +from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams + +try: + from aic import AICVadParameter +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use the AIC filter, you need to `pip install pipecat-ai[aic]`.") + raise Exception(f"Missing module: {e}") + + +class AICVADAnalyzer(VADAnalyzer): + """VAD analyzer that lazily instantiates the AIC VoiceActivityDetector via a factory. + + The analyzer can be constructed before the AIC Model exists. Once the filter has + started and the Model is available, the provided factory will succeed and the + backend VAD will be created. We then switch to single-sample updates where + num_frames_required() returns 1 and confidence is derived from the backend's + boolean is_speech_detected() state. + + AIC VAD runtime parameters: + - lookback_buffer_size: + Controls the lookback buffer size used by the VAD, i.e. the number of + window-length audio buffers used as a lookback buffer. Larger values improve + stability but increase latency. + Range: 1.0 .. 20.0 + Default (SDK): 6.0 + - sensitivity: + Controls the energy threshold sensitivity. Higher values make the detector + less sensitive (require more energy to count as speech). + Range: 1.0 .. 15.0 + Formula: Energy threshold = 10 ** (-sensitivity) + Default (SDK): 6.0 + """ + + def __init__( + self, + *, + vad_factory: Optional[Callable[[], Any]] = None, + lookback_buffer_size: Optional[float] = None, + sensitivity: Optional[float] = None, + ): + """Create an AIC VAD analyzer. + + Args: + vad_factory: + Zero-arg callable that returns an initialized AIC VoiceActivityDetector. + This may raise until the filter's Model has been created; the analyzer + will retry on set_sample_rate/first use. + lookback_buffer_size: + Optional override for AIC VAD lookback buffer size. + Range: 1.0 .. 20.0. Larger values increase stability at the cost of latency. + If None, the SDK default (6.0) is used. + sensitivity: + Optional override for AIC VAD sensitivity (energy threshold). + Range: 1.0 .. 15.0. Energy threshold = 10 ** (-sensitivity). + If None, the SDK default (6.0) is used. + """ + # Use fixed VAD parameters for AIC: no user override + fixed_params = VADParams(confidence=0.5, start_secs=0.0, stop_secs=0.0, min_volume=0.0) + super().__init__(sample_rate=None, params=fixed_params) + self._vad_factory = vad_factory + self._backend_vad: Optional[Any] = None + self._pending_lookback: Optional[float] = lookback_buffer_size + self._pending_sensitivity: Optional[float] = sensitivity + + def bind_vad_factory(self, vad_factory: Callable[[], Any]): + """Attach or replace the factory post-construction.""" + self._vad_factory = vad_factory + self._ensure_backend_initialized() + + def _apply_backend_params(self): + """Apply optional AIC VAD parameters if available.""" + if self._backend_vad is None or AICVadParameter is None: + return + try: + if self._pending_lookback is not None: + self._backend_vad.set_parameter( + AICVadParameter.LOOKBACK_BUFFER_SIZE, float(self._pending_lookback) + ) + if self._pending_sensitivity is not None: + self._backend_vad.set_parameter( + AICVadParameter.SENSITIVITY, float(self._pending_sensitivity) + ) + except Exception as e: # noqa: BLE001 + logger.debug(f"AIC VAD parameter application deferred/failed: {e}") + + def _ensure_backend_initialized(self): + if self._backend_vad is not None: + return + if not self._vad_factory: + return + try: + self._backend_vad = self._vad_factory() + self._apply_backend_params() + # With backend ready, recompute internal frame sizing + super().set_params(self._params) + logger.debug("AIC VAD backend initialized in analyzer.") + except Exception as e: # noqa: BLE001 + # Filter may not be started yet; try again later + logger.debug(f"Deferring AIC VAD backend initialization: {e}") + + def set_sample_rate(self, sample_rate: int): + """Set the sample rate for audio processing. + + Args: + sample_rate: Audio sample rate in Hz. + """ + # Set rate and attempt backend initialization once we know SR + self._sample_rate = self._init_sample_rate or sample_rate + self._ensure_backend_initialized() + # Ensure params are initialized even if backend not ready yet + try: + super().set_params(self._params) + except Exception: + pass + + def num_frames_required(self) -> int: + """Get the number of audio frames required for analysis. + + Returns: + Number of frames needed for VAD processing. + """ + # Use 10 ms windows based on sample rate + return int(self.sample_rate * 0.01) if self.sample_rate > 0 else 160 + + def voice_confidence(self, buffer: bytes) -> float: + """Calculate voice activity confidence for the given audio buffer. + + Args: + buffer: Audio buffer to analyze. + + Returns: + Voice confidence score is 0.0 or 1.0. + """ + # Ensure backend exists (filter might have started since last call) + self._ensure_backend_initialized() + if self._backend_vad is None: + return 0.0 + + # We do not need to analyze 'buffer' here since the model's VAD is updated + # as part of the enhancement pipeline. Simply query the boolean and map it. + try: + is_speech = self._backend_vad.is_speech_detected() + return 1.0 if is_speech else 0.0 + except Exception as e: # noqa: BLE001 + logger.error(f"AIC VAD inference error: {e}") + return 0.0 diff --git a/uv.lock b/uv.lock index eabe03714..7fb531dea 100644 --- a/uv.lock +++ b/uv.lock @@ -4624,6 +4624,7 @@ dev = [ { name = "coverage" }, { name = "grpcio-tools" }, { name = "pip-tools" }, + { name = "pipecat-ai", extra = ["aic", "daily", "deepgram", "local-smart-turn-v3", "openai", "runner", "silero", "webrtc"] }, { name = "pre-commit" }, { name = "pyright" }, { name = "pytest" }, @@ -4697,23 +4698,23 @@ requires-dist = [ { name = "opentelemetry-sdk", marker = "extra == 'tracing'", specifier = ">=1.33.0" }, { name = "ormsgpack", marker = "extra == 'fish'", specifier = "~=1.7.0" }, { name = "pillow", specifier = ">=11.1.0,<12" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'assemblyai'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'asyncai'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'aws'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'cartesia'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'elevenlabs'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'fish'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'gladia'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'google'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'heygen'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'lmnt'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'neuphonic'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'openai'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'playht'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'rime'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'sarvam'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'soniox'" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'websocket'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'assemblyai'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'asyncai'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'aws'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'cartesia'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'elevenlabs'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'fish'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'gladia'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'google'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'heygen'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'lmnt'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'neuphonic'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'openai'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'playht'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'rime'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'sarvam'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'soniox'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'websocket'", editable = "." }, { name = "pipecat-ai-krisp", marker = "extra == 'krisp'", specifier = "~=0.4.0" }, { name = "pipecat-ai-small-webrtc-prebuilt", marker = "extra == 'runner'", specifier = ">=1.0.0" }, { name = "protobuf", specifier = "~=5.29.3" }, @@ -4753,6 +4754,7 @@ dev = [ { name = "coverage", specifier = "~=7.9.1" }, { name = "grpcio-tools", specifier = "~=1.67.1" }, { name = "pip-tools", specifier = "~=7.4.1" }, + { name = "pipecat-ai", extras = ["aic", "daily", "deepgram", "local-smart-turn-v3", "openai", "runner", "silero", "webrtc"], editable = "." }, { name = "pre-commit", specifier = "~=4.2.0" }, { name = "pyright", specifier = ">=1.1.404,<1.2" }, { name = "pytest", specifier = "~=8.4.1" }, From 2fab3e228638952705f1b30206c10caab965da4b Mon Sep 17 00:00:00 2001 From: Corvin Jaedicke Date: Thu, 13 Nov 2025 14:39:26 +0100 Subject: [PATCH 3/4] fix formatting --- pyproject.toml | 82 ++++++++++++++++++++++++-------------------------- uv.lock | 42 ++++++++++++-------------- 2 files changed, 59 insertions(+), 65 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4e7a6130d..e313c789d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,23 +1,23 @@ [build-system] -requires = [ "setuptools>=64", "setuptools_scm>=8" ] +requires = ["setuptools>=64", "setuptools_scm>=8"] build-backend = "setuptools.build_meta" [project] name = "pipecat-ai" -dynamic = [ "version" ] +dynamic = ["version"] description = "An open source framework for voice (and multimodal) assistants" license = "BSD-2-Clause" -license-files = [ "LICENSE" ] +license-files = ["LICENSE"] readme = "README.md" requires-python = ">=3.10" -keywords = [ "webrtc", "audio", "video", "ai" ] +keywords = ["webrtc", "audio", "video", "ai"] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Topic :: Communications :: Conferencing", "Topic :: Multimedia :: Sound/Audio", "Topic :: Multimedia :: Video", - "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Scientific/Engineering :: Artificial Intelligence" ] dependencies = [ "aiofiles>=24.1.0,<25", @@ -51,24 +51,24 @@ assemblyai = [ "pipecat-ai[websockets-base]" ] asyncai = [ "pipecat-ai[websockets-base]" ] aws = [ "aioboto3~=15.0.0", "pipecat-ai[websockets-base]" ] aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.1.1; python_version>='3.12'" ] -azure = [ "azure-cognitiveservices-speech~=1.42.0" ] +azure = [ "azure-cognitiveservices-speech~=1.42.0"] cartesia = [ "cartesia~=2.0.3", "pipecat-ai[websockets-base]" ] -cerebras = [ ] -deepseek = [ ] +cerebras = [] +deepseek = [] daily = [ "daily-python~=0.21.0" ] deepgram = [ "deepgram-sdk~=4.7.0" ] elevenlabs = [ "pipecat-ai[websockets-base]" ] fal = [ "fal-client~=0.5.9" ] -fireworks = [ ] +fireworks = [] fish = [ "ormsgpack~=1.7.0", "pipecat-ai[websockets-base]" ] gladia = [ "pipecat-ai[websockets-base]" ] google = [ "google-cloud-speech>=2.33.0,<3", "google-cloud-texttospeech>=2.31.0,<3", "google-genai>=1.41.0,<2", "pipecat-ai[websockets-base]" ] -grok = [ ] +grok = [] groq = [ "groq~=0.23.0" ] gstreamer = [ "pygobject~=3.50.0" ] heygen = [ "livekit>=1.0.13", "pipecat-ai[websockets-base]" ] hume = [ "hume>=0.11.2" ] -inworld = [ ] +inworld = [] krisp = [ "pipecat-ai-krisp~=0.4.0" ] koala = [ "pvkoala~=2.0.3" ] langchain = [ "langchain~=0.3.20", "langchain-community~=0.3.20", "langchain-openai~=0.3.9" ] @@ -77,35 +77,35 @@ lmnt = [ "pipecat-ai[websockets-base]" ] local = [ "pyaudio~=0.2.14" ] mcp = [ "mcp[cli]>=1.11.0,<2" ] mem0 = [ "mem0ai~=0.1.94" ] -mistral = [ ] +mistral = [] mlx-whisper = [ "mlx-whisper~=0.4.2" ] moondream = [ "accelerate~=1.10.0", "einops~=0.8.0", "pyvips[binary]~=3.0.0", "timm~=1.0.13", "transformers>=4.48.0" ] -nim = [ ] +nim = [] neuphonic = [ "pipecat-ai[websockets-base]" ] noisereduce = [ "noisereduce~=3.0.3" ] openai = [ "pipecat-ai[websockets-base]" ] openpipe = [ "openpipe>=4.50.0,<6" ] -openrouter = [ ] -perplexity = [ ] +openrouter = [] +perplexity = [] playht = [ "pipecat-ai[websockets-base]" ] -qwen = [ ] +qwen = [] rime = [ "pipecat-ai[websockets-base]" ] riva = [ "nvidia-riva-client~=2.21.1" ] -runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<0.122.0", "pipecat-ai-small-webrtc-prebuilt>=1.0.0" ] -sambanova = [ ] +runner = [ "python-dotenv>=1.0.0,<2.0.0", "uvicorn>=0.32.0,<1.0.0", "fastapi>=0.115.6,<0.122.0", "pipecat-ai-small-webrtc-prebuilt>=1.0.0"] +sambanova = [] sarvam = [ "sarvamai==0.1.21", "pipecat-ai[websockets-base]" ] sentry = [ "sentry-sdk>=2.28.0,<3" ] local-smart-turn = [ "coremltools>=8.0", "transformers", "torch>=2.5.0,<3", "torchaudio>=2.5.0,<3" ] local-smart-turn-v3 = [ "transformers", "onnxruntime>=1.20.1,<2" ] -remote-smart-turn = [ ] +remote-smart-turn = [] silero = [ "onnxruntime>=1.20.1,<2" ] -simli = [ "simli-ai~=0.1.25" ] +simli = [ "simli-ai~=0.1.25"] soniox = [ "pipecat-ai[websockets-base]" ] soundfile = [ "soundfile~=0.13.1" ] speechmatics = [ "speechmatics-rt>=0.5.0" ] strands = [ "strands-agents>=1.9.1,<2" ] -tavus = [ ] -together = [ ] +tavus=[] +together = [] tracing = [ "opentelemetry-sdk>=1.33.0", "opentelemetry-api>=1.33.0", "opentelemetry-instrumentation>=0.54b0" ] ultravox = [ "transformers>=4.48.0", "vllm>=0.9.0" ] webrtc = [ "aiortc>=1.13.0,<2", "opencv-python>=4.11.0.86,<5" ] @@ -128,7 +128,6 @@ dev = [ "setuptools~=78.1.1", "setuptools_scm~=8.3.1", "python-dotenv>=1.0.1,<2.0.0", - "pipecat-ai[aic,daily,deepgram,local-smart-turn-v3,openai,runner,silero,webrtc]", ] docs = [ @@ -140,10 +139,10 @@ docs = [ ] [tool.setuptools.packages.find] -where = [ "src" ] +where = ["src"] [tool.setuptools.package-data] -"pipecat" = [ "py.typed" ] +"pipecat" = ["py.typed"] "pipecat.audio.dtmf" = [ "src/pipecat/audio/dtmf/dtmf-0.wav", "src/pipecat/audio/dtmf/dtmf-1.wav", @@ -158,13 +157,13 @@ where = [ "src" ] "src/pipecat/audio/dtmf/dtmf-pound.wav", "src/pipecat/audio/dtmf/dtmf-star.wav", ] -"pipecat.services.aws_nova_sonic" = [ "src/pipecat/services/aws_nova_sonic/ready.wav" ] -"pipecat.audio.turn.smart_turn.data" = [ "src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx" ] +"pipecat.services.aws_nova_sonic" = ["src/pipecat/services/aws_nova_sonic/ready.wav"] +"pipecat.audio.turn.smart_turn.data" = ["src/pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx"] [tool.pytest.ini_options] addopts = "--verbose" -testpaths = [ "tests" ] -pythonpath = [ "src" ] +testpaths = ["tests"] +pythonpath = ["src"] asyncio_default_fixture_loop_scope = "function" filterwarnings = [ "ignore:'audioop' is deprecated:DeprecationWarning", @@ -175,7 +174,7 @@ local_scheme = "no-local-version" fallback_version = "0.0.0-dev" [tool.ruff] -exclude = [ ".git", "*_pb2.py" ] +exclude = [".git", "*_pb2.py"] line-length = 100 [tool.ruff.lint] @@ -184,28 +183,25 @@ select = [ "I", # Import rules ] ignore = [ - "D105", # Missing docstring in magic methods (__str__, __repr__, etc.) + "D105", # Missing docstring in magic methods (__str__, __repr__, etc.) ] [tool.ruff.lint.per-file-ignores] # Skip docstring checks for non-source code -"examples/**/*.py" = [ "D" ] -"tests/**/*.py" = [ "D" ] -"scripts/**/*.py" = [ "D" ] -"docs/**/*.py" = [ "D" ] +"examples/**/*.py" = ["D"] +"tests/**/*.py" = ["D"] +"scripts/**/*.py" = ["D"] +"docs/**/*.py" = ["D"] # Skip D104 (missing docstring in public package) for __init__.py files -"**/__init__.py" = [ "D104" ] +"**/__init__.py" = ["D104"] # Skip specific rules for generated protobuf files -"**/*_pb2.py" = [ "D" ] -"src/pipecat/services/__init__.py" = [ "D" ] +"**/*_pb2.py" = ["D"] +"src/pipecat/services/__init__.py" = ["D"] [tool.ruff.lint.pydocstyle] convention = "google" [tool.coverage.run] command_line = "--module pytest" -source = [ "src" ] -omit = [ "*/tests/*" ] - -[tool.uv.sources] -pipecat-ai = { workspace = true } +source = ["src"] +omit = ["*/tests/*"] diff --git a/uv.lock b/uv.lock index 7fb531dea..0dc6c74f8 100644 --- a/uv.lock +++ b/uv.lock @@ -36,12 +36,12 @@ wheels = [ [[package]] name = "aic-sdk" -version = "1.1.0" +version = "1.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/99/83/bf38b95d98c67b8ebc574fb4a4f23c07a3740b51992d7522976173d30b98/aic_sdk-1.1.0.tar.gz", hash = "sha256:04e08df695581c8cb4db8acca20e73815e9f449e7bd08e0162fd55518c727963", size = 34954, upload-time = "2025-11-11T20:45:24.25Z" } +sdist = { url = "https://files.pythonhosted.org/packages/51/90/b02e853e863c303f8456c689b42ac24ad403b781adc9642d0a91ed4bed7e/aic_sdk-1.0.2.tar.gz", hash = "sha256:239097dd3aaa8a8a0fd7542b75d2510cb34144caec796370639b7c636acbc56e", size = 32059, upload-time = "2025-08-24T09:20:03.9Z" } [[package]] name = "aioboto3" @@ -4624,7 +4624,6 @@ dev = [ { name = "coverage" }, { name = "grpcio-tools" }, { name = "pip-tools" }, - { name = "pipecat-ai", extra = ["aic", "daily", "deepgram", "local-smart-turn-v3", "openai", "runner", "silero", "webrtc"] }, { name = "pre-commit" }, { name = "pyright" }, { name = "pytest" }, @@ -4648,7 +4647,7 @@ docs = [ [package.metadata] requires-dist = [ { name = "accelerate", marker = "extra == 'moondream'", specifier = "~=1.10.0" }, - { name = "aic-sdk", marker = "extra == 'aic'", specifier = "~=1.1.0" }, + { name = "aic-sdk", marker = "extra == 'aic'", specifier = "~=1.0.1" }, { name = "aioboto3", marker = "extra == 'aws'", specifier = "~=15.0.0" }, { name = "aiofiles", specifier = ">=24.1.0,<25" }, { name = "aiohttp", specifier = ">=3.11.12,<4" }, @@ -4698,23 +4697,23 @@ requires-dist = [ { name = "opentelemetry-sdk", marker = "extra == 'tracing'", specifier = ">=1.33.0" }, { name = "ormsgpack", marker = "extra == 'fish'", specifier = "~=1.7.0" }, { name = "pillow", specifier = ">=11.1.0,<12" }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'assemblyai'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'asyncai'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'aws'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'cartesia'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'elevenlabs'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'fish'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'gladia'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'google'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'heygen'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'lmnt'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'neuphonic'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'openai'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'playht'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'rime'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'sarvam'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'soniox'", editable = "." }, - { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'websocket'", editable = "." }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'assemblyai'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'asyncai'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'aws'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'cartesia'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'elevenlabs'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'fish'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'gladia'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'google'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'heygen'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'lmnt'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'neuphonic'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'openai'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'playht'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'rime'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'sarvam'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'soniox'" }, + { name = "pipecat-ai", extras = ["websockets-base"], marker = "extra == 'websocket'" }, { name = "pipecat-ai-krisp", marker = "extra == 'krisp'", specifier = "~=0.4.0" }, { name = "pipecat-ai-small-webrtc-prebuilt", marker = "extra == 'runner'", specifier = ">=1.0.0" }, { name = "protobuf", specifier = "~=5.29.3" }, @@ -4754,7 +4753,6 @@ dev = [ { name = "coverage", specifier = "~=7.9.1" }, { name = "grpcio-tools", specifier = "~=1.67.1" }, { name = "pip-tools", specifier = "~=7.4.1" }, - { name = "pipecat-ai", extras = ["aic", "daily", "deepgram", "local-smart-turn-v3", "openai", "runner", "silero", "webrtc"], editable = "." }, { name = "pre-commit", specifier = "~=4.2.0" }, { name = "pyright", specifier = ">=1.1.404,<1.2" }, { name = "pytest", specifier = "~=8.4.1" }, From fbbad27d3769a95d460ebe7a40c559aa33ce1a12 Mon Sep 17 00:00:00 2001 From: Corvin Jaedicke Date: Fri, 14 Nov 2025 13:30:06 +0100 Subject: [PATCH 4/4] add changelog info --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cca64597..9fb339a13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Prevented `HeyGenVideoService` from automatically disconnecting after 5 minutes. +### Added + +- Added ai-coustics integrated VAD (`AICVADAnalyzer`) with `AICFilter` factory and + example wiring; leverages the enhancement model for robust detection with no + ONNX dependency or added processing complexity. + ## [0.0.94] - 2025-11-10 ### Deprecated