wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Krisp VIVA SDK Filter and Turn support. (#3261)

Browse Source 
* Krisp VIVA SDK Filter and Turn support.

* Reverted the krisp_filter.py as it's already deprectaed.

* enabled test with krisp_audio mock.

* More review comment fixes.
reverted the state logic in viva filter to be similar to the existing impl on main branch.
Fixed tests, ruff, etc.

* More review comments for Turn detection.
removed integration tests.

* Moved the SDK init/deinit into start/stop
This commit is contained in:
Garegin Harutyunyan
2026-01-09 17:15:08 +04:00
committed by GitHub
parent 72a44c2fcd
commit 16819a5caa
10 changed files with 2369 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/pipecat/audio/filters/krisp_filter.py
View File

@@ -61,7 +61,6 @@ class KrispFilter(BaseAudioFilter):
Provides real-time noise reduction for audio streams using Krisp's
proprietary noise suppression algorithms. Requires a Krisp model file
for operation.
.. deprecated:: 0.0.94
The KrispFilter is deprecated and will be removed in a future version.
Use KrispVivaFilter instead.

212
src/pipecat/audio/filters/krisp_viva_filter.py
View File

@@ -9,111 +9,121 @@
This module provides an audio filter implementation using Krisp VIVA SDK.
"""
import asyncio
import os
import numpy as np
from loguru import logger
from pipecat.audio.filters.base_audio_filter import BaseAudioFilter
from pipecat.audio.krisp_instance import (
KrispVivaSDKManager,
int_to_krisp_frame_duration,
int_to_krisp_sample_rate,
)
from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame
try:
import krisp_audio
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use the Krisp filter, you need to install krisp_audio.")
logger.error("In order to use KrispVivaFilter, you need to install krisp_audio.")
raise Exception(f"Missing module: {e}")
def _log_callback(log_message, log_level):
logger.info(f"[{log_level}] {log_message}")
class KrispVivaFilter(BaseAudioFilter):
"""Audio filter using the Krisp VIVA SDK.
Provides real-time noise reduction for audio streams using Krisp's
proprietary noise suppression algorithms. This filter requires a
valid Krisp model file to operate.
Supported sample rates:
- 8000 Hz
- 16000 Hz
- 24000 Hz
- 32000 Hz
- 44100 Hz
- 48000 Hz
"""
# Initialize Krisp Audio SDK globally
krisp_audio.globalInit("", _log_callback, krisp_audio.LogLevel.Off)
SDK_VERSION = krisp_audio.getVersion()
logger.debug(
f"Krisp Audio Python SDK Version: {SDK_VERSION.major}."
f"{SDK_VERSION.minor}.{SDK_VERSION.patch}"
)
SAMPLE_RATES = {
8000: krisp_audio.SamplingRate.Sr8000Hz,
16000: krisp_audio.SamplingRate.Sr16000Hz,
24000: krisp_audio.SamplingRate.Sr24000Hz,
32000: krisp_audio.SamplingRate.Sr32000Hz,
44100: krisp_audio.SamplingRate.Sr44100Hz,
48000: krisp_audio.SamplingRate.Sr48000Hz,
}
FRAME_SIZE_MS = 10 # Krisp requires audio frames of 10ms duration for processing.
def __init__(self, model_path: str = None, noise_suppression_level: int = 100) -> None:
def __init__(
self, model_path: str = None, frame_duration: int = 10, noise_suppression_level: int = 100
) -> None:
"""Initialize the Krisp noise reduction filter.
Args:
model_path: Path to the Krisp model file (.kef extension).
If None, uses KRISP_VIVA_MODEL_PATH environment variable.
If None, uses KRISP_VIVA_FILTER_MODEL_PATH environment variable.
frame_duration: Frame duration in milliseconds.
noise_suppression_level: Noise suppression level.
Raises:
ValueError: If model_path is not provided and KRISP_VIVA_MODEL_PATH is not set.
ValueError: If model_path is not provided and KRISP_VIVA_FILTER_MODEL_PATH is not set.
Exception: If model file doesn't have .kef extension.
FileNotFoundError: If model file doesn't exist.
RuntimeError: If Krisp SDK initialization fails.
"""
super().__init__()
# Set model path, checking environment if not specified
self._model_path = model_path or os.getenv("KRISP_VIVA_MODEL_PATH")
if not self._model_path:
logger.error("Model path is not provided and KRISP_VIVA_MODEL_PATH is not set.")
raise ValueError("Model path for KrispAudioProcessor must be provided.")
try:
# Set model path, checking environment if not specified
if model_path:
self._model_path = model_path
else:
# Check new environment variable first
self._model_path = os.getenv("KRISP_VIVA_FILTER_MODEL_PATH")
# Fall back to old environment variable for backward compatibility
if not self._model_path:
self._model_path = os.getenv("KRISP_VIVA_MODEL_PATH")
if self._model_path:
logger.warning(
"KRISP_VIVA_MODEL_PATH is deprecated. "
"Please use KRISP_VIVA_FILTER_MODEL_PATH instead."
)
if not self._model_path:
logger.error(
"Model path is not provided and KRISP_VIVA_FILTER_MODEL_PATH is not set."
)
raise ValueError("Model path for KrispAudioProcessor must be provided.")
if not self._model_path.endswith(".kef"):
raise Exception("Model is expected with .kef extension")
if not self._model_path.endswith(".kef"):
raise Exception("Model is expected with .kef extension")
if not os.path.isfile(self._model_path):
raise FileNotFoundError(f"Model file not found: {self._model_path}")
if not os.path.isfile(self._model_path):
raise FileNotFoundError(f"Model file not found: {self._model_path}")
self._filtering = True
self._session = None
self._samples_per_frame = None
self._noise_suppression_level = noise_suppression_level
self._session = None
self._samples_per_frame = None
self._noise_suppression_level = noise_suppression_level
self._frame_duration_ms = frame_duration
self._audio_buffer = bytearray()
self._filtering = True
# Audio buffer to accumulate samples for complete frames
self._audio_buffer = bytearray()
except Exception:
# If initialization fails, release the SDK reference
KrispVivaSDKManager.release()
raise
def _int_to_sample_rate(self, sample_rate):
"""Convert integer sample rate to krisp_audio SamplingRate enum.
def _create_session(self, sample_rate: int, frame_duration: int):
"""Create a Krisp session with a specific sample rate.
Args:
sample_rate: Sample rate as integer
Returns:
krisp_audio.SamplingRate enum value
sample_rate: Sample rate for the session
frame_duration: Frame duration in milliseconds
Raises:
ValueError: If sample rate is not supported
Exception: If session creation fails
"""
if sample_rate not in self.SAMPLE_RATES:
raise ValueError("Unsupported sample rate")
return self.SAMPLE_RATES[sample_rate]
try:
model_info = krisp_audio.ModelInfo()
model_info.path = self._model_path
nc_cfg = krisp_audio.NcSessionConfig()
nc_cfg.inputSampleRate = int_to_krisp_sample_rate(sample_rate)
nc_cfg.inputFrameDuration = int_to_krisp_frame_duration(frame_duration)
nc_cfg.outputSampleRate = nc_cfg.inputSampleRate
nc_cfg.modelInfo = model_info
self._samples_per_frame = int((sample_rate * frame_duration) / 1000)
self._current_sample_rate = sample_rate
session = krisp_audio.NcInt16.create(nc_cfg)
return session
except Exception as e:
logger.error(f"Failed to create Krisp session: {e}", exc_info=True)
raise RuntimeError(f"Failed to create Krisp processing session: {e}") from e
async def start(self, sample_rate: int):
"""Initialize the Krisp processor with the transport's sample rate.
@@ -121,21 +131,24 @@ class KrispVivaFilter(BaseAudioFilter):
Args:
sample_rate: The sample rate of the input transport in Hz.
"""
model_info = krisp_audio.ModelInfo()
model_info.path = self._model_path
nc_cfg = krisp_audio.NcSessionConfig()
nc_cfg.inputSampleRate = self._int_to_sample_rate(sample_rate)
nc_cfg.inputFrameDuration = krisp_audio.FrameDuration.Fd10ms
nc_cfg.outputSampleRate = nc_cfg.inputSampleRate
nc_cfg.modelInfo = model_info
self._samples_per_frame = int((sample_rate * self.FRAME_SIZE_MS) / 1000)
self._session = krisp_audio.NcInt16.create(nc_cfg)
try:
# Acquire SDK reference (will initialize on first call)
KrispVivaSDKManager.acquire()
self._session = self._create_session(sample_rate, self._frame_duration_ms)
except Exception as e:
logger.error(f"Failed to start Krisp session: {e}", exc_info=True)
self._session = None
raise RuntimeError(f"Failed to create Krisp processing session: {e}") from e
async def stop(self):
"""Clean up the Krisp processor when stopping."""
self._session = None
try:
self._session = None
self._audio_buffer.clear()
KrispVivaSDKManager.release()
except Exception as e:
logger.error(f"Error in stop: {e}", exc_info=True)
raise RuntimeError(f"Failed to stop Krisp processor: {e}") from e
async def process_frame(self, frame: FilterControlFrame):
"""Process control frames to enable/disable filtering.
@@ -158,36 +171,41 @@ class KrispVivaFilter(BaseAudioFilter):
if not self._filtering:
return audio
# Add incoming audio to our buffer
self._audio_buffer.extend(audio)
try:
# Add incoming audio to our buffer
self._audio_buffer.extend(audio)
# Calculate how many complete frames we can process
total_samples = len(self._audio_buffer) // 2 # 2 bytes per int16 sample
num_complete_frames = total_samples // self._samples_per_frame
# Calculate how many complete frames we can process
total_samples = len(self._audio_buffer) // 2 # 2 bytes per int16 sample
num_complete_frames = total_samples // self._samples_per_frame
if num_complete_frames == 0:
# Not enough samples for a complete frame yet, return empty
return b""
if num_complete_frames == 0:
# Not enough samples for a complete frame yet, return empty
return b""
# Calculate how many bytes we need for complete frames
complete_samples_count = num_complete_frames * self._samples_per_frame
bytes_to_process = complete_samples_count * 2 # 2 bytes per sample
# Calculate how many bytes we need for complete frames
complete_samples_count = num_complete_frames * self._samples_per_frame
bytes_to_process = complete_samples_count * 2 # 2 bytes per sample
# Extract the bytes we can process
audio_to_process = bytes(self._audio_buffer[:bytes_to_process])
# Extract the bytes we can process
audio_to_process = bytes(self._audio_buffer[:bytes_to_process])
# Remove processed bytes from buffer, keep the remainder
self._audio_buffer = self._audio_buffer[bytes_to_process:]
# Remove processed bytes from buffer, keep the remainder
self._audio_buffer = self._audio_buffer[bytes_to_process:]
# Process the complete frames
samples = np.frombuffer(audio_to_process, dtype=np.int16)
frames = samples.reshape(-1, self._samples_per_frame)
processed_samples = np.empty_like(samples)
# Process the complete frames
samples = np.frombuffer(audio_to_process, dtype=np.int16)
frames = samples.reshape(-1, self._samples_per_frame)
processed_samples = np.empty_like(samples)
for i, frame in enumerate(frames):
cleaned_frame = self._session.process(frame, self._noise_suppression_level)
processed_samples[i * self._samples_per_frame : (i + 1) * self._samples_per_frame] = (
cleaned_frame
)
for i, frame in enumerate(frames):
cleaned_frame = self._session.process(frame, self._noise_suppression_level)
processed_samples[
i * self._samples_per_frame : (i + 1) * self._samples_per_frame
] = cleaned_frame
return processed_samples.tobytes()
return processed_samples.tobytes()
except Exception as e:
logger.error(f"Error during Krisp filtering: {e}", exc_info=True)
return audio

183
src/pipecat/audio/krisp_instance.py Normal file
View File

@@ -0,0 +1,183 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Krisp Instance manager for pipecat audio."""
import atexit
from threading import Lock
from loguru import logger
try:
import krisp_audio
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use the Krisp instance, you need to install krisp_audio.")
raise Exception(f"Missing module: {e}")
# Mapping of sample rates (Hz) to Krisp SDK SamplingRate enums
KRISP_SAMPLE_RATES = {
8000: krisp_audio.SamplingRate.Sr8000Hz,
16000: krisp_audio.SamplingRate.Sr16000Hz,
24000: krisp_audio.SamplingRate.Sr24000Hz,
32000: krisp_audio.SamplingRate.Sr32000Hz,
44100: krisp_audio.SamplingRate.Sr44100Hz,
48000: krisp_audio.SamplingRate.Sr48000Hz,
}
KRISP_FRAME_DURATIONS = {
10: krisp_audio.FrameDuration.Fd10ms,
15: krisp_audio.FrameDuration.Fd15ms,
20: krisp_audio.FrameDuration.Fd20ms,
30: krisp_audio.FrameDuration.Fd30ms,
32: krisp_audio.FrameDuration.Fd32ms,
}
def int_to_krisp_sample_rate(sample_rate: int):
"""Convert integer sample rate to Krisp SDK enum value.
Args:
sample_rate: Sample rate in Hz (e.g., 16000, 24000, 48000).
Returns:
Corresponding Krisp SDK SampleRate enum value.
Raises:
ValueError: If the sample rate is not supported by Krisp SDK.
"""
if sample_rate not in KRISP_SAMPLE_RATES:
supported_rates = ", ".join(str(rate) for rate in sorted(KRISP_SAMPLE_RATES.keys()))
raise ValueError(
f"Unsupported sample rate: {sample_rate} Hz. Supported rates: {supported_rates} Hz"
)
return KRISP_SAMPLE_RATES[sample_rate]
def int_to_krisp_frame_duration(frame_duration_ms: int):
"""Convert integer frame duration to Krisp SDK enum value.
Args:
frame_duration_ms: Frame duration in milliseconds (e.g., 10, 20, 30).
Returns:
Corresponding Krisp SDK FrameDuration enum value.
Raises:
ValueError: If the frame duration is not supported by Krisp SDK.
"""
if frame_duration_ms not in KRISP_FRAME_DURATIONS:
supported_durations = ", ".join(
str(duration) for duration in sorted(KRISP_FRAME_DURATIONS.keys())
)
raise ValueError(
f"Unsupported frame duration: {frame_duration_ms} ms. "
f"Supported durations: {supported_durations} ms"
)
return KRISP_FRAME_DURATIONS[frame_duration_ms]
class KrispVivaSDKManager:
"""Singleton manager for Krisp VIVA SDK with reference counting."""
_initialized = False
_lock = Lock()
_reference_count = 0
@staticmethod
def _log_callback(log_message, log_level):
"""Thread-safe callback for Krisp SDK logging."""
logger.info(f"[{log_level}] {log_message}")
@classmethod
def acquire(cls):
"""Acquire a reference to the SDK (initializes if needed).
Call this when creating a filter instance.
Raises:
Exception: If SDK initialization fails (propagated from krisp_audio)
"""
with cls._lock:
# Initialize SDK on first acquire
if cls._reference_count == 0:
try:
krisp_audio.globalInit("", cls._log_callback, krisp_audio.LogLevel.Off)
cls._initialized = True
SDK_VERSION = krisp_audio.getVersion()
logger.debug(
f"Krisp Audio Python SDK initialized - Version: "
f"{SDK_VERSION.major}.{SDK_VERSION.minor}.{SDK_VERSION.patch}"
)
# Register cleanup on program exit (failsafe)
atexit.register(cls._force_cleanup)
except Exception as e:
cls._initialized = False
logger.error(f"Krisp SDK initialization failed: {e}")
raise
cls._reference_count += 1
logger.debug(f"Krisp SDK reference count: {cls._reference_count}")
@classmethod
def release(cls):
"""Release a reference to the SDK (destroys if last reference).
Call this when destroying a filter instance.
"""
with cls._lock:
if cls._reference_count > 0:
cls._reference_count -= 1
logger.debug(f"Krisp SDK reference count: {cls._reference_count}")
# Destroy SDK when last reference is released
if cls._reference_count == 0 and cls._initialized:
try:
krisp_audio.globalDestroy()
cls._initialized = False
logger.debug("Krisp Audio SDK destroyed (all references released)")
except Exception as e:
logger.error(f"Error during Krisp SDK cleanup: {e}")
cls._initialized = False
@classmethod
def get_reference_count(cls) -> int:
"""Get the current reference count.
Returns:
Current number of active references to the SDK.
"""
with cls._lock:
return cls._reference_count
@classmethod
def is_initialized(cls) -> bool:
"""Check if the SDK is currently initialized.
Returns:
True if SDK is initialized, False otherwise.
"""
with cls._lock:
return cls._initialized
@classmethod
def _force_cleanup(cls):
"""Force cleanup on program exit (failsafe)."""
with cls._lock:
if cls._initialized:
try:
logger.warning(
f"Force cleaning up Krisp SDK at exit (ref count: {cls._reference_count})"
)
krisp_audio.globalDestroy()
cls._initialized = False
except Exception as e:
logger.error(f"Error during forced Krisp SDK cleanup: {e}")

353
src/pipecat/audio/turn/krisp_viva_turn.py Normal file
View File

@@ -0,0 +1,353 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Krisp turn analyzer for end-of-turn detection using Krisp VIVA SDK.
This module provides a turn analyzer implementation using Krisp's turn detection
(Tt) API to determine when a user has finished speaking in a conversation.
Note: This analyzer uses a different model than KrispVivaFilter. The model path
can be specified via the KRISP_VIVA_TURN_MODEL_PATH environment variable or
passed directly to the constructor.
"""
import os
from typing import Optional, Tuple
import numpy as np
from loguru import logger
from pipecat.audio.krisp_instance import (
KrispVivaSDKManager,
int_to_krisp_frame_duration,
int_to_krisp_sample_rate,
)
from pipecat.audio.turn.base_turn_analyzer import BaseTurnAnalyzer, BaseTurnParams, EndOfTurnState
from pipecat.metrics.metrics import MetricsData
try:
import krisp_audio
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use KrispVivaTurn, you need to install krisp_audio.")
raise Exception(f"Missing module: {e}")
class KrispTurnParams(BaseTurnParams):
"""Configuration parameters for Krisp turn analysis.
Parameters:
threshold: Probability threshold for turn completion (0.0 to 1.0).
Higher values require more confidence before marking turn as complete.
frame_duration_ms: Frame duration in milliseconds for turn detection.
Supported values: 10, 15, 20, 30, 32.
"""
threshold: float = 0.5
frame_duration_ms: int = 20
class KrispVivaTurn(BaseTurnAnalyzer):
"""Turn analyzer using Krisp VIVA SDK for end-of-turn detection.
Uses Krisp's turn detection (Tt) API to determine when a user has finished
speaking. This analyzer requires a valid Krisp model file to operate.
"""
def __init__(
self,
*,
model_path: Optional[str] = None,
sample_rate: Optional[int] = None,
params: Optional[KrispTurnParams] = None,
) -> None:
"""Initialize the Krisp turn analyzer.
Args:
model_path: Path to the Krisp turn detection model file (.kef extension).
If None, uses KRISP_VIVA_TURN_MODEL_PATH environment variable.
sample_rate: Optional initial sample rate for audio processing.
If provided, this will be used as the fixed sample rate.
params: Configuration parameters for turn analysis behavior.
Raises:
ValueError: If model_path is not provided and KRISP_VIVA_TURN_MODEL_PATH is not set.
Exception: If model file doesn't have .kef extension.
FileNotFoundError: If model file doesn't exist.
RuntimeError: If Krisp SDK initialization fails.
"""
super().__init__(sample_rate=sample_rate)
# Acquire SDK reference (will initialize on first call)
try:
KrispVivaSDKManager.acquire()
self._sdk_acquired = True
except Exception as e:
self._sdk_acquired = False
raise RuntimeError(f"Failed to initialize Krisp SDK: {e}")
try:
# Set model path, checking environment if not specified
self._model_path = model_path or os.getenv("KRISP_VIVA_TURN_MODEL_PATH")
if not self._model_path:
logger.error(
"Model path is not provided and KRISP_VIVA_TURN_MODEL_PATH is not set."
)
raise ValueError("Model path for KrispVivaTurn must be provided.")
if not self._model_path.endswith(".kef"):
raise Exception("Model is expected with .kef extension")
if not os.path.isfile(self._model_path):
raise FileNotFoundError(f"Model file not found: {self._model_path}")
self._params = params or KrispTurnParams()
self._tt_session = None
self._preload_tt_session = None
self._samples_per_frame = None
self._audio_buffer = bytearray()
# State tracking
self._speech_triggered = False
self._last_probability = None
self._frame_probabilities = []
self._last_state = EndOfTurnState.INCOMPLETE
# Create session with provided sample rate or default to 16000 Hz
# This preloads the model to improve latency when set_sample_rate is called later
preload_sample_rate = sample_rate if sample_rate else 16000
try:
self._preload_tt_session = self._create_tt_session(preload_sample_rate)
except Exception as e:
logger.error(f"Failed to create turn detection session: {e}", exc_info=True)
self._preload_tt_session = None
raise RuntimeError(f"Failed to create turn detection session: {e}") from e
except Exception:
# If initialization fails, release the SDK reference
if self._sdk_acquired:
KrispVivaSDKManager.release()
self._sdk_acquired = False
raise
def __del__(self):
"""Release SDK reference when analyzer is destroyed."""
if self._sdk_acquired:
try:
# Clean up session first
if hasattr(self, "_tt_session") and self._tt_session is not None:
self._tt_session = None
if hasattr(self, "_preload_tt_session") and self._preload_tt_session is not None:
self._preload_tt_session = None
KrispVivaSDKManager.release()
self._sdk_acquired = False
except Exception as e:
logger.error(f"Error in __del__: {e}", exc_info=True)
def _create_tt_session(self, sample_rate: int):
"""Create a turn detection session with the specified sample rate.
Args:
sample_rate: Sample rate for the session
Returns:
krisp_audio.TtFloat instance
Raises:
ValueError: If sample rate or frame duration is not supported
RuntimeError: If session creation fails
"""
try:
model_info = krisp_audio.ModelInfo()
model_info.path = self._model_path
tt_cfg = krisp_audio.TtSessionConfig()
tt_cfg.inputSampleRate = int_to_krisp_sample_rate(sample_rate)
tt_cfg.inputFrameDuration = int_to_krisp_frame_duration(self._params.frame_duration_ms)
tt_cfg.modelInfo = model_info
# Calculate samples per frame for this sample rate
self._samples_per_frame = int((sample_rate * self._params.frame_duration_ms) / 1000)
tt_instance = krisp_audio.TtFloat.create(tt_cfg)
return tt_instance
except Exception as e:
logger.error(f"Failed to create Krisp turn detection session: {e}", exc_info=True)
raise RuntimeError(f"Failed to create Krisp turn detection session: {e}") from e
def set_sample_rate(self, sample_rate: int):
"""Set the sample rate and create/update the turn detection session.
Args:
sample_rate: The sample rate to set.
"""
if self._sample_rate == sample_rate:
return
super().set_sample_rate(sample_rate)
# Create session when sample rate is set
try:
self._tt_session = self._create_tt_session(self._sample_rate)
# Clear buffer when sample rate changes
self._audio_buffer.clear()
except Exception as e:
logger.error(f"Failed to create turn detection session: {e}", exc_info=True)
self._tt_session = None
@property
def frame_probabilities(self) -> list:
"""Get all probabilities from the last append_audio call.
Returns:
List of probability values for each frame processed in the last append_audio call.
"""
return self._frame_probabilities
@property
def last_probability(self) -> Optional[float]:
"""Get the last turn probability value computed.
Returns:
Last probability value, or None if no frames have been processed yet.
"""
return self._last_probability
@property
def speech_triggered(self) -> bool:
"""Check if speech has been detected and triggered analysis.
Returns:
True if speech has been detected and turn analysis is active.
"""
return self._speech_triggered
@property
def params(self) -> KrispTurnParams:
"""Get the current turn analyzer parameters.
Returns:
Current turn analyzer configuration parameters.
"""
return self._params
def append_audio(self, buffer: bytes, is_speech: bool) -> EndOfTurnState:
"""Append audio data for turn analysis.
Args:
buffer: Raw audio data bytes to append for analysis.
is_speech: Whether the audio buffer contains detected speech.
Returns:
Current end-of-turn state after processing the audio.
"""
if self._tt_session is None:
logger.warning("Turn detection session not initialized, returning INCOMPLETE")
self._last_state = EndOfTurnState.INCOMPLETE
return EndOfTurnState.INCOMPLETE
if self._samples_per_frame is None:
logger.warning("Samples per frame not initialized, returning INCOMPLETE")
self._last_state = EndOfTurnState.INCOMPLETE
return EndOfTurnState.INCOMPLETE
try:
# Add incoming audio to our buffer
self._audio_buffer.extend(buffer)
# Clear frame probabilities from previous call
self._frame_probabilities = []
total_samples = len(self._audio_buffer) // 2 # 2 bytes per int16 sample
num_complete_frames = total_samples // self._samples_per_frame
if num_complete_frames == 0:
# Not enough samples for a complete frame yet, return current state
self._last_state = EndOfTurnState.INCOMPLETE
return EndOfTurnState.INCOMPLETE
complete_samples_count = num_complete_frames * self._samples_per_frame
bytes_to_process = complete_samples_count * 2 # 2 bytes per sample
audio_to_process = bytes(self._audio_buffer[:bytes_to_process])
self._audio_buffer = self._audio_buffer[bytes_to_process:]
audio_int16 = np.frombuffer(audio_to_process, dtype=np.int16)
audio_float32 = audio_int16.astype(np.float32) / 32768.0
frames = audio_float32.reshape(-1, self._samples_per_frame)
state = EndOfTurnState.INCOMPLETE
# Process each complete frame
for frame in frames:
if is_speech:
# Track speech start time
if not self._speech_triggered:
logger.trace("Speech detected, turn analysis started")
self._speech_triggered = True
# Note: We don't immediately mark as complete on silence detection.
# Instead, we wait for the model's probability check below to confirm
# end-of-turn based on the threshold.
prob = self._tt_session.process(frame.tolist())
# Negative values indicate the model is not ready yet (working with 100ms data)
# Skip processing until we get positive probabilities
if prob < 0:
continue
# Store the probability for external access
self._last_probability = prob
self._frame_probabilities.append(prob)
# Check if turn is complete based on probability threshold
# Only mark as complete if we've detected speech and the model
# confirms with sufficient confidence
if self._speech_triggered and prob >= self._params.threshold:
state = EndOfTurnState.COMPLETE
self._clear(state)
break
# Store the last state for analyze_end_of_turn()
self._last_state = state
return state
except Exception as e:
logger.error(f"Error during Krisp turn detection: {e}", exc_info=True)
error_state = EndOfTurnState.INCOMPLETE
self._last_state = error_state
return error_state
async def analyze_end_of_turn(self) -> Tuple[EndOfTurnState, Optional[MetricsData]]:
"""Analyze the current audio state to determine if turn has ended.
Returns:
Tuple containing the end-of-turn state and optional metrics data.
Returns the last state determined by append_audio().
"""
# For real-time processing, the state is determined in append_audio
# Return the last state that was computed
return self._last_state, None
def clear(self):
"""Reset the turn analyzer to its initial state."""
self._clear(EndOfTurnState.COMPLETE)
def _clear(self, turn_state: EndOfTurnState):
"""Clear internal state based on turn completion status.
Args:
turn_state: The end-of-turn state to use for clearing.
"""
# If the state is still incomplete, keep the _speech_triggered as True
self._speech_triggered = turn_state == EndOfTurnState.INCOMPLETE
# Clear audio buffer on turn completion
if turn_state == EndOfTurnState.COMPLETE:
self._audio_buffer.clear()
# Reset last state when clearing
self._last_state = EndOfTurnState.INCOMPLETE