diff --git a/src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py b/src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py index b9e2a7663..01c3746c8 100644 --- a/src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py +++ b/src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py @@ -20,6 +20,9 @@ from transformers import WhisperFeatureExtractor from pipecat.audio.turn.smart_turn.base_smart_turn import BaseSmartTurn from pipecat.utils.env import env_truthy +# The Whisper-based ONNX model expects 16 kHz audio input. +_MODEL_SAMPLE_RATE = 16000 + class LocalSmartTurnAnalyzerV3(BaseSmartTurn): """Local turn analyzer using the smart-turn-v3 ONNX model. @@ -42,6 +45,7 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn): super().__init__(**kwargs) self._log_data = env_truthy("PIPECAT_SMART_TURN_LOG_DATA", default=False) + self._resample_warned = False if not smart_turn_model_path: # Load bundled model @@ -77,7 +81,7 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn): logger.debug("Loaded Local Smart Turn v3.x") def _write_audio_to_wav( - self, audio_array: np.ndarray, sample_rate: int = 16000, suffix: str = "" + self, audio_array: np.ndarray, sample_rate: int = _MODEL_SAMPLE_RATE, suffix: str = "" ) -> None: """Write audio data to a WAV file in a background thread. @@ -119,10 +123,39 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn): thread = threading.Thread(target=write_wav, daemon=True) thread.start() + def _resample_to_model_rate(self, audio_array: np.ndarray) -> np.ndarray: + """Resample audio to the model's expected sample rate (16 kHz). + + Args: + audio_array: Audio data as a float32 numpy array. + + Returns: + Resampled audio array at 16 kHz. + """ + actual_rate = self._sample_rate or _MODEL_SAMPLE_RATE + if actual_rate == _MODEL_SAMPLE_RATE: + return audio_array + + if not self._resample_warned: + logger.warning( + f"Smart Turn v3 model expects {_MODEL_SAMPLE_RATE}Hz audio but received " + f"{actual_rate}Hz. Audio will be resampled automatically." + ) + self._resample_warned = True + + num_output_samples = int(len(audio_array) * _MODEL_SAMPLE_RATE / actual_rate) + return np.interp( + np.linspace(0, len(audio_array), num_output_samples, endpoint=False), + np.arange(len(audio_array)), + audio_array, + ) + def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]: """Predict end-of-turn using local ONNX model.""" - def truncate_audio_to_last_n_seconds(audio_array, n_seconds=8, sample_rate=16000): + def truncate_audio_to_last_n_seconds( + audio_array, n_seconds=8, sample_rate=_MODEL_SAMPLE_RATE + ): """Truncate audio to last n seconds or pad with zeros to meet n seconds.""" max_samples = n_seconds * sample_rate if len(audio_array) > max_samples: @@ -134,6 +167,10 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn): return audio_array audio_for_logging = audio_array + actual_rate = self._sample_rate or _MODEL_SAMPLE_RATE + + # Resample to 16 kHz if the pipeline uses a different sample rate + audio_array = self._resample_to_model_rate(audio_array) # Truncate to 8 seconds (keeping the end) or pad to 8 seconds audio_array = truncate_audio_to_last_n_seconds(audio_array, n_seconds=8) @@ -141,10 +178,10 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn): # Process audio using Whisper's feature extractor inputs = self._feature_extractor( audio_array, - sampling_rate=16000, + sampling_rate=_MODEL_SAMPLE_RATE, return_tensors="np", padding="max_length", - max_length=8 * 16000, + max_length=8 * _MODEL_SAMPLE_RATE, truncation=True, do_normalize=True, ) @@ -164,7 +201,7 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn): if self._log_data: suffix = "_complete" if prediction == 1 else "_incomplete" - self._write_audio_to_wav(audio_for_logging, sample_rate=16000, suffix=suffix) + self._write_audio_to_wav(audio_for_logging, sample_rate=actual_rate, suffix=suffix) return { "prediction": prediction,