From 442ea6a97e0c3835757f5651f1ea7f37d5485930 Mon Sep 17 00:00:00 2001
From: Rupesh <rupesh@wayline.com>
Date: Thu, 26 Feb 2026 19:36:21 -0800
Subject: [PATCH] Fix Smart Turn v3 producing incorrect predictions at
 non-16kHz sample rates

The Whisper-based ONNX model expects 16 kHz audio, but the
_predict_endpoint method had five hardcoded references to 16000 without
checking the actual pipeline sample rate. When running at 8 kHz (e.g.
Twilio telephony), audio was fed to the feature extractor at the wrong
rate, causing the model to perceive speech at 2x speed with shifted
formant frequencies and produce incorrect end-of-turn predictions.

Add automatic resampling via numpy interpolation before feature
extraction and replace all hardcoded sample rate values with a
_MODEL_SAMPLE_RATE constant. Also fix the WAV debug logger to write
files with the correct sample rate header.

Fixes #3844
---
 .../turn/smart_turn/local_smart_turn_v3.py    | 47 +++++++++++++++++--
 1 file changed, 42 insertions(+), 5 deletions(-)

diff --git a/src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py b/src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py
index b9e2a7663..01c3746c8 100644
--- a/src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py
+++ b/src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py
@@ -20,6 +20,9 @@ from transformers import WhisperFeatureExtractor
 from pipecat.audio.turn.smart_turn.base_smart_turn import BaseSmartTurn
 from pipecat.utils.env import env_truthy
 
+# The Whisper-based ONNX model expects 16 kHz audio input.
+_MODEL_SAMPLE_RATE = 16000
+
 
 class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
     """Local turn analyzer using the smart-turn-v3 ONNX model.
@@ -42,6 +45,7 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
         super().__init__(**kwargs)
 
         self._log_data = env_truthy("PIPECAT_SMART_TURN_LOG_DATA", default=False)
+        self._resample_warned = False
 
         if not smart_turn_model_path:
             # Load bundled model
@@ -77,7 +81,7 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
         logger.debug("Loaded Local Smart Turn v3.x")
 
     def _write_audio_to_wav(
-        self, audio_array: np.ndarray, sample_rate: int = 16000, suffix: str = ""
+        self, audio_array: np.ndarray, sample_rate: int = _MODEL_SAMPLE_RATE, suffix: str = ""
     ) -> None:
         """Write audio data to a WAV file in a background thread.
 
@@ -119,10 +123,39 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
         thread = threading.Thread(target=write_wav, daemon=True)
         thread.start()
 
+    def _resample_to_model_rate(self, audio_array: np.ndarray) -> np.ndarray:
+        """Resample audio to the model's expected sample rate (16 kHz).
+
+        Args:
+            audio_array: Audio data as a float32 numpy array.
+
+        Returns:
+            Resampled audio array at 16 kHz.
+        """
+        actual_rate = self._sample_rate or _MODEL_SAMPLE_RATE
+        if actual_rate == _MODEL_SAMPLE_RATE:
+            return audio_array
+
+        if not self._resample_warned:
+            logger.warning(
+                f"Smart Turn v3 model expects {_MODEL_SAMPLE_RATE}Hz audio but received "
+                f"{actual_rate}Hz. Audio will be resampled automatically."
+            )
+            self._resample_warned = True
+
+        num_output_samples = int(len(audio_array) * _MODEL_SAMPLE_RATE / actual_rate)
+        return np.interp(
+            np.linspace(0, len(audio_array), num_output_samples, endpoint=False),
+            np.arange(len(audio_array)),
+            audio_array,
+        )
+
     def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
         """Predict end-of-turn using local ONNX model."""
 
-        def truncate_audio_to_last_n_seconds(audio_array, n_seconds=8, sample_rate=16000):
+        def truncate_audio_to_last_n_seconds(
+            audio_array, n_seconds=8, sample_rate=_MODEL_SAMPLE_RATE
+        ):
             """Truncate audio to last n seconds or pad with zeros to meet n seconds."""
             max_samples = n_seconds * sample_rate
             if len(audio_array) > max_samples:
@@ -134,6 +167,10 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
             return audio_array
 
         audio_for_logging = audio_array
+        actual_rate = self._sample_rate or _MODEL_SAMPLE_RATE
+
+        # Resample to 16 kHz if the pipeline uses a different sample rate
+        audio_array = self._resample_to_model_rate(audio_array)
 
         # Truncate to 8 seconds (keeping the end) or pad to 8 seconds
         audio_array = truncate_audio_to_last_n_seconds(audio_array, n_seconds=8)
@@ -141,10 +178,10 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
         # Process audio using Whisper's feature extractor
         inputs = self._feature_extractor(
             audio_array,
-            sampling_rate=16000,
+            sampling_rate=_MODEL_SAMPLE_RATE,
             return_tensors="np",
             padding="max_length",
-            max_length=8 * 16000,
+            max_length=8 * _MODEL_SAMPLE_RATE,
             truncation=True,
             do_normalize=True,
         )
@@ -164,7 +201,7 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
 
         if self._log_data:
             suffix = "_complete" if prediction == 1 else "_incomplete"
-            self._write_audio_to_wav(audio_for_logging, sample_rate=16000, suffix=suffix)
+            self._write_audio_to_wav(audio_for_logging, sample_rate=actual_rate, suffix=suffix)
 
         return {
             "prediction": prediction,