Loading the smart turn model.

dot-env.template

@@ -92,4 +92,7 @@ ASSEMBLYAI_API_KEY=...
 OPENROUTER_API_KEY=...
 
 # Piper
-PIPER_BASE_URL=...
+PIPER_BASE_URL=...
+
+# Local Smart turn
+LOCAL_SMART_TURN_MODEL_PATH=

pyproject.toml

@@ -79,6 +79,7 @@ qwen = []
 rime = [ "websockets~=13.1" ]
 riva = [ "nvidia-riva-client~=2.19.0" ]
 sentry = [ "sentry-sdk~=2.23.1" ]
+local-smart-turn = [ "coremltools>=8.0", "transformers" ]
 silero = [ "onnxruntime~=1.20.1" ]
 simli = [ "simli-ai~=0.1.10"]
 soundfile = [ "soundfile~=0.13.0" ]

src/pipecat/audio/turn/local_smart_turn.py

@@ -5,11 +5,23 @@
 #
 
 
+import os
+
 import numpy as np
 from loguru import logger
 
 from pipecat.audio.turn.base_turn_analyzer import BaseEndOfTurnAnalyzer, EndOfTurnState
 
+try:
+    import coremltools as ct
+    from transformers import AutoFeatureExtractor
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error(
+        "In order to use the LocalSmartTurnAnalyzer, you need to `pip install pipecat-ai[local-smart-turn]`."
+    )
+    raise Exception(f"Missing module: {e}")
+
 
 class LocalSmartTurnAnalyzer(BaseEndOfTurnAnalyzer):
     def __init__(self):
@@ -18,7 +30,32 @@ class LocalSmartTurnAnalyzer(BaseEndOfTurnAnalyzer):
 
         logger.debug("Loading Local Smart Turn model...")
 
-        # TODO: implement it
+        # To use this locally, set the environment variable LOCAL_SMART_TURN_MODEL_PATH
+        # to the path where the smart-turn repo is cloned.
+        #
+        # Example setup:
+        #
+        #   # Git LFS (Large File Storage)
+        #   brew install git-lfs
+        #   # Hugging Face uses LFS to store large model files, including .mlpackage
+        #   git lfs install
+        #   # Clone the repo with the smart_turn_classifier.mlpackage
+        #   git clone https://huggingface.co/pipecat-ai/smart-turn
+        #
+        # Then set the env variable:
+        #   export LOCAL_SMART_TURN_MODEL_PATH=./smart-turn
+        # or add it to your .env file
+        smart_turn_model_path = os.getenv("LOCAL_SMART_TURN_MODEL_PATH")
+
+        if not smart_turn_model_path:
+            logger.error("LOCAL_SMART_TURN_MODEL_PATH is not set.")
+            raise Exception("LOCAL_SMART_TURN_MODEL_PATH environment variable must be provided.")
+
+        core_ml_model_path = f"{smart_turn_model_path}/coreml/smart_turn_classifier.mlpackage"
+
+        # Only load the processor, not the torch model
+        processor = AutoFeatureExtractor.from_pretrained(smart_turn_model_path)
+        model = ct.models.MLModel(core_ml_model_path)
 
         logger.debug("Loaded Local Smart Turn")