diff --git a/dot-env.template b/dot-env.template index f0b5bdc0f..ae0df90e5 100644 --- a/dot-env.template +++ b/dot-env.template @@ -92,4 +92,7 @@ ASSEMBLYAI_API_KEY=... OPENROUTER_API_KEY=... # Piper -PIPER_BASE_URL=... \ No newline at end of file +PIPER_BASE_URL=... + +# Local Smart turn +LOCAL_SMART_TURN_MODEL_PATH= \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 8b7c8546a..10b44508f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,7 @@ qwen = [] rime = [ "websockets~=13.1" ] riva = [ "nvidia-riva-client~=2.19.0" ] sentry = [ "sentry-sdk~=2.23.1" ] +local-smart-turn = [ "coremltools>=8.0", "transformers" ] silero = [ "onnxruntime~=1.20.1" ] simli = [ "simli-ai~=0.1.10"] soundfile = [ "soundfile~=0.13.0" ] diff --git a/src/pipecat/audio/turn/local_smart_turn.py b/src/pipecat/audio/turn/local_smart_turn.py index c511ae2ba..497487e06 100644 --- a/src/pipecat/audio/turn/local_smart_turn.py +++ b/src/pipecat/audio/turn/local_smart_turn.py @@ -5,11 +5,23 @@ # +import os + import numpy as np from loguru import logger from pipecat.audio.turn.base_turn_analyzer import BaseEndOfTurnAnalyzer, EndOfTurnState +try: + import coremltools as ct + from transformers import AutoFeatureExtractor +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use the LocalSmartTurnAnalyzer, you need to `pip install pipecat-ai[local-smart-turn]`." + ) + raise Exception(f"Missing module: {e}") + class LocalSmartTurnAnalyzer(BaseEndOfTurnAnalyzer): def __init__(self): @@ -18,7 +30,32 @@ class LocalSmartTurnAnalyzer(BaseEndOfTurnAnalyzer): logger.debug("Loading Local Smart Turn model...") - # TODO: implement it + # To use this locally, set the environment variable LOCAL_SMART_TURN_MODEL_PATH + # to the path where the smart-turn repo is cloned. + # + # Example setup: + # + # # Git LFS (Large File Storage) + # brew install git-lfs + # # Hugging Face uses LFS to store large model files, including .mlpackage + # git lfs install + # # Clone the repo with the smart_turn_classifier.mlpackage + # git clone https://huggingface.co/pipecat-ai/smart-turn + # + # Then set the env variable: + # export LOCAL_SMART_TURN_MODEL_PATH=./smart-turn + # or add it to your .env file + smart_turn_model_path = os.getenv("LOCAL_SMART_TURN_MODEL_PATH") + + if not smart_turn_model_path: + logger.error("LOCAL_SMART_TURN_MODEL_PATH is not set.") + raise Exception("LOCAL_SMART_TURN_MODEL_PATH environment variable must be provided.") + + core_ml_model_path = f"{smart_turn_model_path}/coreml/smart_turn_classifier.mlpackage" + + # Only load the processor, not the torch model + processor = AutoFeatureExtractor.from_pretrained(smart_turn_model_path) + model = ct.models.MLModel(core_ml_model_path) logger.debug("Loaded Local Smart Turn")