From f0709e22ba5f2aea3fcf884f83ffca328e04b231 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Fri, 2 May 2025 14:21:29 -0300 Subject: [PATCH 1/4] Creating a local smart turn using torch. --- .../audio/turn/smart_turn/local_smart_turn.py | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 src/pipecat/audio/turn/smart_turn/local_smart_turn.py diff --git a/src/pipecat/audio/turn/smart_turn/local_smart_turn.py b/src/pipecat/audio/turn/smart_turn/local_smart_turn.py new file mode 100644 index 000000000..f8a8aa91a --- /dev/null +++ b/src/pipecat/audio/turn/smart_turn/local_smart_turn.py @@ -0,0 +1,71 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +from typing import Any, Dict + +import numpy as np +from loguru import logger + +from pipecat.audio.turn.smart_turn.base_smart_turn import BaseSmartTurn + +try: + import torch + from transformers import Wav2Vec2BertForSequenceClassification, AutoFeatureExtractor +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use the LocalSmartTurnAnalyzer, you need to `pip install pipecat-ai[local-smart-turn]`." + ) + raise Exception(f"Missing module: {e}") + + +class LocalSmartTurnAnalyzer(BaseSmartTurn): + def __init__(self, *, smart_turn_model_path: str, **kwargs): + super().__init__(**kwargs) + + if not smart_turn_model_path: + # Define the path to the pretrained model on Hugging Face + smart_turn_model_path = "pipecat-ai/smart-turn" + + logger.debug("Loading Local Smart Turn model...") + # Load the pretrained model for sequence classification + self._turn_model = Wav2Vec2BertForSequenceClassification.from_pretrained(smart_turn_model_path) + # Load the corresponding feature extractor for preprocessing audio + self._turn_processor = AutoFeatureExtractor.from_pretrained(smart_turn_model_path) + # Set device to GPU if available, else CPU + self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + # Move model to selected device and set it to evaluation mode + self._turn_model = self._turn_model.to(self._device) + self._turn_model.eval() + logger.debug("Loaded Local Smart Turn") + + async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]: + inputs = self._turn_processor( + audio_array, + sampling_rate=16000, + padding="max_length", + truncation=True, + max_length=800, # Maximum length as specified in training + return_attention_mask=True, + return_tensors="pt", + ) + + # Move input tensors to the same device as the model + inputs = {k: v.to(self._device) for k, v in inputs.items()} + + # Disable gradient calculation for inference + with torch.no_grad(): + outputs = self._turn_model(**inputs) + logits = outputs.logits + probabilities = torch.nn.functional.softmax(logits, dim=1) + completion_prob = probabilities[0, 1].item() # Probability of class 1 (Complete) + prediction = 1 if completion_prob > 0.5 else 0 + + return { + "prediction": prediction, + "probability": completion_prob, + } From 19dc0f2bfb6f15800fe3cc555e92ac742bf5bc72 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Fri, 2 May 2025 14:21:42 -0300 Subject: [PATCH 2/4] New example using the local smart turn --- examples/foundational/38b-smart-turn-local.py | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 examples/foundational/38b-smart-turn-local.py diff --git a/examples/foundational/38b-smart-turn-local.py b/examples/foundational/38b-smart-turn-local.py new file mode 100644 index 000000000..e95d06eac --- /dev/null +++ b/examples/foundational/38b-smart-turn-local.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import argparse +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn import LocalSmartTurnAnalyzer +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection + +load_dotenv(override=True) + + +async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + logger.info(f"Starting bot") + + # To use this locally, set the environment variable LOCAL_SMART_TURN_MODEL_PATH + # to the path where the smart-turn repo is cloned. + # + # Example setup: + # + # # Git LFS (Large File Storage) + # brew install git-lfs + # # Hugging Face uses LFS to store large model files, including .mlpackage + # git lfs install + # # Clone the repo with the smart_turn_classifier.mlpackage + # git clone https://huggingface.co/pipecat-ai/smart-turn + # + # Then set the env variable: + # export LOCAL_SMART_TURN_MODEL_PATH=./smart-turn + # or add it to your .env file + smart_turn_model_path = os.getenv("LOCAL_SMART_TURN_MODEL_PATH") + + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzer( + smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams() + ), + ), + ) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) + + +if __name__ == "__main__": + from run import main + + main() From 779f09af7064999d9b9916364741dec29b67d0f0 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Fri, 2 May 2025 14:22:38 -0300 Subject: [PATCH 3/4] Fixing lint. --- src/pipecat/audio/turn/smart_turn/local_smart_turn.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pipecat/audio/turn/smart_turn/local_smart_turn.py b/src/pipecat/audio/turn/smart_turn/local_smart_turn.py index f8a8aa91a..a3ee7ebf9 100644 --- a/src/pipecat/audio/turn/smart_turn/local_smart_turn.py +++ b/src/pipecat/audio/turn/smart_turn/local_smart_turn.py @@ -14,7 +14,7 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import BaseSmartTurn try: import torch - from transformers import Wav2Vec2BertForSequenceClassification, AutoFeatureExtractor + from transformers import AutoFeatureExtractor, Wav2Vec2BertForSequenceClassification except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error( @@ -33,7 +33,9 @@ class LocalSmartTurnAnalyzer(BaseSmartTurn): logger.debug("Loading Local Smart Turn model...") # Load the pretrained model for sequence classification - self._turn_model = Wav2Vec2BertForSequenceClassification.from_pretrained(smart_turn_model_path) + self._turn_model = Wav2Vec2BertForSequenceClassification.from_pretrained( + smart_turn_model_path + ) # Load the corresponding feature extractor for preprocessing audio self._turn_processor = AutoFeatureExtractor.from_pretrained(smart_turn_model_path) # Set device to GPU if available, else CPU From cb8a551db814e91a5e51569f5a3864361e67dca3 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Fri, 2 May 2025 14:32:18 -0300 Subject: [PATCH 4/4] Mentioning the new LocalSmartTurnAnalyzer in the changelog. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8402013f8..079142b9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added support for cross-platform local smart turn detection. You can use + `LocalSmartTurnAnalyzer` for on-device inference using Torch. + - `BaseOutputTransport` now allows multiple destinations if the transport implementation supports it (e.g. Daily's custom tracks). With multiple destinations it is possible to send different audio or video tracks with a