diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index de7ceb8c5..c1cba90a7 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -18,6 +18,7 @@ from pipecat.processors.aggregators.llm_response import ( from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.vad.silero import SileroVADAnalyzer from runner import configure @@ -41,6 +42,7 @@ async def main(room_url: str, token): audio_out_enabled=True, transcription_enabled=True, vad_enabled=True, + vad_analyzer=SileroVADAnalyzer() ) ) diff --git a/src/pipecat/transports/base_transport.py b/src/pipecat/transports/base_transport.py index 7b3561394..7f22d2c2c 100644 --- a/src/pipecat/transports/base_transport.py +++ b/src/pipecat/transports/base_transport.py @@ -6,12 +6,16 @@ from abc import ABC, abstractmethod +from pydantic import ConfigDict from pydantic.main import BaseModel from pipecat.processors.frame_processor import FrameProcessor +from pipecat.vad.vad_analyzer import VADAnalyzer class TransportParams(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + camera_out_enabled: bool = False camera_out_is_live: bool = False camera_out_width: int = 1024 @@ -27,6 +31,7 @@ class TransportParams(BaseModel): audio_in_channels: int = 1 vad_enabled: bool = False vad_audio_passthrough: bool = False + vad_analyzer: VADAnalyzer | None = None class BaseTransport(ABC): diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 506c962ea..e0d406d8e 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -161,12 +161,6 @@ class DailyTransportClient(EventHandler): "speaker", sample_rate=self._params.audio_in_sample_rate, channels=self._params.audio_in_channels) Daily.select_speaker_device("speaker") - self._vad_analyzer = None - if self._params.vad_enabled: - self._vad_analyzer = WebRTCVADAnalyzer( - sample_rate=self._params.audio_in_sample_rate, - num_channels=self._params.audio_in_channels) - @property def participant_id(self) -> str: return self._participant_id @@ -174,12 +168,6 @@ class DailyTransportClient(EventHandler): def set_callbacks(self, callbacks: DailyCallbacks): self._callbacks = callbacks - def vad_analyze(self, audio_frames: bytes) -> VADState: - state = VADState.QUIET - if self._vad_analyzer: - state = self._vad_analyzer.analyze_audio(audio_frames) - return state - def send_message(self, frame: DailyTransportMessageFrame): self._client.send_app_message(frame.message, frame.participant_id) @@ -435,6 +423,12 @@ class DailyInputTransport(BaseInputTransport): self._video_renderers = {} self._camera_in_queue = queue.Queue() + self._vad_analyzer = params.vad_analyzer + if params.vad_enabled and not params.vad_analyzer: + self._vad_analyzer = WebRTCVADAnalyzer( + sample_rate=self._params.audio_in_sample_rate, + num_channels=self._params.audio_in_channels) + async def start(self, frame: StartFrame): if self._running: return @@ -461,7 +455,10 @@ class DailyInputTransport(BaseInputTransport): await self._client.cleanup() def vad_analyze(self, audio_frames: bytes) -> VADState: - return self._client.vad_analyze(audio_frames) + state = VADState.QUIET + if self._vad_analyzer: + state = self._vad_analyzer.analyze_audio(audio_frames) + return state def read_raw_audio_frames(self, frame_count: int) -> bytes: return self._client.read_raw_audio_frames(frame_count) diff --git a/src/pipecat/vad/silero.py b/src/pipecat/vad/silero.py index cddfb9bf6..a6bb17e89 100644 --- a/src/pipecat/vad/silero.py +++ b/src/pipecat/vad/silero.py @@ -39,11 +39,10 @@ def int2float(sound): return sound -class SileroVAD(FrameProcessor, VADAnalyzer): +class SileroVADAnalyzer(VADAnalyzer): - def __init__(self, sample_rate=16000, audio_passthrough=False): - FrameProcessor.__init__(self) - VADAnalyzer.__init__(self, sample_rate=sample_rate, num_channels=1) + def __init__(self, sample_rate=16000): + super().__init__(sample_rate=sample_rate, num_channels=1) logger.debug("Loading Silero VAD model...") @@ -52,7 +51,6 @@ class SileroVAD(FrameProcessor, VADAnalyzer): ) self._processor_vad_state: VADState = VADState.QUIET - self._audio_passthrough = audio_passthrough logger.debug("Loaded Silero VAD") @@ -74,6 +72,15 @@ class SileroVAD(FrameProcessor, VADAnalyzer): logger.error(f"Error analyzing audio with Silero VAD: {e}") return 0 + +class SileroVAD(FrameProcessor): + + def __init__(self, sample_rate=16000, audio_passthrough=False): + super().__init__() + + self._vad_analyzer = SileroVADAnalyzer(sample_rate=sample_rate) + self._audio_passthrough = audio_passthrough + # # FrameProcessor # @@ -89,7 +96,7 @@ class SileroVAD(FrameProcessor, VADAnalyzer): async def _analyze_audio(self, frame: AudioRawFrame): # Check VAD and push event if necessary. We just care about changes # from QUIET to SPEAKING and vice versa. - new_vad_state = self.analyze_audio(frame.audio) + new_vad_state = self._vad_analyzer.analyze_audio(frame.audio) if new_vad_state != self._processor_vad_state and new_vad_state != VADState.STARTING and new_vad_state != VADState.STOPPING: new_frame = None