diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b6cdcab1..6d9d90df3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Introduce output transport audio mixers. Output transport audio mixers can be - used, for example, to add background sounds or any other audio mixing - functionality before the output audio is actually written to the transport. +- Introduce input transport audio filters (`BaseAudioFilter`). Audio filters can + be used to remove background noises before audio is sent to VAD. + +- Introduce output transport audio mixers (`BaseAudioMixer`). Output transport + audio mixers can be used, for example, to add background sounds or any other + audio mixing functionality before the output audio is actually written to the + transport. - Added `GatedOpenAILLMContextAggregator`. This aggregator keeps the last received OpenAI LLM context frame and it doesn't let it through until the diff --git a/src/pipecat/audio/filters/__init__.py b/src/pipecat/audio/filters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/audio/filters/base_audio_filter.py b/src/pipecat/audio/filters/base_audio_filter.py new file mode 100644 index 000000000..e635bb1ba --- /dev/null +++ b/src/pipecat/audio/filters/base_audio_filter.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from abc import ABC, abstractmethod + +from pipecat.frames.frames import FilterControlFrame + + +class BaseAudioFilter(ABC): + """This is a base class for input transport audio filters. If an audio + filter is provided to the input transport it will be used to process audio + before VAD and before pushing it downstream. There are control frames to + update filter settings or to enable or disable the filter at runtime. + + """ + + @abstractmethod + async def start(self, sample_rate: int): + """This will be called from the input transport when the transport is + started. It can be used to initialize the filter. The input transport + sample rate is provided so the filter can adjust to that sample rate. + + """ + pass + + @abstractmethod + async def stop(self): + """This will be called from the input transport when the transport is + stopping. + + """ + pass + + @abstractmethod + async def process_frame(self, frame: FilterControlFrame): + """This will be called when the input transport receives a + FilterControlFrame. + + """ + pass + + @abstractmethod + async def filter(self, audio: bytes) -> bytes: + pass diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index faa029635..7f7972f98 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -584,9 +584,30 @@ class VADParamsUpdateFrame(ControlFrame): params: VADParams +@dataclass +class FilterControlFrame(ControlFrame): + """Base control frame for other audio filter frames.""" + + pass + + +@dataclass +class FilterUpdateSettingsFrame(FilterControlFrame): + """Control frame to update filter settings.""" + + settings: Mapping[str, Any] + + +@dataclass +class FilterEnableFrame(FilterControlFrame): + """Control frame to enable or disable the filter at runtime.""" + + enable: bool + + @dataclass class MixerControlFrame(ControlFrame): - """Base control frame for other mixer frames.""" + """Base control frame for other audio mixer frames.""" pass diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index d66e8aa71..3dc393846 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -41,6 +41,9 @@ class BaseInputTransport(FrameProcessor): self._audio_task = None async def start(self, frame: StartFrame): + # Start audio filter. + if self._params.audio_in_filter: + await self._params.audio_in_filter.start(self._params.audio_in_sample_rate) # Create audio input queue and task if needed. if self._params.audio_in_enabled or self._params.vad_enabled: self._audio_in_queue = asyncio.Queue() @@ -52,6 +55,9 @@ class BaseInputTransport(FrameProcessor): self._audio_task.cancel() await self._audio_task self._audio_task = None + # Stop audio filter. + if self._params.audio_in_filter: + await self._params.audio_in_filter.stop() async def cancel(self, frame: CancelFrame): # Cancel and wait for the audio input task to finish. @@ -165,6 +171,17 @@ class BaseInputTransport(FrameProcessor): audio_passthrough = True + # If an audio filter is available, run it before VAD. + if self._params.audio_in_filter: + audio = await self._params.audio_in_filter.filter( + frame.audio, frame.sample_rate, frame.num_channels + ) + frame = InputAudioRawFrame( + audio=audio, + sample_rate=frame.sample_rate, + num_channels=frame.num_channels, + ) + # Check VAD and push event if necessary. We just care about # changes from QUIET to SPEAKING and vice versa. if self._params.vad_enabled: diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index 15091255d..e6c006145 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -75,6 +75,7 @@ class BaseOutputTransport(FrameProcessor): self._bot_speaking = False async def start(self, frame: StartFrame): + # Start audio mixer. if self._params.audio_out_mixer: await self._params.audio_out_mixer.start(self._params.audio_out_sample_rate) self._create_output_tasks() @@ -82,6 +83,7 @@ class BaseOutputTransport(FrameProcessor): async def stop(self, frame: EndFrame): await self._cancel_output_tasks() + # Stop audio mixer. if self._params.audio_out_mixer: await self._params.audio_out_mixer.stop() diff --git a/src/pipecat/transports/base_transport.py b/src/pipecat/transports/base_transport.py index 946a8fb90..3eac820f2 100644 --- a/src/pipecat/transports/base_transport.py +++ b/src/pipecat/transports/base_transport.py @@ -13,6 +13,7 @@ from typing import Optional from pydantic import ConfigDict from pydantic.main import BaseModel +from pipecat.audio.filters.base_audio_filter import BaseAudioFilter from pipecat.audio.mixers.base_audio_mixer import BaseAudioMixer from pipecat.audio.vad.vad_analyzer import VADAnalyzer from pipecat.processors.frame_processor import FrameProcessor @@ -39,6 +40,7 @@ class TransportParams(BaseModel): audio_in_enabled: bool = False audio_in_sample_rate: int = 16000 audio_in_channels: int = 1 + audio_in_filter: Optional[BaseAudioFilter] = None vad_enabled: bool = False vad_audio_passthrough: bool = False vad_analyzer: VADAnalyzer | None = None