diff --git a/CHANGELOG.md b/CHANGELOG.md index 01a0d3fa9..08e3e5231 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 grained control of what media subscriptions you want for each participant in a room. +- Added audio filter `KrispFilter`. + ### Changed - The following `DailyTransport` functions are now `async` which means they need diff --git a/README.md b/README.md index f387f9a87..52e570a32 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,24 @@ Pipecat makes use of WebRTC VAD by default when using a WebRTC transport layer. pip install pipecat-ai[silero] ``` +## Running the Krisp Audio Filter + +To use the Krisp Filter in this project, you’ll need access to the **Krisp C++ SDK**. + +### Step 1: Obtain Access to the Krisp SDK +1. **Create a Krisp Account**: If you don’t already have an account, [sign up at Krisp](https://krisp.ai/) to access the SDK. +2. **Download the SDK**: Once you have an account, follow the instructions on the Krisp platform to download the [Krisp's desktop SDKs](https://sdk.krisp.ai/sdk/desktop). +3. **Export the path to you krisp SDK**: +`export KRISP_SDK_PATH=/PATH/TO/KRISP/SDK` + +### Step 2: Install the `pipecat-krisp` Module +Once the environment variable `KRISP_SDK_PATH` is exported, activate your Python virtual environment and install it with `pip`: + +```shell +source venv/bin/activate +pip install pipecat-ai[krisp] +``` + ## Hacking on the framework itself _Note that you may need to set up a virtual environment before following the instructions below. For instance, you might need to run the following from the root of the repo:_ diff --git a/dot-env.template b/dot-env.template index 56a69fb78..4781f9beb 100644 --- a/dot-env.template +++ b/dot-env.template @@ -52,4 +52,7 @@ OPENPIPE_API_KEY=... # Tavus TAVUS_API_KEY=... TAVUS_REPLICA_ID=... -TAVUS_PERSONA_ID=... \ No newline at end of file +TAVUS_PERSONA_ID=... + +#Krisp +KRISP_MODEL_PATH=... \ No newline at end of file diff --git a/examples/foundational/07p-interruptible-krisp.py b/examples/foundational/07p-interruptible-krisp.py new file mode 100644 index 000000000..834e59037 --- /dev/null +++ b/examples/foundational/07p-interruptible-krisp.py @@ -0,0 +1,95 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.frames.frames import LLMMessagesFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_response import ( + LLMAssistantResponseAggregator, + LLMUserResponseAggregator, +) +from pipecat.services.deepgram import DeepgramSTTService, DeepgramTTSService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.vad.silero import SileroVADAnalyzer +from pipecat.audio.filters.krisp_filter import KrispFilter + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + audio_in_filter=KrispFilter(), + ), + ) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en") + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + tma_in = LLMUserResponseAggregator(messages) + tma_out = LLMAssistantResponseAggregator(messages) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + tma_in, # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + tma_out, # Assistant spoken responses + ] + ) + + task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMMessagesFrame(messages)]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml index 81c4113e3..f9d55b3e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ gladia = [ "websockets~=13.1" ] google = [ "google-generativeai~=0.8.3", "google-cloud-texttospeech~=2.17.2" ] gstreamer = [ "pygobject~=3.48.2" ] fireworks = [ "openai~=1.37.2" ] +krisp = [ "pipecat-ai-krisp~=0.2.0" ] langchain = [ "langchain~=0.2.14", "langchain-community~=0.2.12", "langchain-openai~=0.1.20" ] livekit = [ "livekit~=0.17.5", "livekit-api~=0.7.1", "tenacity~=8.5.0" ] lmnt = [ "lmnt~=1.1.4" ] diff --git a/src/pipecat/audio/filters/krisp_filter.py b/src/pipecat/audio/filters/krisp_filter.py new file mode 100644 index 000000000..0055c672b --- /dev/null +++ b/src/pipecat/audio/filters/krisp_filter.py @@ -0,0 +1,78 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import numpy as np +import os + +from pipecat.audio.filters.base_audio_filter import BaseAudioFilter +from loguru import logger +from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame + +try: + from pipecat_ai_krisp.audio.krisp_processor import KrispAudioProcessor +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use the Krisp filter, you need to `pip install pipecat-ai[krisp]`.") + raise Exception(f"Missing module: {e}") + + +class KrispFilter(BaseAudioFilter): + def __init__( + self, sample_type: str = "PCM_16", channels: int = 1, model_path: str = None + ) -> None: + """ + Initializes the KrispAudioProcessor with customizable audio processing settings. + + :param sample_type: The type of audio sample, default is 'PCM_16'. + :param channels: Number of audio channels, default is 1. + :param model_path: Path to the Krisp model; defaults to environment variable KRISP_MODEL_PATH if not provided. + """ + super().__init__() + + # Set model path, checking environment if not specified + self._model_path = model_path or os.getenv("KRISP_MODEL_PATH") + if not self._model_path: + logger.error( + "Model path for KrispAudioProcessor is not provided and KRISP_MODEL_PATH is not set." + ) + raise ValueError("Model path for KrispAudioProcessor must be provided.") + + self._sample_type = sample_type + self._channels = channels + self._sample_rate = 0 + self._filtering = True + self._krisp_processor = None + + async def start(self, sample_rate: int): + self._sample_rate = sample_rate + self._krisp_processor = KrispAudioProcessor( + self._sample_rate, self._sample_type, self._channels, self._model_path + ) + + async def stop(self): + self._krisp_processor = None + + async def process_frame(self, frame: FilterControlFrame): + if isinstance(frame, FilterEnableFrame): + self._filtering = frame.enable + + async def filter(self, audio: bytes) -> bytes: + if not self._filtering: + return audio + + data = np.frombuffer(audio, dtype=np.int16) + + # Add a small epsilon to avoid division by zero. + epsilon = 1e-10 + data = data.astype(np.float32) + epsilon + + # Process the audio chunk to reduce noise + reduced_noise = self._krisp_processor.process(data) + + # Clip and set processed audio back to frame + audio = np.clip(reduced_noise, -32768, 32767).astype(np.int16).tobytes() + + return audio