Added support for Krisp audio filter

CHANGELOG.md

@@ -41,6 +41,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   grained control of what media subscriptions you want for each participant in a
   room.
 
+- Added audio filter `KrispFilter`.
+
 ### Changed
 
 - The following `DailyTransport` functions are now `async` which means they need

README.md

@@ -129,6 +129,24 @@ Pipecat makes use of WebRTC VAD by default when using a WebRTC transport layer.
 pip install pipecat-ai[silero]
 ```
 
+## Running the Krisp Audio Filter
+
+To use the Krisp Filter in this project, you’ll need access to the **Krisp C++ SDK**.
+
+### Step 1: Obtain Access to the Krisp SDK
+1. **Create a Krisp Account**: If you don’t already have an account, [sign up at Krisp](https://krisp.ai/) to access the SDK.
+2. **Download the SDK**: Once you have an account, follow the instructions on the Krisp platform to download the [Krisp's desktop SDKs](https://sdk.krisp.ai/sdk/desktop).
+3. **Export the path to you krisp SDK**:
+`export KRISP_SDK_PATH=/PATH/TO/KRISP/SDK`
+
+### Step 2: Install the `pipecat-krisp` Module
+Once the environment variable `KRISP_SDK_PATH` is exported, activate your Python virtual environment and install it with `pip`:
+
+```shell
+source venv/bin/activate
+pip install pipecat-ai[krisp]
+```
+
 ## Hacking on the framework itself
 
 _Note that you may need to set up a virtual environment before following the instructions below. For instance, you might need to run the following from the root of the repo:_

dot-env.template

@@ -52,4 +52,7 @@ OPENPIPE_API_KEY=...
 # Tavus
 TAVUS_API_KEY=...
 TAVUS_REPLICA_ID=...
-TAVUS_PERSONA_ID=...
+TAVUS_PERSONA_ID=...
+
+#Krisp
+KRISP_MODEL_PATH=...

examples/foundational/07p-interruptible-krisp.py

@@ -0,0 +1,95 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import os
+import sys
+
+import aiohttp
+from dotenv import load_dotenv
+from loguru import logger
+from runner import configure
+
+from pipecat.frames.frames import LLMMessagesFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.llm_response import (
+    LLMAssistantResponseAggregator,
+    LLMUserResponseAggregator,
+)
+from pipecat.services.deepgram import DeepgramSTTService, DeepgramTTSService
+from pipecat.services.openai import OpenAILLMService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+from pipecat.vad.silero import SileroVADAnalyzer
+from pipecat.audio.filters.krisp_filter import KrispFilter
+
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        (room_url, token) = await configure(session)
+
+        transport = DailyTransport(
+            room_url,
+            token,
+            "Respond bot",
+            DailyParams(
+                audio_out_enabled=True,
+                vad_enabled=True,
+                vad_analyzer=SileroVADAnalyzer(),
+                vad_audio_passthrough=True,
+                audio_in_filter=KrispFilter(),
+            ),
+        )
+
+        stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
+
+        tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en")
+
+        llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
+
+        messages = [
+            {
+                "role": "system",
+                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
+            },
+        ]
+
+        tma_in = LLMUserResponseAggregator(messages)
+        tma_out = LLMAssistantResponseAggregator(messages)
+
+        pipeline = Pipeline(
+            [
+                transport.input(),  # Transport user input
+                stt,  # STT
+                tma_in,  # User responses
+                llm,  # LLM
+                tts,  # TTS
+                transport.output(),  # Transport bot output
+                tma_out,  # Assistant spoken responses
+            ]
+        )
+
+        task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))
+
+        @transport.event_handler("on_first_participant_joined")
+        async def on_first_participant_joined(transport, participant):
+            # Kick off the conversation.
+            messages.append({"role": "system", "content": "Please introduce yourself to the user."})
+            await task.queue_frames([LLMMessagesFrame(messages)])
+
+        runner = PipelineRunner()
+
+        await runner.run(task)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

pyproject.toml

@@ -51,6 +51,7 @@ gladia = [ "websockets~=13.1" ]
 google = [ "google-generativeai~=0.8.3", "google-cloud-texttospeech~=2.17.2" ]
 gstreamer = [ "pygobject~=3.48.2" ]
 fireworks = [ "openai~=1.37.2" ]
+krisp = [ "pipecat-ai-krisp~=0.2.0" ]
 langchain = [ "langchain~=0.2.14", "langchain-community~=0.2.12", "langchain-openai~=0.1.20" ]
 livekit = [ "livekit~=0.17.5", "livekit-api~=0.7.1", "tenacity~=8.5.0" ]
 lmnt = [ "lmnt~=1.1.4" ]

src/pipecat/audio/filters/krisp_filter.py

@@ -0,0 +1,78 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import numpy as np
+import os
+
+from pipecat.audio.filters.base_audio_filter import BaseAudioFilter
+from loguru import logger
+from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame
+
+try:
+    from pipecat_ai_krisp.audio.krisp_processor import KrispAudioProcessor
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error("In order to use the Krisp filter, you need to `pip install pipecat-ai[krisp]`.")
+    raise Exception(f"Missing module: {e}")
+
+
+class KrispFilter(BaseAudioFilter):
+    def __init__(
+        self, sample_type: str = "PCM_16", channels: int = 1, model_path: str = None
+    ) -> None:
+        """
+        Initializes the KrispAudioProcessor with customizable audio processing settings.
+
+        :param sample_type: The type of audio sample, default is 'PCM_16'.
+        :param channels: Number of audio channels, default is 1.
+        :param model_path: Path to the Krisp model; defaults to environment variable KRISP_MODEL_PATH if not provided.
+        """
+        super().__init__()
+
+        # Set model path, checking environment if not specified
+        self._model_path = model_path or os.getenv("KRISP_MODEL_PATH")
+        if not self._model_path:
+            logger.error(
+                "Model path for KrispAudioProcessor is not provided and KRISP_MODEL_PATH is not set."
+            )
+            raise ValueError("Model path for KrispAudioProcessor must be provided.")
+
+        self._sample_type = sample_type
+        self._channels = channels
+        self._sample_rate = 0
+        self._filtering = True
+        self._krisp_processor = None
+
+    async def start(self, sample_rate: int):
+        self._sample_rate = sample_rate
+        self._krisp_processor = KrispAudioProcessor(
+            self._sample_rate, self._sample_type, self._channels, self._model_path
+        )
+
+    async def stop(self):
+        self._krisp_processor = None
+
+    async def process_frame(self, frame: FilterControlFrame):
+        if isinstance(frame, FilterEnableFrame):
+            self._filtering = frame.enable
+
+    async def filter(self, audio: bytes) -> bytes:
+        if not self._filtering:
+            return audio
+
+        data = np.frombuffer(audio, dtype=np.int16)
+
+        # Add a small epsilon to avoid division by zero.
+        epsilon = 1e-10
+        data = data.astype(np.float32) + epsilon
+
+        # Process the audio chunk to reduce noise
+        reduced_noise = self._krisp_processor.process(data)
+
+        # Clip and set processed audio back to frame
+        audio = np.clip(reduced_noise, -32768, 32767).astype(np.int16).tobytes()
+
+        return audio