diff --git a/CHANGELOG.md b/CHANGELOG.md index ec717db58..02dbb7865 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated `pyproject.toml` to once again pin `numba` to `>=0.61.2` in order to resolve package versioning issues. +- Updated the `STTMuteFilter` to include `VADUserStartedSpeakingFrame` and `VADUserStoppedSpeakingFrame` in the list of frames to filter when the filtering is on. ### Other diff --git a/src/pipecat/processors/filters/stt_mute_filter.py b/src/pipecat/processors/filters/stt_mute_filter.py index 5e77b60ee..bce767164 100644 --- a/src/pipecat/processors/filters/stt_mute_filter.py +++ b/src/pipecat/processors/filters/stt_mute_filter.py @@ -32,6 +32,8 @@ from pipecat.frames.frames import ( TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, + VADUserStartedSpeakingFrame, + VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -205,6 +207,8 @@ class STTMuteFilter(FrameProcessor): ( StartInterruptionFrame, StopInterruptionFrame, + VADUserStartedSpeakingFrame, + VADUserStoppedSpeakingFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, InputAudioRawFrame, diff --git a/tests/test_stt_mute_filter.py b/tests/test_stt_mute_filter.py index f0c0d7d57..161962224 100644 --- a/tests/test_stt_mute_filter.py +++ b/tests/test_stt_mute_filter.py @@ -17,6 +17,8 @@ from pipecat.frames.frames import ( TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, + VADUserStartedSpeakingFrame, + VADUserStoppedSpeakingFrame, ) from pipecat.processors.filters.stt_mute_filter import STTMuteConfig, STTMuteFilter, STTMuteStrategy from pipecat.tests.utils import SleepFrame, run_test @@ -28,15 +30,19 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): frames_to_send = [ BotStartedSpeakingFrame(), # First bot speech starts + VADUserStartedSpeakingFrame(), # Should be suppressed UserStartedSpeakingFrame(), # Should be suppressed InputAudioRawFrame( audio=b"", sample_rate=16000, num_channels=1 ), # Should be suppressed + VADUserStoppedSpeakingFrame(), # Should be suppressed UserStoppedSpeakingFrame(), # Should be suppressed BotStoppedSpeakingFrame(), # First bot speech ends BotStartedSpeakingFrame(), # Second bot speech + VADUserStartedSpeakingFrame(), # Should pass through UserStartedSpeakingFrame(), # Should pass through InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through + VADUserStoppedSpeakingFrame(), # Should pass through UserStoppedSpeakingFrame(), # Should pass through BotStoppedSpeakingFrame(), ] @@ -47,8 +53,10 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): BotStoppedSpeakingFrame, STTMuteFrame, # mute=False BotStartedSpeakingFrame, + VADUserStartedSpeakingFrame, # Now passes through UserStartedSpeakingFrame, # Now passes through InputAudioRawFrame, # Now passes through + VADUserStoppedSpeakingFrame, # Now passes through UserStoppedSpeakingFrame, # Now passes through BotStoppedSpeakingFrame, ] @@ -64,20 +72,26 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): frames_to_send = [ BotStartedSpeakingFrame(), # First speech starts + VADUserStartedSpeakingFrame(), # Should be suppressed UserStartedSpeakingFrame(), # Should be suppressed InputAudioRawFrame( audio=b"", sample_rate=16000, num_channels=1 ), # Should be suppressed + VADUserStoppedSpeakingFrame(), # Should be suppressed UserStoppedSpeakingFrame(), # Should be suppressed BotStoppedSpeakingFrame(), # First speech ends + VADUserStartedSpeakingFrame(), # Should pass through UserStartedSpeakingFrame(), # Should pass through InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through + VADUserStoppedSpeakingFrame(), # Should pass through UserStoppedSpeakingFrame(), # Should pass through BotStartedSpeakingFrame(), # Second speech starts + VADUserStartedSpeakingFrame(), # Should be suppressed again UserStartedSpeakingFrame(), # Should be suppressed again InputAudioRawFrame( audio=b"", sample_rate=16000, num_channels=1 ), # Should be suppressed again + VADUserStoppedSpeakingFrame(), # Should be suppressed again UserStoppedSpeakingFrame(), # Should be suppressed again BotStoppedSpeakingFrame(), # Second speech ends ] @@ -87,8 +101,10 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): STTMuteFrame, # mute=True BotStoppedSpeakingFrame, STTMuteFrame, # mute=False + VADUserStartedSpeakingFrame, UserStartedSpeakingFrame, InputAudioRawFrame, + VADUserStoppedSpeakingFrame, UserStoppedSpeakingFrame, BotStartedSpeakingFrame, STTMuteFrame, # mute=True @@ -146,14 +162,18 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): # filter = STTMuteFilter(config=STTMuteConfig(strategies={STTMuteStrategy.FUNCTION_CALL})) # frames_to_send = [ + # VADUserStartedSpeakingFrame(), # Should pass through initially # UserStartedSpeakingFrame(), # Should pass through initially + # VADUserStoppedSpeakingFrame(), # UserStoppedSpeakingFrame(), # FunctionCallInProgressFrame( # function_name="get_weather", # tool_call_id="call_123", # arguments='{"location": "San Francisco"}', # ), # Start function call + # VADUserStartedSpeakingFrame(), # Should be suppressed # UserStartedSpeakingFrame(), # Should be suppressed + # VADUserStoppedSpeakingFrame(), # Should be suppressed # UserStoppedSpeakingFrame(), # Should be suppressed # FunctionCallResultFrame( # function_name="get_weather", @@ -161,18 +181,24 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): # arguments='{"location": "San Francisco"}', # result={"temperature": 22}, # ), # End function call + # VADUserStartedSpeakingFrame(), # Should pass through again # UserStartedSpeakingFrame(), # Should pass through again + # VADUserStoppedSpeakingFrame(), # UserStoppedSpeakingFrame(), # ] # expected_returned_frames = [ + # VADUserStartedSpeakingFrame, # UserStartedSpeakingFrame, + # VADUserStoppedSpeakingFrame, # UserStoppedSpeakingFrame, # FunctionCallInProgressFrame, # STTMuteFrame, # mute=True # FunctionCallResultFrame, # STTMuteFrame, # mute=False + # VADUserStartedSpeakingFrame, # UserStartedSpeakingFrame, + # VADUserStoppedSpeakingFrame, # UserStoppedSpeakingFrame, # ] @@ -188,24 +214,32 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): ) frames_to_send = [ + VADUserStartedSpeakingFrame(), # Should be suppressed (starts muted) UserStartedSpeakingFrame(), # Should be suppressed (starts muted) InputAudioRawFrame( audio=b"", sample_rate=16000, num_channels=1 ), # Should be suppressed + VADUserStoppedSpeakingFrame(), # Should be suppressed UserStoppedSpeakingFrame(), # Should be suppressed BotStartedSpeakingFrame(), # First bot speech + VADUserStartedSpeakingFrame(), # Should be suppressed UserStartedSpeakingFrame(), # Should be suppressed InputAudioRawFrame( audio=b"", sample_rate=16000, num_channels=1 ), # Should be suppressed + VADUserStoppedSpeakingFrame(), # Should be suppressed UserStoppedSpeakingFrame(), # Should be suppressed BotStoppedSpeakingFrame(), # First speech ends, unmutes + VADUserStartedSpeakingFrame(), # Should pass through UserStartedSpeakingFrame(), # Should pass through InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through + VADUserStoppedSpeakingFrame(), # Should pass through UserStoppedSpeakingFrame(), # Should pass through BotStartedSpeakingFrame(), # Second speech + VADUserStartedSpeakingFrame(), # Should pass through UserStartedSpeakingFrame(), # Should pass through InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through + VADUserStoppedSpeakingFrame(), # Should pass through UserStoppedSpeakingFrame(), # Should pass through BotStoppedSpeakingFrame(), ] @@ -215,12 +249,16 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): BotStartedSpeakingFrame, BotStoppedSpeakingFrame, STTMuteFrame, # mute=False after first speech + VADUserStartedSpeakingFrame, UserStartedSpeakingFrame, InputAudioRawFrame, + VADUserStoppedSpeakingFrame, UserStoppedSpeakingFrame, BotStartedSpeakingFrame, + VADUserStartedSpeakingFrame, UserStartedSpeakingFrame, InputAudioRawFrame, + VADUserStoppedSpeakingFrame, UserStoppedSpeakingFrame, BotStoppedSpeakingFrame, ] @@ -254,31 +292,41 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): ) frames_to_send = [ + VADUserStartedSpeakingFrame(), # Should pass through UserStartedSpeakingFrame(), # Should pass through InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through + VADUserStoppedSpeakingFrame(), # Should pass through UserStoppedSpeakingFrame(), # Should pass through BotStartedSpeakingFrame(), # Bot starts speaking + VADUserStartedSpeakingFrame(), # Should be suppressed UserStartedSpeakingFrame(), # Should be suppressed InputAudioRawFrame( audio=b"", sample_rate=16000, num_channels=1 ), # Should be suppressed + VADUserStoppedSpeakingFrame(), # Should be suppressed UserStoppedSpeakingFrame(), # Should be suppressed BotStoppedSpeakingFrame(), # Bot stops speaking + VADUserStartedSpeakingFrame(), # Should pass through UserStartedSpeakingFrame(), # Should pass through InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through + VADUserStoppedSpeakingFrame(), # Should pass through UserStoppedSpeakingFrame(), # Should pass through ] expected_returned_frames = [ + VADUserStartedSpeakingFrame, UserStartedSpeakingFrame, InputAudioRawFrame, + VADUserStoppedSpeakingFrame, UserStoppedSpeakingFrame, BotStartedSpeakingFrame, STTMuteFrame, # mute=True BotStoppedSpeakingFrame, STTMuteFrame, # mute=False + VADUserStartedSpeakingFrame, UserStartedSpeakingFrame, InputAudioRawFrame, + VADUserStoppedSpeakingFrame, UserStoppedSpeakingFrame, ]