wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2413 from captaincaius/fix-stt-mute-filter-vad-frames-20250810

Browse Source 
Add VADUserStartSpeakingFrame VADUserStopSpeakingFrame to STTMuteFilter (fix #2412)
This commit is contained in:
Mark Backman
2025-08-11 13:54:34 -07:00
committed by GitHub
parent 2c01c2b5b3 7b8aeef4cc
commit 998ac32627
3 changed files with 53 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Updated `pyproject.toml` to once again pin `numba` to `>=0.61.2` in order to - Updated `pyproject.toml` to once again pin `numba` to `>=0.61.2` in order to
resolve package versioning issues. resolve package versioning issues.
- Updated the `STTMuteFilter` to include `VADUserStartedSpeakingFrame` and `VADUserStoppedSpeakingFrame` in the list of frames to filter when the filtering is on.
### Other ### Other

4
src/pipecat/processors/filters/stt_mute_filter.py
View File

@@ -32,6 +32,8 @@ from pipecat.frames.frames import (
TranscriptionFrame, TranscriptionFrame,
UserStartedSpeakingFrame, UserStartedSpeakingFrame,
UserStoppedSpeakingFrame, UserStoppedSpeakingFrame,
VADUserStartedSpeakingFrame,
VADUserStoppedSpeakingFrame,
) )
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -205,6 +207,8 @@ class STTMuteFilter(FrameProcessor):
( (
StartInterruptionFrame, StartInterruptionFrame,
StopInterruptionFrame, StopInterruptionFrame,
VADUserStartedSpeakingFrame,
VADUserStoppedSpeakingFrame,
UserStartedSpeakingFrame, UserStartedSpeakingFrame,
UserStoppedSpeakingFrame, UserStoppedSpeakingFrame,
InputAudioRawFrame, InputAudioRawFrame,

48
tests/test_stt_mute_filter.py
View File

@@ -17,6 +17,8 @@ from pipecat.frames.frames import (
TranscriptionFrame, TranscriptionFrame,
UserStartedSpeakingFrame, UserStartedSpeakingFrame,
UserStoppedSpeakingFrame, UserStoppedSpeakingFrame,
VADUserStartedSpeakingFrame,
VADUserStoppedSpeakingFrame,
) )
from pipecat.processors.filters.stt_mute_filter import STTMuteConfig, STTMuteFilter, STTMuteStrategy from pipecat.processors.filters.stt_mute_filter import STTMuteConfig, STTMuteFilter, STTMuteStrategy
from pipecat.tests.utils import SleepFrame, run_test from pipecat.tests.utils import SleepFrame, run_test
@@ -28,15 +30,19 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase):
frames_to_send = [ frames_to_send = [
BotStartedSpeakingFrame(), # First bot speech starts BotStartedSpeakingFrame(), # First bot speech starts
VADUserStartedSpeakingFrame(), # Should be suppressed
UserStartedSpeakingFrame(), # Should be suppressed UserStartedSpeakingFrame(), # Should be suppressed
InputAudioRawFrame( InputAudioRawFrame(
audio=b"", sample_rate=16000, num_channels=1 audio=b"", sample_rate=16000, num_channels=1
), # Should be suppressed ), # Should be suppressed
VADUserStoppedSpeakingFrame(), # Should be suppressed
UserStoppedSpeakingFrame(), # Should be suppressed UserStoppedSpeakingFrame(), # Should be suppressed
BotStoppedSpeakingFrame(), # First bot speech ends BotStoppedSpeakingFrame(), # First bot speech ends
BotStartedSpeakingFrame(), # Second bot speech BotStartedSpeakingFrame(), # Second bot speech
VADUserStartedSpeakingFrame(), # Should pass through
UserStartedSpeakingFrame(), # Should pass through UserStartedSpeakingFrame(), # Should pass through
InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through
VADUserStoppedSpeakingFrame(), # Should pass through
UserStoppedSpeakingFrame(), # Should pass through UserStoppedSpeakingFrame(), # Should pass through
BotStoppedSpeakingFrame(), BotStoppedSpeakingFrame(),
] ]
@@ -47,8 +53,10 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase):
BotStoppedSpeakingFrame, BotStoppedSpeakingFrame,
STTMuteFrame, # mute=False STTMuteFrame, # mute=False
BotStartedSpeakingFrame, BotStartedSpeakingFrame,
VADUserStartedSpeakingFrame, # Now passes through
UserStartedSpeakingFrame, # Now passes through UserStartedSpeakingFrame, # Now passes through
InputAudioRawFrame, # Now passes through InputAudioRawFrame, # Now passes through
VADUserStoppedSpeakingFrame, # Now passes through
UserStoppedSpeakingFrame, # Now passes through UserStoppedSpeakingFrame, # Now passes through
BotStoppedSpeakingFrame, BotStoppedSpeakingFrame,
] ]
@@ -64,20 +72,26 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase):
frames_to_send = [ frames_to_send = [
BotStartedSpeakingFrame(), # First speech starts BotStartedSpeakingFrame(), # First speech starts
VADUserStartedSpeakingFrame(), # Should be suppressed
UserStartedSpeakingFrame(), # Should be suppressed UserStartedSpeakingFrame(), # Should be suppressed
InputAudioRawFrame( InputAudioRawFrame(
audio=b"", sample_rate=16000, num_channels=1 audio=b"", sample_rate=16000, num_channels=1
), # Should be suppressed ), # Should be suppressed
VADUserStoppedSpeakingFrame(), # Should be suppressed
UserStoppedSpeakingFrame(), # Should be suppressed UserStoppedSpeakingFrame(), # Should be suppressed
BotStoppedSpeakingFrame(), # First speech ends BotStoppedSpeakingFrame(), # First speech ends
VADUserStartedSpeakingFrame(), # Should pass through
UserStartedSpeakingFrame(), # Should pass through UserStartedSpeakingFrame(), # Should pass through
InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through
VADUserStoppedSpeakingFrame(), # Should pass through
UserStoppedSpeakingFrame(), # Should pass through UserStoppedSpeakingFrame(), # Should pass through
BotStartedSpeakingFrame(), # Second speech starts BotStartedSpeakingFrame(), # Second speech starts
VADUserStartedSpeakingFrame(), # Should be suppressed again
UserStartedSpeakingFrame(), # Should be suppressed again UserStartedSpeakingFrame(), # Should be suppressed again
InputAudioRawFrame( InputAudioRawFrame(
audio=b"", sample_rate=16000, num_channels=1 audio=b"", sample_rate=16000, num_channels=1
), # Should be suppressed again ), # Should be suppressed again
VADUserStoppedSpeakingFrame(), # Should be suppressed again
UserStoppedSpeakingFrame(), # Should be suppressed again UserStoppedSpeakingFrame(), # Should be suppressed again
BotStoppedSpeakingFrame(), # Second speech ends BotStoppedSpeakingFrame(), # Second speech ends
] ]
@@ -87,8 +101,10 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase):
STTMuteFrame, # mute=True STTMuteFrame, # mute=True
BotStoppedSpeakingFrame, BotStoppedSpeakingFrame,
STTMuteFrame, # mute=False STTMuteFrame, # mute=False
VADUserStartedSpeakingFrame,
UserStartedSpeakingFrame, UserStartedSpeakingFrame,
InputAudioRawFrame, InputAudioRawFrame,
VADUserStoppedSpeakingFrame,
UserStoppedSpeakingFrame, UserStoppedSpeakingFrame,
BotStartedSpeakingFrame, BotStartedSpeakingFrame,
STTMuteFrame, # mute=True STTMuteFrame, # mute=True
@@ -146,14 +162,18 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase):
# filter = STTMuteFilter(config=STTMuteConfig(strategies={STTMuteStrategy.FUNCTION_CALL})) # filter = STTMuteFilter(config=STTMuteConfig(strategies={STTMuteStrategy.FUNCTION_CALL}))
# frames_to_send = [ # frames_to_send = [
# VADUserStartedSpeakingFrame(), # Should pass through initially
# UserStartedSpeakingFrame(), # Should pass through initially # UserStartedSpeakingFrame(), # Should pass through initially
# VADUserStoppedSpeakingFrame(),
# UserStoppedSpeakingFrame(), # UserStoppedSpeakingFrame(),
# FunctionCallInProgressFrame( # FunctionCallInProgressFrame(
# function_name="get_weather", # function_name="get_weather",
# tool_call_id="call_123", # tool_call_id="call_123",
# arguments='{"location": "San Francisco"}', # arguments='{"location": "San Francisco"}',
# ), # Start function call # ), # Start function call
# VADUserStartedSpeakingFrame(), # Should be suppressed
# UserStartedSpeakingFrame(), # Should be suppressed # UserStartedSpeakingFrame(), # Should be suppressed
# VADUserStoppedSpeakingFrame(), # Should be suppressed
# UserStoppedSpeakingFrame(), # Should be suppressed # UserStoppedSpeakingFrame(), # Should be suppressed
# FunctionCallResultFrame( # FunctionCallResultFrame(
# function_name="get_weather", # function_name="get_weather",
@@ -161,18 +181,24 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase):
# arguments='{"location": "San Francisco"}', # arguments='{"location": "San Francisco"}',
# result={"temperature": 22}, # result={"temperature": 22},
# ), # End function call # ), # End function call
# VADUserStartedSpeakingFrame(), # Should pass through again
# UserStartedSpeakingFrame(), # Should pass through again # UserStartedSpeakingFrame(), # Should pass through again
# VADUserStoppedSpeakingFrame(),
# UserStoppedSpeakingFrame(), # UserStoppedSpeakingFrame(),
# ] # ]
# expected_returned_frames = [ # expected_returned_frames = [
# VADUserStartedSpeakingFrame,
# UserStartedSpeakingFrame, # UserStartedSpeakingFrame,
# VADUserStoppedSpeakingFrame,
# UserStoppedSpeakingFrame, # UserStoppedSpeakingFrame,
# FunctionCallInProgressFrame, # FunctionCallInProgressFrame,
# STTMuteFrame, # mute=True # STTMuteFrame, # mute=True
# FunctionCallResultFrame, # FunctionCallResultFrame,
# STTMuteFrame, # mute=False # STTMuteFrame, # mute=False
# VADUserStartedSpeakingFrame,
# UserStartedSpeakingFrame, # UserStartedSpeakingFrame,
# VADUserStoppedSpeakingFrame,
# UserStoppedSpeakingFrame, # UserStoppedSpeakingFrame,
# ] # ]
@@ -188,24 +214,32 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase):
) )
frames_to_send = [ frames_to_send = [
VADUserStartedSpeakingFrame(), # Should be suppressed (starts muted)
UserStartedSpeakingFrame(), # Should be suppressed (starts muted) UserStartedSpeakingFrame(), # Should be suppressed (starts muted)
InputAudioRawFrame( InputAudioRawFrame(
audio=b"", sample_rate=16000, num_channels=1 audio=b"", sample_rate=16000, num_channels=1
), # Should be suppressed ), # Should be suppressed
VADUserStoppedSpeakingFrame(), # Should be suppressed
UserStoppedSpeakingFrame(), # Should be suppressed UserStoppedSpeakingFrame(), # Should be suppressed
BotStartedSpeakingFrame(), # First bot speech BotStartedSpeakingFrame(), # First bot speech
VADUserStartedSpeakingFrame(), # Should be suppressed
UserStartedSpeakingFrame(), # Should be suppressed UserStartedSpeakingFrame(), # Should be suppressed
InputAudioRawFrame( InputAudioRawFrame(
audio=b"", sample_rate=16000, num_channels=1 audio=b"", sample_rate=16000, num_channels=1
), # Should be suppressed ), # Should be suppressed
VADUserStoppedSpeakingFrame(), # Should be suppressed
UserStoppedSpeakingFrame(), # Should be suppressed UserStoppedSpeakingFrame(), # Should be suppressed
BotStoppedSpeakingFrame(), # First speech ends, unmutes BotStoppedSpeakingFrame(), # First speech ends, unmutes
VADUserStartedSpeakingFrame(), # Should pass through
UserStartedSpeakingFrame(), # Should pass through UserStartedSpeakingFrame(), # Should pass through
InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through
VADUserStoppedSpeakingFrame(), # Should pass through
UserStoppedSpeakingFrame(), # Should pass through UserStoppedSpeakingFrame(), # Should pass through
BotStartedSpeakingFrame(), # Second speech BotStartedSpeakingFrame(), # Second speech
VADUserStartedSpeakingFrame(), # Should pass through
UserStartedSpeakingFrame(), # Should pass through UserStartedSpeakingFrame(), # Should pass through
InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through
VADUserStoppedSpeakingFrame(), # Should pass through
UserStoppedSpeakingFrame(), # Should pass through UserStoppedSpeakingFrame(), # Should pass through
BotStoppedSpeakingFrame(), BotStoppedSpeakingFrame(),
] ]
@@ -215,12 +249,16 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase):
BotStartedSpeakingFrame, BotStartedSpeakingFrame,
BotStoppedSpeakingFrame, BotStoppedSpeakingFrame,
STTMuteFrame, # mute=False after first speech STTMuteFrame, # mute=False after first speech
VADUserStartedSpeakingFrame,
UserStartedSpeakingFrame, UserStartedSpeakingFrame,
InputAudioRawFrame, InputAudioRawFrame,
VADUserStoppedSpeakingFrame,
UserStoppedSpeakingFrame, UserStoppedSpeakingFrame,
BotStartedSpeakingFrame, BotStartedSpeakingFrame,
VADUserStartedSpeakingFrame,
UserStartedSpeakingFrame, UserStartedSpeakingFrame,
InputAudioRawFrame, InputAudioRawFrame,
VADUserStoppedSpeakingFrame,
UserStoppedSpeakingFrame, UserStoppedSpeakingFrame,
BotStoppedSpeakingFrame, BotStoppedSpeakingFrame,
] ]
@@ -254,31 +292,41 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase):
) )
frames_to_send = [ frames_to_send = [
VADUserStartedSpeakingFrame(), # Should pass through
UserStartedSpeakingFrame(), # Should pass through UserStartedSpeakingFrame(), # Should pass through
InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through
VADUserStoppedSpeakingFrame(), # Should pass through
UserStoppedSpeakingFrame(), # Should pass through UserStoppedSpeakingFrame(), # Should pass through
BotStartedSpeakingFrame(), # Bot starts speaking BotStartedSpeakingFrame(), # Bot starts speaking
VADUserStartedSpeakingFrame(), # Should be suppressed
UserStartedSpeakingFrame(), # Should be suppressed UserStartedSpeakingFrame(), # Should be suppressed
InputAudioRawFrame( InputAudioRawFrame(
audio=b"", sample_rate=16000, num_channels=1 audio=b"", sample_rate=16000, num_channels=1
), # Should be suppressed ), # Should be suppressed
VADUserStoppedSpeakingFrame(), # Should be suppressed
UserStoppedSpeakingFrame(), # Should be suppressed UserStoppedSpeakingFrame(), # Should be suppressed
BotStoppedSpeakingFrame(), # Bot stops speaking BotStoppedSpeakingFrame(), # Bot stops speaking
VADUserStartedSpeakingFrame(), # Should pass through
UserStartedSpeakingFrame(), # Should pass through UserStartedSpeakingFrame(), # Should pass through
InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through
VADUserStoppedSpeakingFrame(), # Should pass through
UserStoppedSpeakingFrame(), # Should pass through UserStoppedSpeakingFrame(), # Should pass through
] ]
expected_returned_frames = [ expected_returned_frames = [
VADUserStartedSpeakingFrame,
UserStartedSpeakingFrame, UserStartedSpeakingFrame,
InputAudioRawFrame, InputAudioRawFrame,
VADUserStoppedSpeakingFrame,
UserStoppedSpeakingFrame, UserStoppedSpeakingFrame,
BotStartedSpeakingFrame, BotStartedSpeakingFrame,
STTMuteFrame, # mute=True STTMuteFrame, # mute=True
BotStoppedSpeakingFrame, BotStoppedSpeakingFrame,
STTMuteFrame, # mute=False STTMuteFrame, # mute=False
VADUserStartedSpeakingFrame,
UserStartedSpeakingFrame, UserStartedSpeakingFrame,
InputAudioRawFrame, InputAudioRawFrame,
VADUserStoppedSpeakingFrame,
UserStoppedSpeakingFrame, UserStoppedSpeakingFrame,
] ]