diff --git a/changelog/3292.fixed.md b/changelog/3292.fixed.md new file mode 100644 index 000000000..4d3df66b0 --- /dev/null +++ b/changelog/3292.fixed.md @@ -0,0 +1 @@ +- Fixed a bug in `STTMuteFilter` where the user was not always muted during function calls, especially when there were multiple simultaneous calls. diff --git a/src/pipecat/processors/filters/stt_mute_filter.py b/src/pipecat/processors/filters/stt_mute_filter.py index 0b822b54a..8a0dca17d 100644 --- a/src/pipecat/processors/filters/stt_mute_filter.py +++ b/src/pipecat/processors/filters/stt_mute_filter.py @@ -21,8 +21,9 @@ from pipecat.frames.frames import ( BotStartedSpeakingFrame, BotStoppedSpeakingFrame, Frame, - FunctionCallInProgressFrame, + FunctionCallCancelFrame, FunctionCallResultFrame, + FunctionCallsStartedFrame, InputAudioRawFrame, InterimTranscriptionFrame, InterruptionFrame, @@ -116,7 +117,7 @@ class STTMuteFilter(FrameProcessor): self._config = config self._first_speech_handled = False self._bot_is_speaking = False - self._function_call_in_progress = False + self._function_call_in_progress = set() self._is_muted = False async def _handle_mute_state(self, should_mute: bool): @@ -176,11 +177,12 @@ class STTMuteFilter(FrameProcessor): # Process frames to determine mute state if isinstance(frame, StartFrame): should_mute = await self._should_mute() - elif isinstance(frame, FunctionCallInProgressFrame): - self._function_call_in_progress = True + elif isinstance(frame, FunctionCallsStartedFrame): + for f in frame.function_calls: + self._function_call_in_progress.add(f.tool_call_id) should_mute = await self._should_mute() - elif isinstance(frame, FunctionCallResultFrame): - self._function_call_in_progress = False + elif isinstance(frame, (FunctionCallCancelFrame, FunctionCallResultFrame)): + self._function_call_in_progress.remove(frame.tool_call_id) should_mute = await self._should_mute() elif isinstance(frame, BotStartedSpeakingFrame): self._bot_is_speaking = True diff --git a/tests/test_stt_mute_filter.py b/tests/test_stt_mute_filter.py index 2bd041b65..c36693259 100644 --- a/tests/test_stt_mute_filter.py +++ b/tests/test_stt_mute_filter.py @@ -9,8 +9,10 @@ import unittest from pipecat.frames.frames import ( BotStartedSpeakingFrame, BotStoppedSpeakingFrame, + FunctionCallFromLLM, FunctionCallInProgressFrame, FunctionCallResultFrame, + FunctionCallsStartedFrame, InputAudioRawFrame, InterimTranscriptionFrame, TranscriptionFrame, @@ -148,54 +150,59 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): expected_down_frames=expected_returned_frames, ) - # TODO: Revisit once we figure out how to test SystemFrames and DataFrames - # async def test_function_call_strategy(self): - # filter = STTMuteFilter(config=STTMuteConfig(strategies={STTMuteStrategy.FUNCTION_CALL})) + async def test_function_call_strategy(self): + filter = STTMuteFilter(config=STTMuteConfig(strategies={STTMuteStrategy.FUNCTION_CALL})) - # frames_to_send = [ - # VADUserStartedSpeakingFrame(), # Should pass through initially - # UserStartedSpeakingFrame(), # Should pass through initially - # VADUserStoppedSpeakingFrame(), - # UserStoppedSpeakingFrame(), - # FunctionCallInProgressFrame( - # function_name="get_weather", - # tool_call_id="call_123", - # arguments='{"location": "San Francisco"}', - # ), # Start function call - # VADUserStartedSpeakingFrame(), # Should be suppressed - # UserStartedSpeakingFrame(), # Should be suppressed - # VADUserStoppedSpeakingFrame(), # Should be suppressed - # UserStoppedSpeakingFrame(), # Should be suppressed - # FunctionCallResultFrame( - # function_name="get_weather", - # tool_call_id="call_123", - # arguments='{"location": "San Francisco"}', - # result={"temperature": 22}, - # ), # End function call - # VADUserStartedSpeakingFrame(), # Should pass through again - # UserStartedSpeakingFrame(), # Should pass through again - # VADUserStoppedSpeakingFrame(), - # UserStoppedSpeakingFrame(), - # ] + frames_to_send = [ + VADUserStartedSpeakingFrame(), # Should pass through initially + UserStartedSpeakingFrame(), # Should pass through initially + VADUserStoppedSpeakingFrame(), # Should pass through initially + UserStoppedSpeakingFrame(), # Should pass through initially + FunctionCallsStartedFrame( + function_calls=[ + FunctionCallFromLLM( + function_name="get_weather", + tool_call_id="call_123", + arguments='{"location": "San Francisco"}', + context=None, + ) + ] + ), # Start function call + VADUserStartedSpeakingFrame(), # Should be suppressed + UserStartedSpeakingFrame(), # Should be suppressed + VADUserStoppedSpeakingFrame(), # Should be suppressed + UserStoppedSpeakingFrame(), # Should be suppressed + FunctionCallResultFrame( + function_name="get_weather", + tool_call_id="call_123", + arguments='{"location": "San Francisco"}', + result={"temperature": 22}, + ), # End function call + SleepFrame(), + VADUserStartedSpeakingFrame(), # Should pass through again + UserStartedSpeakingFrame(), # Should pass through again + VADUserStoppedSpeakingFrame(), + UserStoppedSpeakingFrame(), + ] - # expected_returned_frames = [ - # VADUserStartedSpeakingFrame, - # UserStartedSpeakingFrame, - # VADUserStoppedSpeakingFrame, - # UserStoppedSpeakingFrame, - # FunctionCallInProgressFrame, - # FunctionCallResultFrame, - # VADUserStartedSpeakingFrame, - # UserStartedSpeakingFrame, - # VADUserStoppedSpeakingFrame, - # UserStoppedSpeakingFrame, - # ] + expected_returned_frames = [ + VADUserStartedSpeakingFrame, + UserStartedSpeakingFrame, + VADUserStoppedSpeakingFrame, + UserStoppedSpeakingFrame, + FunctionCallsStartedFrame, + FunctionCallResultFrame, + VADUserStartedSpeakingFrame, + UserStartedSpeakingFrame, + VADUserStoppedSpeakingFrame, + UserStoppedSpeakingFrame, + ] - # await run_test( - # filter, - # frames_to_send=frames_to_send, - # expected_down_frames=expected_returned_frames, - # ) + await run_test( + filter, + frames_to_send=frames_to_send, + expected_down_frames=expected_returned_frames, + ) async def test_mute_until_first_bot_complete_strategy(self): filter = STTMuteFilter(