From 29e09b2053f9ae2bf01307e9f873501b7c628a7c Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 8 Aug 2025 01:27:42 -0400 Subject: [PATCH 01/22] POC demo in progress --- examples/foundational/voicemail_test.py | 265 ++++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 examples/foundational/voicemail_test.py diff --git a/examples/foundational/voicemail_test.py b/examples/foundational/voicemail_test.py new file mode 100644 index 000000000..f45172a54 --- /dev/null +++ b/examples/foundational/voicemail_test.py @@ -0,0 +1,265 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +from typing import Optional + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import ( + BotInterruptionFrame, + CancelFrame, + EndFrame, + Frame, + LLMTextFrame, + StartFrame, + TTSSpeakFrame, +) +from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.llm_service import LLMService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.sync.base_notifier import BaseNotifier +from pipecat.sync.event_notifier import EventNotifier +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams + +load_dotenv(override=True) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +class VoicemailDetector(ParallelPipeline): + def __init__(self, llm: LLMService): + # Initialize LLM + self._classifier_llm = llm + self._messages = [ + { + "role": "system", + "content": """You are a voicemail detection classifier. Your job is to determine if the caller is leaving a voicemail message or trying to have a live conversation. + +VOICEMAIL INDICATORS (respond "YES"): +- One-way communication (caller talks without expecting immediate responses) +- Messages like "Hi, this is [name], please call me back" +- "I'm calling about..." followed by details without pausing for response +- "Leave me a message" or "call me when you get this" +- Monologue-style speech patterns +- Mentions of time/date when they're calling +- Business-like messages with contact information + +CONVERSATION INDICATORS (respond "NO"): +- Interactive speech ("Hello?", "Are you there?", "Can you hear me?") +- Questions directed at the recipient expecting immediate answers +- Responses to prompts or questions +- Back-and-forth dialogue patterns +- Greetings expecting responses ("Hi, how are you?") +- Real-time problem solving or discussion + +Respond with ONLY "YES" if it's a voicemail, or "NO" if it's a conversation attempt. Do not explain your reasoning.""", + }, + ] + self._context = OpenAILLMContext(self._messages) + self._context_aggregator = llm.create_context_aggregator(self._context) + self._conversation_notifier = EventNotifier() + self._classifier_gate = self.ClassifierGate(self._conversation_notifier) + self._voicemail_processor = self.VoicemailProcessor(self._conversation_notifier) + self._passthrough_processor = self.PassThroughProcessor() + + super().__init__( + # Conversation branch + [self._passthrough_processor], + # Classifer branch + [ + self._classifier_gate, + self._context_aggregator.user(), + self._classifier_llm, + self._voicemail_processor, + self._context_aggregator.assistant(), + ], + ) + + class ClassifierGate(FrameProcessor): + def __init__(self, notifier: BaseNotifier): + super().__init__() + self._notifier = notifier + self._gate_opened = True + self._gate_task: Optional[asyncio.Task] = None + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, StartFrame): + # Start the task immediately, don't wait for other conditions + self._gate_task = self.create_task(self._wait_for_notification()) + logger.info(f"{self}: Gate task started, waiting for notification") + + elif isinstance(frame, (EndFrame, CancelFrame)): + if self._gate_task: + await self.cancel_task(self._gate_task) + self._gate_task = None + + if self._gate_opened: + await self.push_frame(frame, direction) + elif not self._gate_opened and isinstance(frame, BotInterruptionFrame): + await self.push_frame(frame, direction) + + async def _wait_for_notification(self): + try: + logger.info(f"{self}: Waiting for notification...") + await self._notifier.wait() + logger.info(f"{self}: Received notification!") + + if self._gate_opened: + self._gate_opened = False + logger.info(f"{self}: Gate closed") + except asyncio.CancelledError: + logger.debug(f"{self}: Gate task was cancelled") + raise + except Exception as e: + logger.exception(f"{self}: Error in gate task: {e}") + raise + + class VoicemailProcessor(FrameProcessor): + def __init__(self, notifier: BaseNotifier): + super().__init__() + self._notifier = notifier + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, LLMTextFrame): + # Check if the frame is a NO response, notify the notifier + response = frame.text.strip().upper() + print(f"Response from LLM: {response}") + if "NO" in response: + logger.info(f"{self}: User conversation, notifying to close gate") + await self._notifier.notify() + elif "YES" in response: + logger.info(f"{self}: User is leaving a voicemail, push BotInterruptionFrame") + # If the user is leaving a voicemail, we push a BotInterruptionFrame + await self._notifier.notify() + await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM) + # How do we know when to send this?! + await asyncio.sleep(3) + await self.push_frame(TTSSpeakFrame("This is Mark. Call me back later.")) + + else: + # Push the frame + await self.push_frame(frame, direction) + + class PassThroughProcessor(FrameProcessor): + def __init__(self): + super().__init__() + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + await self.push_frame(frame, direction) + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + voicemail_detector_llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + voicemail_detector = VoicemailDetector(voicemail_detector_llm) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + voicemail_detector, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # # Kick off the conversation. + # messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + # await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() From 90ae85bab23f8f935a4d78f5a5e3f54279ac1ebf Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 8 Aug 2025 10:28:44 -0400 Subject: [PATCH 02/22] =?UTF-8?q?More=20updates=E2=80=94added=20new=20voic?= =?UTF-8?q?email=20module?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/foundational/voicemail_test.py | 166 +++---------- src/pipecat/utils/voicemail/__init__.py | 0 .../utils/voicemail/voicemail_detector.py | 224 ++++++++++++++++++ 3 files changed, 253 insertions(+), 137 deletions(-) create mode 100644 src/pipecat/utils/voicemail/__init__.py create mode 100644 src/pipecat/utils/voicemail/voicemail_detector.py diff --git a/examples/foundational/voicemail_test.py b/examples/foundational/voicemail_test.py index f45172a54..728e46676 100644 --- a/examples/foundational/voicemail_test.py +++ b/examples/foundational/voicemail_test.py @@ -6,38 +6,27 @@ import asyncio import os -from typing import Optional from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import ( - BotInterruptionFrame, - CancelFrame, - EndFrame, - Frame, - LLMTextFrame, - StartFrame, - TTSSpeakFrame, -) -from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.frames.frames import EndFrame, EndTaskFrame, TTSSpeakFrame +from pipecat.observers.loggers.debug_log_observer import DebugLogObserver from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.processors.frame_processor import FrameDirection from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.llm_service import LLMService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.sync.base_notifier import BaseNotifier -from pipecat.sync.event_notifier import EventNotifier from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams from pipecat.transports.services.daily import DailyParams +from pipecat.utils.voicemail.voicemail_detector import VoicemailDetector load_dotenv(override=True) @@ -63,130 +52,23 @@ transport_params = { } -class VoicemailDetector(ParallelPipeline): - def __init__(self, llm: LLMService): - # Initialize LLM - self._classifier_llm = llm - self._messages = [ - { - "role": "system", - "content": """You are a voicemail detection classifier. Your job is to determine if the caller is leaving a voicemail message or trying to have a live conversation. +async def handle_voicemail(processor): + """Called when a voicemail is detected. -VOICEMAIL INDICATORS (respond "YES"): -- One-way communication (caller talks without expecting immediate responses) -- Messages like "Hi, this is [name], please call me back" -- "I'm calling about..." followed by details without pausing for response -- "Leave me a message" or "call me when you get this" -- Monologue-style speech patterns -- Mentions of time/date when they're calling -- Business-like messages with contact information + Args: + processor: The VoicemailProcessor instance. processor.push_frame() is + available to push frames. + """ + logger.info("Voicemail detected! Playing greeting...") -CONVERSATION INDICATORS (respond "NO"): -- Interactive speech ("Hello?", "Are you there?", "Can you hear me?") -- Questions directed at the recipient expecting immediate answers -- Responses to prompts or questions -- Back-and-forth dialogue patterns -- Greetings expecting responses ("Hi, how are you?") -- Real-time problem solving or discussion + # Wait a moment for interruption to clear + await asyncio.sleep(1) -Respond with ONLY "YES" if it's a voicemail, or "NO" if it's a conversation attempt. Do not explain your reasoning.""", - }, - ] - self._context = OpenAILLMContext(self._messages) - self._context_aggregator = llm.create_context_aggregator(self._context) - self._conversation_notifier = EventNotifier() - self._classifier_gate = self.ClassifierGate(self._conversation_notifier) - self._voicemail_processor = self.VoicemailProcessor(self._conversation_notifier) - self._passthrough_processor = self.PassThroughProcessor() - - super().__init__( - # Conversation branch - [self._passthrough_processor], - # Classifer branch - [ - self._classifier_gate, - self._context_aggregator.user(), - self._classifier_llm, - self._voicemail_processor, - self._context_aggregator.assistant(), - ], - ) - - class ClassifierGate(FrameProcessor): - def __init__(self, notifier: BaseNotifier): - super().__init__() - self._notifier = notifier - self._gate_opened = True - self._gate_task: Optional[asyncio.Task] = None - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - if isinstance(frame, StartFrame): - # Start the task immediately, don't wait for other conditions - self._gate_task = self.create_task(self._wait_for_notification()) - logger.info(f"{self}: Gate task started, waiting for notification") - - elif isinstance(frame, (EndFrame, CancelFrame)): - if self._gate_task: - await self.cancel_task(self._gate_task) - self._gate_task = None - - if self._gate_opened: - await self.push_frame(frame, direction) - elif not self._gate_opened and isinstance(frame, BotInterruptionFrame): - await self.push_frame(frame, direction) - - async def _wait_for_notification(self): - try: - logger.info(f"{self}: Waiting for notification...") - await self._notifier.wait() - logger.info(f"{self}: Received notification!") - - if self._gate_opened: - self._gate_opened = False - logger.info(f"{self}: Gate closed") - except asyncio.CancelledError: - logger.debug(f"{self}: Gate task was cancelled") - raise - except Exception as e: - logger.exception(f"{self}: Error in gate task: {e}") - raise - - class VoicemailProcessor(FrameProcessor): - def __init__(self, notifier: BaseNotifier): - super().__init__() - self._notifier = notifier - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, LLMTextFrame): - # Check if the frame is a NO response, notify the notifier - response = frame.text.strip().upper() - print(f"Response from LLM: {response}") - if "NO" in response: - logger.info(f"{self}: User conversation, notifying to close gate") - await self._notifier.notify() - elif "YES" in response: - logger.info(f"{self}: User is leaving a voicemail, push BotInterruptionFrame") - # If the user is leaving a voicemail, we push a BotInterruptionFrame - await self._notifier.notify() - await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM) - # How do we know when to send this?! - await asyncio.sleep(3) - await self.push_frame(TTSSpeakFrame("This is Mark. Call me back later.")) - - else: - # Push the frame - await self.push_frame(frame, direction) - - class PassThroughProcessor(FrameProcessor): - def __init__(self): - super().__init__() - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - await self.push_frame(frame, direction) + # Push frames using standard Pipecat pattern + await processor.push_frame( + TTSSpeakFrame("This is Mattie. Call me back when you can!"), + ) + await processor.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM) async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): @@ -202,7 +84,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) voicemail_detector_llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - voicemail_detector = VoicemailDetector(voicemail_detector_llm) + voicemail_detector = VoicemailDetector( + llm=voicemail_detector_llm, on_voicemail_detected=handle_voicemail + ) messages = [ { @@ -234,6 +118,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): enable_usage_metrics=True, ), idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + observers=[ + DebugLogObserver( + frame_types={ + EndFrame: None, + EndTaskFrame: None, + } + ), + ], ) @transport.event_handler("on_client_connected") diff --git a/src/pipecat/utils/voicemail/__init__.py b/src/pipecat/utils/voicemail/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/utils/voicemail/voicemail_detector.py b/src/pipecat/utils/voicemail/voicemail_detector.py new file mode 100644 index 000000000..a217345a1 --- /dev/null +++ b/src/pipecat/utils/voicemail/voicemail_detector.py @@ -0,0 +1,224 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Voicemail detection module for Pipecat. + +This module provides voicemail detection capabilities using parallel pipeline +processing to classify incoming calls as either voicemail messages or live +conversations. +""" + +import asyncio +from typing import Awaitable, Callable, Optional + +from loguru import logger + +from pipecat.frames.frames import ( + BotInterruptionFrame, + CancelFrame, + CancelTaskFrame, + EndFrame, + EndTaskFrame, + Frame, + LLMTextFrame, + StartFrame, +) +from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.services.llm_service import LLMService +from pipecat.sync.base_notifier import BaseNotifier +from pipecat.sync.event_notifier import EventNotifier + + +class ClassifierGate(FrameProcessor): + """Gate processor that controls frame flow based on classification decisions. + + The gate starts open and closes permanently once a classification decision + is made (YES or NO). This ensures the classifier only runs until a definitive + decision is reached. + """ + + def __init__(self, notifier: BaseNotifier): + """Initialize the classifier gate. + + Args: + notifier: Notifier that signals when to close the gate. + """ + super().__init__() + self._notifier = notifier + self._gate_opened = True + self._gate_task: Optional[asyncio.Task] = None + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames and control gate state. + + Args: + frame: The frame to process. + direction: The direction of frame flow. + """ + await super().process_frame(frame, direction) + + if isinstance(frame, StartFrame): + # Start the task immediately, don't wait for other conditions + self._gate_task = self.create_task(self._wait_for_notification()) + logger.info(f"{self}: Gate task started, waiting for notification") + + elif isinstance(frame, (EndFrame, CancelFrame)): + if self._gate_task: + await self.cancel_task(self._gate_task) + self._gate_task = None + + if self._gate_opened: + await self.push_frame(frame, direction) + elif not self._gate_opened and isinstance( + frame, (BotInterruptionFrame, EndTaskFrame, EndFrame, CancelTaskFrame, CancelFrame) + ): + await self.push_frame(frame, direction) + + async def _wait_for_notification(self): + """Wait for classification decision notification.""" + try: + logger.info(f"{self}: Waiting for notification...") + await self._notifier.wait() + logger.info(f"{self}: Received notification!") + + if self._gate_opened: + self._gate_opened = False + logger.info(f"{self}: Gate closed") + except asyncio.CancelledError: + logger.debug(f"{self}: Gate task was cancelled") + raise + except Exception as e: + logger.exception(f"{self}: Error in gate task: {e}") + raise + + +class VoicemailProcessor(FrameProcessor): + """Processor that handles LLM classification responses and triggers callbacks. + + Processes LLM text responses to determine if the call is a voicemail (YES) + or conversation (NO), then triggers appropriate actions including developer + callbacks for voicemail detection. + """ + + def __init__( + self, + notifier: BaseNotifier, + on_voicemail_detected: Optional[Callable[["VoicemailProcessor"], Awaitable[None]]] = None, + ): + """Initialize the voicemail processor. + + Args: + notifier: Notifier to signal classification decisions. + on_voicemail_detected: Callback function called when voicemail is detected. + """ + super().__init__() + self._notifier = notifier + self._on_voicemail_detected = on_voicemail_detected + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames and handle LLM classification responses. + + Args: + frame: The frame to process. + direction: The direction of frame flow. + """ + await super().process_frame(frame, direction) + + if isinstance(frame, LLMTextFrame): + response = frame.text.strip().upper() + if "NO" in response: + logger.info(f"{self}: CONVERSATION detected - notifying to close gate") + await self._notifier.notify() + elif "YES" in response: + logger.info(f"{self}: VOICEMAIL detected - triggering callback") + # Notify gate to close (decision is final) + await self._notifier.notify() + # Push BotInterruptionFrame to clear the pipeline + await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM) + # Call developer callback if provided + if self._on_voicemail_detected: + try: + await self._on_voicemail_detected(self) + except Exception as e: + logger.exception(f"{self}: Error in voicemail callback: {e}") + + else: + # Push the frame + await self.push_frame(frame, direction) + + +class VoicemailDetector(ParallelPipeline): + """Parallel pipeline for detecting voicemail vs. live conversation. + + Uses a parallel pipeline architecture with two branches: + 1. Conversation branch: Normal frame flow for live conversations + 2. Classification branch: LLM-based classification that can interrupt for voicemail + + The classifier runs in parallel and makes a one-time decision to either: + - Continue normal conversation flow (NO response) + - Interrupt and trigger voicemail handling (YES response) + """ + + def __init__( + self, + *, + llm: LLMService, + on_voicemail_detected: Optional[Callable[[], Awaitable[None]]] = None, + ): + """Initialize the voicemail detector. + + Args: + llm: LLM service for classification. + on_voicemail_detected: Callback function called when voicemail is detected. + """ + self._classifier_llm = llm + self._messages = [ + { + "role": "system", + "content": """You are a voicemail detection classifier. Your job is to determine if the caller is leaving a voicemail message or trying to have a live conversation. + +VOICEMAIL INDICATORS (respond "YES"): +- One-way communication (caller talks without expecting immediate responses) +- Messages like "Hi, this is [name], please call me back" +- "I'm calling about..." followed by details without pausing for response +- "Leave me a message" or "call me when you get this" +- Monologue-style speech patterns +- Mentions of time/date when they're calling +- Business-like messages with contact information + +CONVERSATION INDICATORS (respond "NO"): +- Interactive speech ("Hello?", "Are you there?", "Can you hear me?") +- Questions directed at the recipient expecting immediate answers +- Responses to prompts or questions +- Back-and-forth dialogue patterns +- Greetings expecting responses ("Hi, how are you?") +- Real-time problem solving or discussion + +Respond with ONLY "YES" if it's a voicemail, or "NO" if it's a conversation attempt. Do not explain your reasoning.""", + }, + ] + self._context = OpenAILLMContext(self._messages) + self._context_aggregator = llm.create_context_aggregator(self._context) + self._conversation_notifier = EventNotifier() + self._classifier_gate = ClassifierGate(self._conversation_notifier) + self._voicemail_processor = VoicemailProcessor( + self._conversation_notifier, on_voicemail_detected + ) + + super().__init__( + # Conversation branch + [], + # Classifer branch + [ + self._classifier_gate, + self._context_aggregator.user(), + self._classifier_llm, + self._voicemail_processor, + self._context_aggregator.assistant(), + ], + ) From 238d6bf9ab63070ebe1560d795e6a58adf0db38a Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 8 Aug 2025 11:33:18 -0400 Subject: [PATCH 03/22] Add buffering logic --- examples/foundational/voicemail_test.py | 9 +- .../utils/voicemail/voicemail_detector.py | 142 ++++++++++++++++-- 2 files changed, 137 insertions(+), 14 deletions(-) diff --git a/examples/foundational/voicemail_test.py b/examples/foundational/voicemail_test.py index 728e46676..04ade85ca 100644 --- a/examples/foundational/voicemail_test.py +++ b/examples/foundational/voicemail_test.py @@ -82,11 +82,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - voicemail_detector_llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + voicemail_llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - voicemail_detector = VoicemailDetector( - llm=voicemail_detector_llm, on_voicemail_detected=handle_voicemail - ) + voicemail = VoicemailDetector(llm=voicemail_llm, on_voicemail_detected=handle_voicemail) messages = [ { @@ -102,10 +100,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): [ transport.input(), # Transport user input stt, - voicemail_detector, + voicemail.detector(), context_aggregator.user(), # User responses llm, # LLM tts, # TTS + voicemail.buffer(), transport.output(), # Transport bot output context_aggregator.assistant(), # Assistant spoken responses ] diff --git a/src/pipecat/utils/voicemail/voicemail_detector.py b/src/pipecat/utils/voicemail/voicemail_detector.py index a217345a1..6473b3675 100644 --- a/src/pipecat/utils/voicemail/voicemail_detector.py +++ b/src/pipecat/utils/voicemail/voicemail_detector.py @@ -12,7 +12,7 @@ conversations. """ import asyncio -from typing import Awaitable, Callable, Optional +from typing import Awaitable, Callable, List, Optional from loguru import logger @@ -25,6 +25,8 @@ from pipecat.frames.frames import ( Frame, LLMTextFrame, StartFrame, + TTSAudioRawFrame, + TTSTextFrame, ) from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext @@ -107,17 +109,24 @@ class VoicemailProcessor(FrameProcessor): def __init__( self, - notifier: BaseNotifier, + *, + gate_notifier: BaseNotifier, + conversation_notifier: BaseNotifier, # Buffer should release frames + voicemail_notifier: BaseNotifier, # Buffer should clear frames on_voicemail_detected: Optional[Callable[["VoicemailProcessor"], Awaitable[None]]] = None, ): """Initialize the voicemail processor. Args: - notifier: Notifier to signal classification decisions. + gate_notifier: Notifier to signal gate about classification decisions. + conversation_notifier: Notifier to signal buffer to release frames. + voicemail_notifier: Notifier to signal buffer to clear frames. on_voicemail_detected: Callback function called when voicemail is detected. """ super().__init__() - self._notifier = notifier + self._gate_notifier = gate_notifier + self._conversation_notifier = conversation_notifier + self._voicemail_notifier = voicemail_notifier self._on_voicemail_detected = on_voicemail_detected async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -133,11 +142,13 @@ class VoicemailProcessor(FrameProcessor): response = frame.text.strip().upper() if "NO" in response: logger.info(f"{self}: CONVERSATION detected - notifying to close gate") - await self._notifier.notify() + await self._gate_notifier.notify() + await self._conversation_notifier.notify() elif "YES" in response: logger.info(f"{self}: VOICEMAIL detected - triggering callback") # Notify gate to close (decision is final) - await self._notifier.notify() + await self._gate_notifier.notify() + await self._voicemail_notifier.notify() # Push BotInterruptionFrame to clear the pipeline await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM) # Call developer callback if provided @@ -152,6 +163,95 @@ class VoicemailProcessor(FrameProcessor): await self.push_frame(frame, direction) +class VoicemailBuffer(FrameProcessor): + """Buffers TTS frames until voicemail classification decision is made. + + Holds TTS frames in a buffer while voicemail classification is in progress. + Releases all buffered frames when conversation is detected, or keeps them + buffered when voicemail is detected. + """ + + def __init__(self, conversation_notifier: BaseNotifier, voicemail_notifier: BaseNotifier): + """Initialize the voicemail buffer. + + Args: + conversation_notifier: Notifier that signals when to release buffered frames. + voicemail_notifier: Notifier that signals when to keep buffered frames. + """ + super().__init__() + self._conversation_notifier = conversation_notifier + self._voicemail_notifier = voicemail_notifier + self._frame_buffer: List[tuple[Frame, FrameDirection]] = [] + self._buffering_active = True + self._conversation_task: Optional[asyncio.Task] = None + self._voicemail_task: Optional[asyncio.Task] = None + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames and handle buffering logic. + + Args: + frame: The frame to process. + direction: The direction of frame flow. + """ + await super().process_frame(frame, direction) + + if isinstance(frame, StartFrame): + self._conversation_task = self.create_task(self._wait_for_conversation()) + self._voicemail_task = self.create_task(self._wait_for_voicemail()) + logger.info(f"{self}: Buffer tasks started") + await self.push_frame(frame, direction) + elif isinstance(frame, (EndFrame, CancelFrame)): + if self._conversation_task: + await self.cancel_task(self._conversation_task) + self._conversation_task = None + if self._voicemail_task: + await self.cancel_task(self._voicemail_task) + self._voicemail_task = None + await self.push_frame(frame, direction) + + # Buffer TTS frames while buffering is active + if self._buffering_active and isinstance(frame, (TTSTextFrame, TTSAudioRawFrame)): + self._frame_buffer.append((frame, direction)) + else: + await self.push_frame(frame, direction) + + async def _wait_for_conversation(self): + """Wait for conversation detection - release buffered frames.""" + try: + await self._conversation_notifier.wait() + logger.info(f"{self}: CONVERSATION - releasing frames") + + self._buffering_active = False + for frame, direction in self._frame_buffer: + await self.push_frame(frame, direction) + self._frame_buffer.clear() + + # Cancel the other task + if self._voicemail_task: + await self.cancel_task(self._voicemail_task) + self._voicemail_task = None + + except asyncio.CancelledError: + raise + + async def _wait_for_voicemail(self): + """Wait for voicemail detection - clear buffered frames.""" + try: + await self._voicemail_notifier.wait() + logger.info(f"{self}: VOICEMAIL - clearing frames") + + self._buffering_active = False + self._frame_buffer.clear() + + # Cancel the other task + if self._conversation_task: + await self.cancel_task(self._conversation_task) + self._conversation_task = None + + except asyncio.CancelledError: + raise + + class VoicemailDetector(ParallelPipeline): """Parallel pipeline for detecting voicemail vs. live conversation. @@ -204,10 +304,18 @@ Respond with ONLY "YES" if it's a voicemail, or "NO" if it's a conversation atte ] self._context = OpenAILLMContext(self._messages) self._context_aggregator = llm.create_context_aggregator(self._context) - self._conversation_notifier = EventNotifier() - self._classifier_gate = ClassifierGate(self._conversation_notifier) + self._gate_notifier = EventNotifier() + self._conversation_notifier = EventNotifier() # For releasing buffer + self._voicemail_notifier = EventNotifier() # For clearing buffer + self._classifier_gate = ClassifierGate(self._gate_notifier) self._voicemail_processor = VoicemailProcessor( - self._conversation_notifier, on_voicemail_detected + gate_notifier=self._gate_notifier, + conversation_notifier=self._conversation_notifier, + voicemail_notifier=self._voicemail_notifier, + on_voicemail_detected=on_voicemail_detected, + ) + self._voicemail_buffer = VoicemailBuffer( + self._conversation_notifier, self._voicemail_notifier ) super().__init__( @@ -222,3 +330,19 @@ Respond with ONLY "YES" if it's a voicemail, or "NO" if it's a conversation atte self._context_aggregator.assistant(), ], ) + + def detector(self) -> "VoicemailDetector": + """Get the detector pipeline (for placement after STT). + + Returns: + The VoicemailDetector instance itself. + """ + return self + + def buffer(self) -> VoicemailBuffer: + """Get the buffer processor (for placement after TTS). + + Returns: + The VoicemailBuffer processor instance. + """ + return self._voicemail_buffer From ab03db5b0c5f6efe5d03c79750ac8278f4892602 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 8 Aug 2025 15:37:46 -0400 Subject: [PATCH 04/22] Updated prompt, add custom system_prompt input --- .../utils/voicemail/voicemail_detector.py | 77 +++++++++++-------- 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/src/pipecat/utils/voicemail/voicemail_detector.py b/src/pipecat/utils/voicemail/voicemail_detector.py index 6473b3675..24ce0b3a7 100644 --- a/src/pipecat/utils/voicemail/voicemail_detector.py +++ b/src/pipecat/utils/voicemail/voicemail_detector.py @@ -40,7 +40,7 @@ class ClassifierGate(FrameProcessor): """Gate processor that controls frame flow based on classification decisions. The gate starts open and closes permanently once a classification decision - is made (YES or NO). This ensures the classifier only runs until a definitive + is made (LIVE or MAIL). This ensures the classifier only runs until a definitive decision is reached. """ @@ -102,9 +102,9 @@ class ClassifierGate(FrameProcessor): class VoicemailProcessor(FrameProcessor): """Processor that handles LLM classification responses and triggers callbacks. - Processes LLM text responses to determine if the call is a voicemail (YES) - or conversation (NO), then triggers appropriate actions including developer - callbacks for voicemail detection. + Processes LLM text responses to determine if the call is a voicemail (MAIL) + or conversation (LIVE), then triggers appropriate actions including + developer callbacks for voicemail detection. """ def __init__( @@ -140,11 +140,11 @@ class VoicemailProcessor(FrameProcessor): if isinstance(frame, LLMTextFrame): response = frame.text.strip().upper() - if "NO" in response: - logger.info(f"{self}: CONVERSATION detected - notifying to close gate") + if "LIVE" in response: + logger.info(f"{self}: LIVE conversation detected - releasing buffer") await self._gate_notifier.notify() await self._conversation_notifier.notify() - elif "YES" in response: + elif "MAIL" in response: logger.info(f"{self}: VOICEMAIL detected - triggering callback") # Notify gate to close (decision is final) await self._gate_notifier.notify() @@ -260,53 +260,68 @@ class VoicemailDetector(ParallelPipeline): 2. Classification branch: LLM-based classification that can interrupt for voicemail The classifier runs in parallel and makes a one-time decision to either: - - Continue normal conversation flow (NO response) - - Interrupt and trigger voicemail handling (YES response) + - Continue normal conversation flow (LIVE response) + - Interrupt and trigger voicemail handling (MAIL response) """ + # Default prompt + DEFAULT_SYSTEM_PROMPT = """You are a voicemail detection classifier for an OUTBOUND calling system. A bot has called a phone number and you need to determine if a human answered or if the call went to voicemail based on the provided text. + +HUMAN ANSWERED - LIVE CONVERSATION (respond "LIVE"): +- Personal greetings: "Hello?", "Hi", "Yeah?", "John speaking" +- Interactive responses: "Who is this?", "What do you want?", "Can I help you?" +- Conversational tone expecting back-and-forth dialogue +- Questions directed at the caller: "Hello? Anyone there?" +- Informal responses: "Yep", "What's up?", "Speaking" +- Natural, spontaneous speech patterns +- Immediate acknowledgment of the call + +VOICEMAIL SYSTEM (respond "MAIL"): +- Automated voicemail greetings: "Hi, you've reached [name], please leave a message" +- Phone carrier messages: "The number you have dialed is not in service", "Please leave a message", "All circuits are busy" +- Professional voicemail: "This is [name], I'm not available right now" +- Instructions about leaving messages: "leave a message", "leave your name and number" +- References to callback or messaging: "call me back", "I'll get back to you" +- Carrier system messages: "mailbox is full", "has not been set up" +- Business hours messages: "our office is currently closed" + +Respond with ONLY "LIVE" if a person answered, or "MAIL" if it's voicemail/recording.""" + def __init__( self, *, llm: LLMService, on_voicemail_detected: Optional[Callable[[], Awaitable[None]]] = None, + system_prompt: Optional[str] = None, ): """Initialize the voicemail detector. Args: llm: LLM service for classification. on_voicemail_detected: Callback function called when voicemail is detected. + system_prompt: Optional custom system prompt for classification. If None, uses + default prompt optimized for outbound calling scenarios. If providing a + custom prompt, ensure it results in a clear "LIVE" or "MAIL" response, where + "LIVE" indicates a human answered and "MAIL" indicates voicemail. """ self._classifier_llm = llm + self._prompt = system_prompt if system_prompt is not None else self.DEFAULT_SYSTEM_PROMPT + + if system_prompt is not None: + self._validate_prompt(system_prompt) + self._messages = [ { "role": "system", - "content": """You are a voicemail detection classifier. Your job is to determine if the caller is leaving a voicemail message or trying to have a live conversation. - -VOICEMAIL INDICATORS (respond "YES"): -- One-way communication (caller talks without expecting immediate responses) -- Messages like "Hi, this is [name], please call me back" -- "I'm calling about..." followed by details without pausing for response -- "Leave me a message" or "call me when you get this" -- Monologue-style speech patterns -- Mentions of time/date when they're calling -- Business-like messages with contact information - -CONVERSATION INDICATORS (respond "NO"): -- Interactive speech ("Hello?", "Are you there?", "Can you hear me?") -- Questions directed at the recipient expecting immediate answers -- Responses to prompts or questions -- Back-and-forth dialogue patterns -- Greetings expecting responses ("Hi, how are you?") -- Real-time problem solving or discussion - -Respond with ONLY "YES" if it's a voicemail, or "NO" if it's a conversation attempt. Do not explain your reasoning.""", + "content": self._prompt, }, ] + self._context = OpenAILLMContext(self._messages) self._context_aggregator = llm.create_context_aggregator(self._context) self._gate_notifier = EventNotifier() - self._conversation_notifier = EventNotifier() # For releasing buffer - self._voicemail_notifier = EventNotifier() # For clearing buffer + self._conversation_notifier = EventNotifier() + self._voicemail_notifier = EventNotifier() self._classifier_gate = ClassifierGate(self._gate_notifier) self._voicemail_processor = VoicemailProcessor( gate_notifier=self._gate_notifier, From 0067c7df47aeaff31d104dc1a316845f34caafeb Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 8 Aug 2025 16:00:15 -0400 Subject: [PATCH 05/22] Add aggregation to classifier LLM output and validate prompt --- .../utils/voicemail/voicemail_detector.py | 119 +++++++++++++----- 1 file changed, 87 insertions(+), 32 deletions(-) diff --git a/src/pipecat/utils/voicemail/voicemail_detector.py b/src/pipecat/utils/voicemail/voicemail_detector.py index 24ce0b3a7..8947f662a 100644 --- a/src/pipecat/utils/voicemail/voicemail_detector.py +++ b/src/pipecat/utils/voicemail/voicemail_detector.py @@ -23,6 +23,8 @@ from pipecat.frames.frames import ( EndFrame, EndTaskFrame, Frame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, LLMTextFrame, StartFrame, TTSAudioRawFrame, @@ -40,8 +42,8 @@ class ClassifierGate(FrameProcessor): """Gate processor that controls frame flow based on classification decisions. The gate starts open and closes permanently once a classification decision - is made (LIVE or MAIL). This ensures the classifier only runs until a definitive - decision is reached. + is made (CONVERSATION or VOICEMAIL). This ensures the classifier only runs until + a definitive decision is reached. """ def __init__(self, notifier: BaseNotifier): @@ -102,9 +104,9 @@ class ClassifierGate(FrameProcessor): class VoicemailProcessor(FrameProcessor): """Processor that handles LLM classification responses and triggers callbacks. - Processes LLM text responses to determine if the call is a voicemail (MAIL) - or conversation (LIVE), then triggers appropriate actions including - developer callbacks for voicemail detection. + Processes LLM text responses to determine if the call is a voicemail (VOICEMAIL) + or conversation (CONVERSATION), then triggers appropriate actions including developer + callbacks for voicemail detection. """ def __init__( @@ -129,6 +131,11 @@ class VoicemailProcessor(FrameProcessor): self._voicemail_notifier = voicemail_notifier self._on_voicemail_detected = on_voicemail_detected + # Aggregation state + self._aggregating = False + self._response_buffer = "" + self._decision_made = False + async def process_frame(self, frame: Frame, direction: FrameDirection): """Process frames and handle LLM classification responses. @@ -138,30 +145,61 @@ class VoicemailProcessor(FrameProcessor): """ await super().process_frame(frame, direction) - if isinstance(frame, LLMTextFrame): - response = frame.text.strip().upper() - if "LIVE" in response: - logger.info(f"{self}: LIVE conversation detected - releasing buffer") - await self._gate_notifier.notify() - await self._conversation_notifier.notify() - elif "MAIL" in response: - logger.info(f"{self}: VOICEMAIL detected - triggering callback") - # Notify gate to close (decision is final) - await self._gate_notifier.notify() - await self._voicemail_notifier.notify() - # Push BotInterruptionFrame to clear the pipeline - await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM) - # Call developer callback if provided - if self._on_voicemail_detected: - try: - await self._on_voicemail_detected(self) - except Exception as e: - logger.exception(f"{self}: Error in voicemail callback: {e}") + if isinstance(frame, LLMFullResponseStartFrame): + # Start aggregating the LLM response + self._aggregating = True + self._response_buffer = "" + logger.debug(f"{self}: Starting LLM response aggregation") + + elif isinstance(frame, LLMFullResponseEndFrame): + # End of LLM response - make decision + if self._aggregating and not self._decision_made: + await self._process_classification(self._response_buffer.strip()) + self._aggregating = False + self._response_buffer = "" + + elif isinstance(frame, LLMTextFrame) and self._aggregating: + # Accumulate text tokens + self._response_buffer += frame.text + logger.debug(f"{self}: Accumulated: '{self._response_buffer}'") else: - # Push the frame + # Always push the frame downstream (for context aggregator) await self.push_frame(frame, direction) + async def _process_classification(self, full_response: str): + """Process the complete LLM classification response. + + Args: + full_response: The complete aggregated response from the LLM. + """ + if self._decision_made: + return + + response = full_response.upper() + logger.info(f"{self}: Processing classification: '{full_response}'") + + if "CONVERSATION" in response: + self._decision_made = True + logger.info(f"{self}: CONVERSATION detected - releasing buffer") + await self._gate_notifier.notify() + await self._conversation_notifier.notify() + + elif "VOICEMAIL" in response: + self._decision_made = True + logger.info(f"{self}: VOICEMAIL detected - triggering callback") + await self._gate_notifier.notify() + await self._voicemail_notifier.notify() + await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM) + + if self._on_voicemail_detected: + try: + await self._on_voicemail_detected(self) + except Exception as e: + logger.exception(f"{self}: Error in voicemail callback: {e}") + else: + logger.warning(f"{self}: Unexpected classification response: '{full_response}'") + class VoicemailBuffer(FrameProcessor): """Buffers TTS frames until voicemail classification decision is made. @@ -260,14 +298,14 @@ class VoicemailDetector(ParallelPipeline): 2. Classification branch: LLM-based classification that can interrupt for voicemail The classifier runs in parallel and makes a one-time decision to either: - - Continue normal conversation flow (LIVE response) - - Interrupt and trigger voicemail handling (MAIL response) + - Continue normal conversation flow (CONVERSATION response) + - Interrupt and trigger voicemail handling (VOICEMAIL response) """ # Default prompt DEFAULT_SYSTEM_PROMPT = """You are a voicemail detection classifier for an OUTBOUND calling system. A bot has called a phone number and you need to determine if a human answered or if the call went to voicemail based on the provided text. -HUMAN ANSWERED - LIVE CONVERSATION (respond "LIVE"): +HUMAN ANSWERED - LIVE CONVERSATION (respond "CONVERSATION"): - Personal greetings: "Hello?", "Hi", "Yeah?", "John speaking" - Interactive responses: "Who is this?", "What do you want?", "Can I help you?" - Conversational tone expecting back-and-forth dialogue @@ -276,7 +314,7 @@ HUMAN ANSWERED - LIVE CONVERSATION (respond "LIVE"): - Natural, spontaneous speech patterns - Immediate acknowledgment of the call -VOICEMAIL SYSTEM (respond "MAIL"): +VOICEMAIL SYSTEM (respond "VOICEMAIL"): - Automated voicemail greetings: "Hi, you've reached [name], please leave a message" - Phone carrier messages: "The number you have dialed is not in service", "Please leave a message", "All circuits are busy" - Professional voicemail: "This is [name], I'm not available right now" @@ -285,7 +323,7 @@ VOICEMAIL SYSTEM (respond "MAIL"): - Carrier system messages: "mailbox is full", "has not been set up" - Business hours messages: "our office is currently closed" -Respond with ONLY "LIVE" if a person answered, or "MAIL" if it's voicemail/recording.""" +Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's voicemail/recording.""" def __init__( self, @@ -301,8 +339,8 @@ Respond with ONLY "LIVE" if a person answered, or "MAIL" if it's voicemail/recor on_voicemail_detected: Callback function called when voicemail is detected. system_prompt: Optional custom system prompt for classification. If None, uses default prompt optimized for outbound calling scenarios. If providing a - custom prompt, ensure it results in a clear "LIVE" or "MAIL" response, where - "LIVE" indicates a human answered and "MAIL" indicates voicemail. + custom prompt, ensure it results in a clear "CONVERSATION" or "VOICEMAIL" response, + where "CONVERSATION" indicates a human answered and "VOICEMAIL" indicates voicemail. """ self._classifier_llm = llm self._prompt = system_prompt if system_prompt is not None else self.DEFAULT_SYSTEM_PROMPT @@ -346,6 +384,23 @@ Respond with ONLY "LIVE" if a person answered, or "MAIL" if it's voicemail/recor ], ) + def _validate_prompt(self, prompt: str) -> None: + """Validate custom prompt contains essential instructions. + + Args: + prompt: The custom system prompt to validate. + """ + # Check for exact response format requirements + has_conversation = "CONVERSATION" in prompt + has_voicemail = "VOICEMAIL" in prompt + + if not has_conversation or not has_voicemail: + logger.warning( + "Custom system prompt should instruct the LLM to respond with exactly " + '"CONVERSATION" or "VOICEMAIL" for proper detection functionality. ' + 'Example: "Respond with ONLY \\"CONVERSATION\\" if a person answered, or \\"VOICEMAIL\\" if it\'s voicemail/recording."' + ) + def detector(self) -> "VoicemailDetector": """Get the detector pipeline (for placement after STT). From 5ca82ec61ecd19fc77f1bd625c759f211bd77730 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 8 Aug 2025 16:42:35 -0400 Subject: [PATCH 06/22] Final docstrings, comments, and cleanup --- ...mail_test.py => 44-voicemail-detection.py} | 20 +- .../utils/voicemail/voicemail_detector.py | 291 +++++++++++++----- 2 files changed, 216 insertions(+), 95 deletions(-) rename examples/foundational/{voicemail_test.py => 44-voicemail-detection.py} (90%) diff --git a/examples/foundational/voicemail_test.py b/examples/foundational/44-voicemail-detection.py similarity index 90% rename from examples/foundational/voicemail_test.py rename to examples/foundational/44-voicemail-detection.py index 04ade85ca..24f5d1762 100644 --- a/examples/foundational/voicemail_test.py +++ b/examples/foundational/44-voicemail-detection.py @@ -82,9 +82,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - voicemail_llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + classifier_llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - voicemail = VoicemailDetector(llm=voicemail_llm, on_voicemail_detected=handle_voicemail) + voicemail = VoicemailDetector(llm=classifier_llm, on_voicemail_detected=handle_voicemail) messages = [ { @@ -98,15 +98,15 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): pipeline = Pipeline( [ - transport.input(), # Transport user input + transport.input(), stt, - voicemail.detector(), - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - voicemail.buffer(), - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses + voicemail.detector(), # Voicemail detection + context_aggregator.user(), + llm, + tts, + voicemail.buffer(), # TTS buffering + transport.output(), + context_aggregator.assistant(), ] ) diff --git a/src/pipecat/utils/voicemail/voicemail_detector.py b/src/pipecat/utils/voicemail/voicemail_detector.py index 8947f662a..97144a4e7 100644 --- a/src/pipecat/utils/voicemail/voicemail_detector.py +++ b/src/pipecat/utils/voicemail/voicemail_detector.py @@ -8,7 +8,8 @@ This module provides voicemail detection capabilities using parallel pipeline processing to classify incoming calls as either voicemail messages or live -conversations. +conversations. It's specifically designed for outbound calling scenarios where +a bot needs to determine if a human answered or if the call went to voicemail. """ import asyncio @@ -41,16 +42,21 @@ from pipecat.sync.event_notifier import EventNotifier class ClassifierGate(FrameProcessor): """Gate processor that controls frame flow based on classification decisions. - The gate starts open and closes permanently once a classification decision - is made (CONVERSATION or VOICEMAIL). This ensures the classifier only runs until - a definitive decision is reached. + The gate starts open to allow initial classification processing and closes + permanently once a classification decision is made (CONVERSATION or VOICEMAIL). + This ensures the classifier only runs until a definitive decision is reached, + preventing unnecessary LLM calls and maintaining system efficiency. + + The gate allows all frames to pass through while open, but once closed, only + allows system frames (interruptions, end frames, cancel frames) to continue. """ def __init__(self, notifier: BaseNotifier): """Initialize the classifier gate. Args: - notifier: Notifier that signals when to close the gate. + notifier: Notifier that signals when a classification decision has + been made and the gate should close. """ super().__init__() self._notifier = notifier @@ -58,24 +64,25 @@ class ClassifierGate(FrameProcessor): self._gate_task: Optional[asyncio.Task] = None async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process frames and control gate state. + """Process frames and control gate state based on classification decisions. Args: frame: The frame to process. - direction: The direction of frame flow. + direction: The direction of frame flow in the pipeline. """ await super().process_frame(frame, direction) if isinstance(frame, StartFrame): - # Start the task immediately, don't wait for other conditions + # Start the notification waiting task immediately self._gate_task = self.create_task(self._wait_for_notification()) - logger.info(f"{self}: Gate task started, waiting for notification") elif isinstance(frame, (EndFrame, CancelFrame)): + # Clean up the gate task when pipeline ends or is cancelled if self._gate_task: await self.cancel_task(self._gate_task) self._gate_task = None + # Gate logic: open gate allows all frames, closed gate only allows specific system frames if self._gate_opened: await self.push_frame(frame, direction) elif not self._gate_opened and isinstance( @@ -84,15 +91,18 @@ class ClassifierGate(FrameProcessor): await self.push_frame(frame, direction) async def _wait_for_notification(self): - """Wait for classification decision notification.""" + """Wait for classification decision notification and close the gate. + + This method blocks until the VoicemailProcessor makes a classification + decision and signals through the notifier. Once notified, the gate + closes permanently to stop further classification processing. + """ try: - logger.info(f"{self}: Waiting for notification...") await self._notifier.wait() - logger.info(f"{self}: Received notification!") if self._gate_opened: self._gate_opened = False - logger.info(f"{self}: Gate closed") + logger.debug(f"{self}: Gate closed - classification complete") except asyncio.CancelledError: logger.debug(f"{self}: Gate task was cancelled") raise @@ -104,26 +114,37 @@ class ClassifierGate(FrameProcessor): class VoicemailProcessor(FrameProcessor): """Processor that handles LLM classification responses and triggers callbacks. - Processes LLM text responses to determine if the call is a voicemail (VOICEMAIL) - or conversation (CONVERSATION), then triggers appropriate actions including developer - callbacks for voicemail detection. + This processor aggregates LLM text tokens into complete responses and analyzes + them to determine if the call reached a voicemail system or a live person. + It uses the LLM response frame delimiters (LLMFullResponseStartFrame and + LLMFullResponseEndFrame) to ensure complete token aggregation regardless + of how the LLM tokenizes the response words. + + The processor expects responses containing either "CONVERSATION" (indicating + a human answered) or "VOICEMAIL" (indicating an automated system). Once a + decision is made, it triggers the appropriate notifications and callbacks. """ def __init__( self, *, gate_notifier: BaseNotifier, - conversation_notifier: BaseNotifier, # Buffer should release frames - voicemail_notifier: BaseNotifier, # Buffer should clear frames + conversation_notifier: BaseNotifier, + voicemail_notifier: BaseNotifier, on_voicemail_detected: Optional[Callable[["VoicemailProcessor"], Awaitable[None]]] = None, ): """Initialize the voicemail processor. Args: - gate_notifier: Notifier to signal gate about classification decisions. - conversation_notifier: Notifier to signal buffer to release frames. - voicemail_notifier: Notifier to signal buffer to clear frames. - on_voicemail_detected: Callback function called when voicemail is detected. + gate_notifier: Notifier to signal the ClassifierGate about classification + decisions so it can close and stop processing. + conversation_notifier: Notifier to signal the VoicemailBuffer to release + all buffered TTS frames for normal conversation flow. + voicemail_notifier: Notifier to signal the VoicemailBuffer to clear + buffered TTS frames since voicemail was detected. + on_voicemail_detected: Optional callback function called when voicemail + is detected. The callback receives this processor instance and can + use it to push custom frames (like voicemail greetings). """ super().__init__() self._gate_notifier = gate_notifier @@ -131,90 +152,117 @@ class VoicemailProcessor(FrameProcessor): self._voicemail_notifier = voicemail_notifier self._on_voicemail_detected = on_voicemail_detected - # Aggregation state - self._aggregating = False + # Aggregation state for collecting complete LLM responses + self._processing_response = False self._response_buffer = "" self._decision_made = False async def process_frame(self, frame: Frame, direction: FrameDirection): """Process frames and handle LLM classification responses. + This method implements a state machine for aggregating LLM responses: + 1. LLMFullResponseStartFrame: Begin collecting tokens + 2. LLMTextFrame: Accumulate text tokens into buffer + 3. LLMFullResponseEndFrame: Process complete response and make decision + Args: frame: The frame to process. - direction: The direction of frame flow. + direction: The direction of frame flow in the pipeline. """ await super().process_frame(frame, direction) if isinstance(frame, LLMFullResponseStartFrame): - # Start aggregating the LLM response - self._aggregating = True + # Begin aggregating a new LLM response + self._processing_response = True self._response_buffer = "" logger.debug(f"{self}: Starting LLM response aggregation") elif isinstance(frame, LLMFullResponseEndFrame): - # End of LLM response - make decision - if self._aggregating and not self._decision_made: + # Complete response received - make classification decision + if self._processing_response and not self._decision_made: await self._process_classification(self._response_buffer.strip()) - self._aggregating = False + self._processing_response = False self._response_buffer = "" - elif isinstance(frame, LLMTextFrame) and self._aggregating: - # Accumulate text tokens + elif isinstance(frame, LLMTextFrame) and self._processing_response: + # Accumulate text tokens from the streaming LLM response self._response_buffer += frame.text - logger.debug(f"{self}: Accumulated: '{self._response_buffer}'") + logger.trace(f"{self}: Buffer: '{self._response_buffer}'") else: - # Always push the frame downstream (for context aggregator) + # Pass all non-LLM frames through + # Blocking LLM frames prevents interferences with the downsteram LLM await self.push_frame(frame, direction) async def _process_classification(self, full_response: str): - """Process the complete LLM classification response. + """Process the complete LLM classification response and trigger actions. + + Analyzes the aggregated response text to determine if it contains + "CONVERSATION" or "VOICEMAIL" and triggers the appropriate notifications + and callbacks based on the classification result. Args: - full_response: The complete aggregated response from the LLM. + full_response: The complete aggregated response text from the LLM. """ if self._decision_made: + logger.debug(f"{self}: Decision already made, ignoring response") return response = full_response.upper() - logger.info(f"{self}: Processing classification: '{full_response}'") + logger.info(f"{self}: Classifying response: '{full_response}'") if "CONVERSATION" in response: + # Human answered - continue normal conversation flow self._decision_made = True - logger.info(f"{self}: CONVERSATION detected - releasing buffer") - await self._gate_notifier.notify() - await self._conversation_notifier.notify() + logger.info(f"{self}: CONVERSATION detected - releasing TTS buffer") + await self._gate_notifier.notify() # Close the classifier gate + await self._conversation_notifier.notify() # Release buffered TTS frames elif "VOICEMAIL" in response: + # Voicemail detected - trigger voicemail handling self._decision_made = True - logger.info(f"{self}: VOICEMAIL detected - triggering callback") - await self._gate_notifier.notify() - await self._voicemail_notifier.notify() + logger.info(f"{self}: VOICEMAIL detected - clearing TTS buffer and triggering callback") + await self._gate_notifier.notify() # Close the classifier gate + await self._voicemail_notifier.notify() # Clear buffered TTS frames + + # Interrupt the current pipeline to stop any ongoing processing await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM) + # Execute developer callback for custom voicemail handling if self._on_voicemail_detected: try: await self._on_voicemail_detected(self) except Exception as e: logger.exception(f"{self}: Error in voicemail callback: {e}") else: + # Unexpected response - log warning but don't crash logger.warning(f"{self}: Unexpected classification response: '{full_response}'") class VoicemailBuffer(FrameProcessor): """Buffers TTS frames until voicemail classification decision is made. - Holds TTS frames in a buffer while voicemail classification is in progress. - Releases all buffered frames when conversation is detected, or keeps them - buffered when voicemail is detected. + This processor holds TTS output frames in a buffer while the voicemail + classification is in progress. This prevents audio from being played + to the caller before determining if they're human or a voicemail system. + + The buffer operates in two modes based on the classification result: + + - CONVERSATION: Releases all buffered frames to continue normal dialogue + - VOICEMAIL: Clears buffered frames since they're not needed for voicemail + + The buffering only applies to TTS-related frames (TTSTextFrame, TTSAudioRawFrame). + All other frames pass through immediately to maintain proper pipeline flow. """ def __init__(self, conversation_notifier: BaseNotifier, voicemail_notifier: BaseNotifier): """Initialize the voicemail buffer. Args: - conversation_notifier: Notifier that signals when to release buffered frames. - voicemail_notifier: Notifier that signals when to keep buffered frames. + conversation_notifier: Notifier that signals when a conversation is + detected and buffered frames should be released for playback. + voicemail_notifier: Notifier that signals when voicemail is detected + and buffered frames should be cleared (not played). """ super().__init__() self._conversation_notifier = conversation_notifier @@ -225,20 +273,26 @@ class VoicemailBuffer(FrameProcessor): self._voicemail_task: Optional[asyncio.Task] = None async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process frames and handle buffering logic. + """Process frames and handle buffering logic based on frame type. + + TTS frames are buffered while classification is active. All other frames + pass through immediately. The buffering state is controlled by the + classification notifications. Args: frame: The frame to process. - direction: The direction of frame flow. + direction: The direction of frame flow in the pipeline. """ await super().process_frame(frame, direction) if isinstance(frame, StartFrame): + # Start notification waiting tasks for both conversation and voicemail self._conversation_task = self.create_task(self._wait_for_conversation()) self._voicemail_task = self.create_task(self._wait_for_voicemail()) - logger.info(f"{self}: Buffer tasks started") await self.push_frame(frame, direction) + elif isinstance(frame, (EndFrame, CancelFrame)): + # Clean up notification tasks when pipeline ends if self._conversation_task: await self.cancel_task(self._conversation_task) self._conversation_task = None @@ -247,62 +301,109 @@ class VoicemailBuffer(FrameProcessor): self._voicemail_task = None await self.push_frame(frame, direction) - # Buffer TTS frames while buffering is active - if self._buffering_active and isinstance(frame, (TTSTextFrame, TTSAudioRawFrame)): + # Core buffering logic: hold TTS frames, pass everything else through + elif self._buffering_active and isinstance(frame, (TTSTextFrame, TTSAudioRawFrame)): + # Buffer TTS frames while waiting for classification decision self._frame_buffer.append((frame, direction)) else: + # Pass through all non-TTS frames immediately await self.push_frame(frame, direction) async def _wait_for_conversation(self): - """Wait for conversation detection - release buffered frames.""" + """Wait for conversation detection notification and release buffered frames. + + When a conversation is detected, all buffered TTS frames are released + in order to continue normal dialogue flow. This allows the bot to + respond naturally to the human caller. + """ try: await self._conversation_notifier.wait() - logger.info(f"{self}: CONVERSATION - releasing frames") + # Release all buffered frames in original order self._buffering_active = False for frame, direction in self._frame_buffer: await self.push_frame(frame, direction) self._frame_buffer.clear() - # Cancel the other task + # Cancel the voicemail task since decision is final if self._voicemail_task: await self.cancel_task(self._voicemail_task) self._voicemail_task = None except asyncio.CancelledError: + logger.debug(f"{self}: Conversation task was cancelled") raise async def _wait_for_voicemail(self): - """Wait for voicemail detection - clear buffered frames.""" + """Wait for voicemail detection notification and clear buffered frames. + + When voicemail is detected, all buffered TTS frames are discarded + since they were intended for human conversation and are not appropriate + for voicemail systems. The developer callback will handle voicemail- + specific audio output. + """ try: await self._voicemail_notifier.wait() - logger.info(f"{self}: VOICEMAIL - clearing frames") + # Clear buffered frames without playing them self._buffering_active = False self._frame_buffer.clear() - # Cancel the other task + # Cancel the conversation task since decision is final if self._conversation_task: await self.cancel_task(self._conversation_task) self._conversation_task = None except asyncio.CancelledError: + logger.debug(f"{self}: Voicemail task was cancelled") raise class VoicemailDetector(ParallelPipeline): - """Parallel pipeline for detecting voicemail vs. live conversation. + """Parallel pipeline for detecting voicemail vs. live conversation in outbound calls. - Uses a parallel pipeline architecture with two branches: - 1. Conversation branch: Normal frame flow for live conversations - 2. Classification branch: LLM-based classification that can interrupt for voicemail + This detector uses a parallel pipeline architecture to perform real-time + classification of outbound phone calls without interrupting the conversation + flow. It determines whether a human answered the phone or if the call went + to a voicemail system. - The classifier runs in parallel and makes a one-time decision to either: - - Continue normal conversation flow (CONVERSATION response) - - Interrupt and trigger voicemail handling (VOICEMAIL response) + Architecture: + + - Conversation branch: Empty pipeline that allows normal frame flow + - Classification branch: Contains the LLM classifier and decision logic + + The system uses a gate mechanism to control when classification runs and + a buffering system to prevent TTS output until classification is complete. + Once a decision is made, the appropriate action is taken: + + - CONVERSATION: Continue normal bot dialogue + - VOICEMAIL: Trigger developer callback for custom voicemail handling + + Example:: + + classification_llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + async def handle_voicemail(processor): + await processor.push_frame(TTSSpeakFrame("Please leave a message.")) + + detector = VoicemailDetector( + llm=classification_llm, + on_voicemail_detected=handle_voicemail + ) + + pipeline = Pipeline([ + transport.input(), + stt, + detector.detector(), # Classification + context_aggregator.user(), + llm, + tts, + detector.buffer(), # TTS buffering + transport.output(), + context_aggregator.assistant(), + ]) """ - # Default prompt DEFAULT_SYSTEM_PROMPT = """You are a voicemail detection classifier for an OUTBOUND calling system. A bot has called a phone number and you need to determine if a human answered or if the call went to voicemail based on the provided text. HUMAN ANSWERED - LIVE CONVERSATION (respond "CONVERSATION"): @@ -329,25 +430,30 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo self, *, llm: LLMService, - on_voicemail_detected: Optional[Callable[[], Awaitable[None]]] = None, + on_voicemail_detected: Optional[Callable[["VoicemailProcessor"], Awaitable[None]]] = None, system_prompt: Optional[str] = None, ): - """Initialize the voicemail detector. + """Initialize the voicemail detector with classification and buffering components. Args: - llm: LLM service for classification. - on_voicemail_detected: Callback function called when voicemail is detected. - system_prompt: Optional custom system prompt for classification. If None, uses - default prompt optimized for outbound calling scenarios. If providing a - custom prompt, ensure it results in a clear "CONVERSATION" or "VOICEMAIL" response, - where "CONVERSATION" indicates a human answered and "VOICEMAIL" indicates voicemail. + llm: LLM service used for voicemail vs conversation classification. + Should be fast and reliable for real-time classification. + on_voicemail_detected: Optional callback function invoked when voicemail + is detected. Receives the VoicemailProcessor instance which can be + used to push frames (like custom voicemail greetings). + system_prompt: Optional custom system prompt for classification. If None, + uses the default prompt optimized for outbound calling scenarios. + Custom prompts should instruct the LLM to respond with exactly + "CONVERSATION" or "VOICEMAIL" for proper detection functionality. """ self._classifier_llm = llm self._prompt = system_prompt if system_prompt is not None else self.DEFAULT_SYSTEM_PROMPT + # Validate custom prompts to ensure they work with the detection logic if system_prompt is not None: self._validate_prompt(system_prompt) + # Set up the LLM context with the classification prompt self._messages = [ { "role": "system", @@ -355,11 +461,16 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo }, ] + # Create the LLM context and aggregators for conversation management self._context = OpenAILLMContext(self._messages) self._context_aggregator = llm.create_context_aggregator(self._context) - self._gate_notifier = EventNotifier() - self._conversation_notifier = EventNotifier() - self._voicemail_notifier = EventNotifier() + + # Create notification system for coordinating between components + self._gate_notifier = EventNotifier() # Signals classification completion + self._conversation_notifier = EventNotifier() # Signals conversation detected + self._voicemail_notifier = EventNotifier() # Signals voicemail detected + + # Create the processor components self._classifier_gate = ClassifierGate(self._gate_notifier) self._voicemail_processor = VoicemailProcessor( gate_notifier=self._gate_notifier, @@ -371,10 +482,11 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo self._conversation_notifier, self._voicemail_notifier ) + # Initialize the parallel pipeline with conversation and classifier branches super().__init__( - # Conversation branch + # Conversation branch: empty pipeline for normal frame flow [], - # Classifer branch + # Classification branch: gate -> context -> LLM -> processor -> context [ self._classifier_gate, self._context_aggregator.user(), @@ -385,12 +497,15 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo ) def _validate_prompt(self, prompt: str) -> None: - """Validate custom prompt contains essential instructions. + """Validate custom prompt contains required response format instructions. + + Custom prompts must instruct the LLM to respond with exactly "CONVERSATION" + or "VOICEMAIL" for the detection logic to work properly. This method + checks for the presence of these keywords and warns if they're missing. Args: prompt: The custom system prompt to validate. """ - # Check for exact response format requirements has_conversation = "CONVERSATION" in prompt has_voicemail = "VOICEMAIL" in prompt @@ -402,15 +517,21 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo ) def detector(self) -> "VoicemailDetector": - """Get the detector pipeline (for placement after STT). + """Get the detector pipeline for placement after STT in the main pipeline. + + This should be placed after the STT service and before the context + aggregator in your main pipeline to enable voicemail classification. Returns: - The VoicemailDetector instance itself. + The VoicemailDetector instance itself (which is a ParallelPipeline). """ return self def buffer(self) -> VoicemailBuffer: - """Get the buffer processor (for placement after TTS). + """Get the buffer processor for placement after TTS in the main pipeline. + + This should be placed after the TTS service and before the transport + output to enable TTS frame buffering during classification. Returns: The VoicemailBuffer processor instance. From 9da33f3897fdaf15022bc2459ddfba76c65d4dd6 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 9 Aug 2025 07:23:51 -0400 Subject: [PATCH 07/22] Handle multiple user inputs from the user when a voicemail is detected; add a configurable timeout to emitting the callback --- .../foundational/44-voicemail-detection.py | 3 - .../utils/voicemail/voicemail_detector.py | 191 ++++++++++++++++-- 2 files changed, 176 insertions(+), 18 deletions(-) diff --git a/examples/foundational/44-voicemail-detection.py b/examples/foundational/44-voicemail-detection.py index 24f5d1762..64e78465f 100644 --- a/examples/foundational/44-voicemail-detection.py +++ b/examples/foundational/44-voicemail-detection.py @@ -61,9 +61,6 @@ async def handle_voicemail(processor): """ logger.info("Voicemail detected! Playing greeting...") - # Wait a moment for interruption to clear - await asyncio.sleep(1) - # Push frames using standard Pipecat pattern await processor.push_frame( TTSSpeakFrame("This is Mattie. Call me back when you can!"), diff --git a/src/pipecat/utils/voicemail/voicemail_detector.py b/src/pipecat/utils/voicemail/voicemail_detector.py index 97144a4e7..4a6b070c2 100644 --- a/src/pipecat/utils/voicemail/voicemail_detector.py +++ b/src/pipecat/utils/voicemail/voicemail_detector.py @@ -28,8 +28,12 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMTextFrame, StartFrame, + StartInterruptionFrame, + StopInterruptionFrame, TTSAudioRawFrame, TTSTextFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, ) from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext @@ -48,18 +52,19 @@ class ClassifierGate(FrameProcessor): preventing unnecessary LLM calls and maintaining system efficiency. The gate allows all frames to pass through while open, but once closed, only - allows system frames (interruptions, end frames, cancel frames) to continue. + allows system frames and user speaking frames to continue. Speaking frames + are needed for voicemail timing control. """ - def __init__(self, notifier: BaseNotifier): + def __init__(self, gate_notifier: BaseNotifier): """Initialize the classifier gate. Args: - notifier: Notifier that signals when a classification decision has + gate_notifier: Notifier that signals when a classification decision has been made and the gate should close. """ super().__init__() - self._notifier = notifier + self._gate_notifier = gate_notifier self._gate_opened = True self._gate_task: Optional[asyncio.Task] = None @@ -86,7 +91,18 @@ class ClassifierGate(FrameProcessor): if self._gate_opened: await self.push_frame(frame, direction) elif not self._gate_opened and isinstance( - frame, (BotInterruptionFrame, EndTaskFrame, EndFrame, CancelTaskFrame, CancelFrame) + frame, + ( + BotInterruptionFrame, + EndTaskFrame, + EndFrame, + CancelTaskFrame, + CancelFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, + StartInterruptionFrame, + StopInterruptionFrame, + ), ): await self.push_frame(frame, direction) @@ -98,7 +114,7 @@ class ClassifierGate(FrameProcessor): closes permanently to stop further classification processing. """ try: - await self._notifier.wait() + await self._gate_notifier.wait() if self._gate_opened: self._gate_opened = False @@ -111,6 +127,86 @@ class ClassifierGate(FrameProcessor): raise +class ConversationGate(FrameProcessor): + """Gate processor that blocks conversation flow when voicemail is detected. + + This gate starts open to allow normal conversation processing but closes + permanently when voicemail is detected. This prevents the main conversation + LLM from processing additional input after voicemail classification. + """ + + def __init__(self, voicemail_notifier: BaseNotifier): + """Initialize the conversation gate. + + Args: + voicemail_notifier: Notifier that signals when voicemail has been + detected and the conversation should be blocked. + """ + super().__init__() + self._voicemail_notifier = voicemail_notifier + self._gate_opened = True + self._voicemail_task: Optional[asyncio.Task] = None + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames and control gate state based on voicemail detection. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ + await super().process_frame(frame, direction) + + if isinstance(frame, StartFrame): + # Start the notification waiting task immediately + self._voicemail_task = self.create_task(self._wait_for_voicemail()) + + elif isinstance(frame, (EndFrame, CancelFrame)): + # Clean up the task when pipeline ends or is cancelled + if self._voicemail_task: + await self.cancel_task(self._voicemail_task) + self._voicemail_task = None + + # Gate logic: open gate allows all frames, closed gate blocks everything + if self._gate_opened: + await self.push_frame(frame, direction) + elif not self._gate_opened and isinstance( + frame, + ( + BotInterruptionFrame, + EndTaskFrame, + EndFrame, + CancelTaskFrame, + CancelFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, + StartInterruptionFrame, + StopInterruptionFrame, + ), + ): + # Only allow system frames and user speaking frames through when closed + await self.push_frame(frame, direction) + # When closed, don't push any frames (complete conversation blocking) + + async def _wait_for_voicemail(self): + """Wait for voicemail detection notification and close the gate. + + This method blocks until voicemail is detected, then closes the gate + permanently to prevent any further conversation processing. + """ + try: + await self._voicemail_notifier.wait() + + if self._gate_opened: + self._gate_opened = False + logger.debug(f"{self}: Conversation gate closed - voicemail detected") + except asyncio.CancelledError: + logger.debug(f"{self}: Conversation gate task was cancelled") + raise + except Exception as e: + logger.exception(f"{self}: Error in conversation gate task: {e}") + raise + + class VoicemailProcessor(FrameProcessor): """Processor that handles LLM classification responses and triggers callbacks. @@ -123,6 +219,9 @@ class VoicemailProcessor(FrameProcessor): The processor expects responses containing either "CONVERSATION" (indicating a human answered) or "VOICEMAIL" (indicating an automated system). Once a decision is made, it triggers the appropriate notifications and callbacks. + + For voicemail detection, the callback timer starts immediately and is cancelled + and restarted based on user speech patterns to ensure proper timing. """ def __init__( @@ -132,6 +231,7 @@ class VoicemailProcessor(FrameProcessor): conversation_notifier: BaseNotifier, voicemail_notifier: BaseNotifier, on_voicemail_detected: Optional[Callable[["VoicemailProcessor"], Awaitable[None]]] = None, + voicemail_response_delay: float, ): """Initialize the voicemail processor. @@ -145,18 +245,26 @@ class VoicemailProcessor(FrameProcessor): on_voicemail_detected: Optional callback function called when voicemail is detected. The callback receives this processor instance and can use it to push custom frames (like voicemail greetings). + voicemail_response_delay: Delay in seconds after user stops speaking + before triggering the voicemail callback. This ensures the voicemail + greeting or user message is complete before responding. """ super().__init__() self._gate_notifier = gate_notifier self._conversation_notifier = conversation_notifier self._voicemail_notifier = voicemail_notifier self._on_voicemail_detected = on_voicemail_detected + self._voicemail_response_delay = voicemail_response_delay # Aggregation state for collecting complete LLM responses self._processing_response = False self._response_buffer = "" self._decision_made = False + # Voicemail timing state + self._voicemail_detected = False + self._voicemail_callback_task: Optional[asyncio.Task] = None + async def process_frame(self, frame: Frame, direction: FrameDirection): """Process frames and handle LLM classification responses. @@ -164,6 +272,7 @@ class VoicemailProcessor(FrameProcessor): 1. LLMFullResponseStartFrame: Begin collecting tokens 2. LLMTextFrame: Accumulate text tokens into buffer 3. LLMFullResponseEndFrame: Process complete response and make decision + 4. UserStartedSpeakingFrame/UserStoppedSpeakingFrame: Manage voicemail timing Args: frame: The frame to process. @@ -189,9 +298,24 @@ class VoicemailProcessor(FrameProcessor): self._response_buffer += frame.text logger.trace(f"{self}: Buffer: '{self._response_buffer}'") + elif isinstance(frame, UserStartedSpeakingFrame): + # User started speaking - cancel voicemail callback timer + if self._voicemail_callback_task: + logger.debug(f"{self}: User started speaking, cancelling voicemail callback") + await self.cancel_task(self._voicemail_callback_task) + self._voicemail_callback_task = None + + elif isinstance(frame, UserStoppedSpeakingFrame): + # User stopped speaking - restart voicemail callback timer if voicemail detected + if self._voicemail_detected and not self._voicemail_callback_task: + logger.debug( + f"{self}: User stopped speaking, restarting voicemail callback timer ({self._voicemail_response_delay}s)" + ) + self._voicemail_callback_task = self.create_task(self._delayed_voicemail_callback()) + else: # Pass all non-LLM frames through - # Blocking LLM frames prevents interferences with the downsteram LLM + # Blocking LLM frames prevents interference with the downstream LLM await self.push_frame(frame, direction) async def _process_classification(self, full_response: str): @@ -214,29 +338,58 @@ class VoicemailProcessor(FrameProcessor): if "CONVERSATION" in response: # Human answered - continue normal conversation flow self._decision_made = True - logger.info(f"{self}: CONVERSATION detected - releasing TTS buffer") + logger.info(f"{self}: CONVERSATION detected") await self._gate_notifier.notify() # Close the classifier gate await self._conversation_notifier.notify() # Release buffered TTS frames elif "VOICEMAIL" in response: # Voicemail detected - trigger voicemail handling self._decision_made = True - logger.info(f"{self}: VOICEMAIL detected - clearing TTS buffer and triggering callback") + self._voicemail_detected = True + logger.info(f"{self}: VOICEMAIL detected") await self._gate_notifier.notify() # Close the classifier gate await self._voicemail_notifier.notify() # Clear buffered TTS frames # Interrupt the current pipeline to stop any ongoing processing await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM) - # Execute developer callback for custom voicemail handling + # Always start the callback timer immediately + # It will be cancelled and restarted if user starts/stops speaking + if not self._voicemail_callback_task: + logger.debug( + f"{self}: Starting voicemail callback timer ({self._voicemail_response_delay}s)" + ) + self._voicemail_callback_task = self.create_task(self._delayed_voicemail_callback()) + + else: + # Unexpected response - log warning but don't crash + logger.warning(f"{self}: Unexpected classification response: '{full_response}'") + + async def _delayed_voicemail_callback(self): + """Execute the voicemail callback after the configured delay. + + This method waits for the specified delay period, then triggers the + developer's voicemail callback. The timer can be cancelled and restarted + based on user speech patterns to ensure proper timing. + """ + try: + logger.debug( + f"{self}: Waiting {self._voicemail_response_delay}s before voicemail callback" + ) + await asyncio.sleep(self._voicemail_response_delay) + + logger.info(f"{self}: Executing voicemail callback") if self._on_voicemail_detected: try: await self._on_voicemail_detected(self) except Exception as e: logger.exception(f"{self}: Error in voicemail callback: {e}") - else: - # Unexpected response - log warning but don't crash - logger.warning(f"{self}: Unexpected classification response: '{full_response}'") + + except asyncio.CancelledError: + logger.debug(f"{self}: Voicemail callback timer was cancelled") + raise + finally: + self._voicemail_callback_task = None class VoicemailBuffer(FrameProcessor): @@ -432,6 +585,7 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo llm: LLMService, on_voicemail_detected: Optional[Callable[["VoicemailProcessor"], Awaitable[None]]] = None, system_prompt: Optional[str] = None, + voicemail_response_delay: float = 2.0, ): """Initialize the voicemail detector with classification and buffering components. @@ -445,9 +599,14 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo uses the default prompt optimized for outbound calling scenarios. Custom prompts should instruct the LLM to respond with exactly "CONVERSATION" or "VOICEMAIL" for proper detection functionality. + voicemail_response_delay: Delay in seconds after user stops speaking + before triggering the voicemail callback. This allows voicemail + responses to be played back after a short delay to ensure the response + occurs during the voicemail recording. Default is 2.0 seconds. """ self._classifier_llm = llm self._prompt = system_prompt if system_prompt is not None else self.DEFAULT_SYSTEM_PROMPT + self._voicemail_response_delay = voicemail_response_delay # Validate custom prompts to ensure they work with the detection logic if system_prompt is not None: @@ -472,11 +631,13 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo # Create the processor components self._classifier_gate = ClassifierGate(self._gate_notifier) + self._conversation_gate = ConversationGate(self._voicemail_notifier) self._voicemail_processor = VoicemailProcessor( gate_notifier=self._gate_notifier, conversation_notifier=self._conversation_notifier, voicemail_notifier=self._voicemail_notifier, on_voicemail_detected=on_voicemail_detected, + voicemail_response_delay=voicemail_response_delay, ) self._voicemail_buffer = VoicemailBuffer( self._conversation_notifier, self._voicemail_notifier @@ -484,8 +645,8 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo # Initialize the parallel pipeline with conversation and classifier branches super().__init__( - # Conversation branch: empty pipeline for normal frame flow - [], + # Conversation branch: gate to blocks after voicemail detection + [self._conversation_gate], # Classification branch: gate -> context -> LLM -> processor -> context [ self._classifier_gate, From 5a07b30c7a490c5aa5f6d2589766039068c180f3 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 9 Aug 2025 08:10:31 -0400 Subject: [PATCH 08/22] Class name changes, add TTSStarted/StoppedFrame to the TTSBuffer --- .../foundational/44-voicemail-detection.py | 4 +- .../utils/voicemail/voicemail_detector.py | 42 +++++++++++-------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/examples/foundational/44-voicemail-detection.py b/examples/foundational/44-voicemail-detection.py index 64e78465f..7be12e642 100644 --- a/examples/foundational/44-voicemail-detection.py +++ b/examples/foundational/44-voicemail-detection.py @@ -97,11 +97,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): [ transport.input(), stt, - voicemail.detector(), # Voicemail detection + voicemail.detector(), # Voicemail detection — between STT and User context aggregator context_aggregator.user(), llm, tts, - voicemail.buffer(), # TTS buffering + voicemail.buffer(), # TTS buffering — Immediately after the TTS service transport.output(), context_aggregator.assistant(), ] diff --git a/src/pipecat/utils/voicemail/voicemail_detector.py b/src/pipecat/utils/voicemail/voicemail_detector.py index 4a6b070c2..c37f9a4ae 100644 --- a/src/pipecat/utils/voicemail/voicemail_detector.py +++ b/src/pipecat/utils/voicemail/voicemail_detector.py @@ -31,6 +31,8 @@ from pipecat.frames.frames import ( StartInterruptionFrame, StopInterruptionFrame, TTSAudioRawFrame, + TTSStartedFrame, + TTSStoppedFrame, TTSTextFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, @@ -109,7 +111,7 @@ class ClassifierGate(FrameProcessor): async def _wait_for_notification(self): """Wait for classification decision notification and close the gate. - This method blocks until the VoicemailProcessor makes a classification + This method blocks until the ClassificationProcessor makes a classification decision and signals through the notifier. Once notified, the gate closes permanently to stop further classification processing. """ @@ -207,7 +209,7 @@ class ConversationGate(FrameProcessor): raise -class VoicemailProcessor(FrameProcessor): +class ClassificationProcessor(FrameProcessor): """Processor that handles LLM classification responses and triggers callbacks. This processor aggregates LLM text tokens into complete responses and analyzes @@ -230,7 +232,9 @@ class VoicemailProcessor(FrameProcessor): gate_notifier: BaseNotifier, conversation_notifier: BaseNotifier, voicemail_notifier: BaseNotifier, - on_voicemail_detected: Optional[Callable[["VoicemailProcessor"], Awaitable[None]]] = None, + on_voicemail_detected: Optional[ + Callable[["ClassificationProcessor"], Awaitable[None]] + ] = None, voicemail_response_delay: float, ): """Initialize the voicemail processor. @@ -238,9 +242,9 @@ class VoicemailProcessor(FrameProcessor): Args: gate_notifier: Notifier to signal the ClassifierGate about classification decisions so it can close and stop processing. - conversation_notifier: Notifier to signal the VoicemailBuffer to release + conversation_notifier: Notifier to signal the TTSBuffer to release all buffered TTS frames for normal conversation flow. - voicemail_notifier: Notifier to signal the VoicemailBuffer to clear + voicemail_notifier: Notifier to signal the TTSBuffer to clear buffered TTS frames since voicemail was detected. on_voicemail_detected: Optional callback function called when voicemail is detected. The callback receives this processor instance and can @@ -392,7 +396,7 @@ class VoicemailProcessor(FrameProcessor): self._voicemail_callback_task = None -class VoicemailBuffer(FrameProcessor): +class TTSBuffer(FrameProcessor): """Buffers TTS frames until voicemail classification decision is made. This processor holds TTS output frames in a buffer while the voicemail @@ -455,7 +459,9 @@ class VoicemailBuffer(FrameProcessor): await self.push_frame(frame, direction) # Core buffering logic: hold TTS frames, pass everything else through - elif self._buffering_active and isinstance(frame, (TTSTextFrame, TTSAudioRawFrame)): + elif self._buffering_active and isinstance( + frame, (TTSStartedFrame, TTSStoppedFrame, TTSTextFrame, TTSAudioRawFrame) + ): # Buffer TTS frames while waiting for classification decision self._frame_buffer.append((frame, direction)) else: @@ -583,22 +589,24 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo self, *, llm: LLMService, - on_voicemail_detected: Optional[Callable[["VoicemailProcessor"], Awaitable[None]]] = None, system_prompt: Optional[str] = None, - voicemail_response_delay: float = 2.0, + on_voicemail_detected: Optional[ + Callable[["ClassificationProcessor"], Awaitable[None]] + ] = None, + voicemail_response_delay: Optional[float] = 2.0, ): """Initialize the voicemail detector with classification and buffering components. Args: llm: LLM service used for voicemail vs conversation classification. Should be fast and reliable for real-time classification. - on_voicemail_detected: Optional callback function invoked when voicemail - is detected. Receives the VoicemailProcessor instance which can be - used to push frames (like custom voicemail greetings). system_prompt: Optional custom system prompt for classification. If None, uses the default prompt optimized for outbound calling scenarios. Custom prompts should instruct the LLM to respond with exactly "CONVERSATION" or "VOICEMAIL" for proper detection functionality. + on_voicemail_detected: Optional callback function invoked when voicemail + is detected. Receives the ClassificationProcessor instance which can be + used to push frames (like custom voicemail greetings). voicemail_response_delay: Delay in seconds after user stops speaking before triggering the voicemail callback. This allows voicemail responses to be played back after a short delay to ensure the response @@ -632,16 +640,14 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo # Create the processor components self._classifier_gate = ClassifierGate(self._gate_notifier) self._conversation_gate = ConversationGate(self._voicemail_notifier) - self._voicemail_processor = VoicemailProcessor( + self._voicemail_processor = ClassificationProcessor( gate_notifier=self._gate_notifier, conversation_notifier=self._conversation_notifier, voicemail_notifier=self._voicemail_notifier, on_voicemail_detected=on_voicemail_detected, voicemail_response_delay=voicemail_response_delay, ) - self._voicemail_buffer = VoicemailBuffer( - self._conversation_notifier, self._voicemail_notifier - ) + self._voicemail_buffer = TTSBuffer(self._conversation_notifier, self._voicemail_notifier) # Initialize the parallel pipeline with conversation and classifier branches super().__init__( @@ -688,13 +694,13 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo """ return self - def buffer(self) -> VoicemailBuffer: + def buffer(self) -> TTSBuffer: """Get the buffer processor for placement after TTS in the main pipeline. This should be placed after the TTS service and before the transport output to enable TTS frame buffering during classification. Returns: - The VoicemailBuffer processor instance. + The TTSBuffer processor instance. """ return self._voicemail_buffer From ce579d426683e989fd6f8423383d55eb6c251e1a Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 9 Aug 2025 08:17:53 -0400 Subject: [PATCH 09/22] Make on_voicemail_detected callback required, cleanup logging --- .../foundational/44-voicemail-detection.py | 2 +- .../utils/voicemail/voicemail_detector.py | 42 ++++++------------- 2 files changed, 13 insertions(+), 31 deletions(-) diff --git a/examples/foundational/44-voicemail-detection.py b/examples/foundational/44-voicemail-detection.py index 7be12e642..358116aed 100644 --- a/examples/foundational/44-voicemail-detection.py +++ b/examples/foundational/44-voicemail-detection.py @@ -59,7 +59,7 @@ async def handle_voicemail(processor): processor: The VoicemailProcessor instance. processor.push_frame() is available to push frames. """ - logger.info("Voicemail detected! Playing greeting...") + logger.info("Voicemail detected! Leaving a message...") # Push frames using standard Pipecat pattern await processor.push_frame( diff --git a/src/pipecat/utils/voicemail/voicemail_detector.py b/src/pipecat/utils/voicemail/voicemail_detector.py index c37f9a4ae..af7692918 100644 --- a/src/pipecat/utils/voicemail/voicemail_detector.py +++ b/src/pipecat/utils/voicemail/voicemail_detector.py @@ -120,7 +120,6 @@ class ClassifierGate(FrameProcessor): if self._gate_opened: self._gate_opened = False - logger.debug(f"{self}: Gate closed - classification complete") except asyncio.CancelledError: logger.debug(f"{self}: Gate task was cancelled") raise @@ -200,7 +199,6 @@ class ConversationGate(FrameProcessor): if self._gate_opened: self._gate_opened = False - logger.debug(f"{self}: Conversation gate closed - voicemail detected") except asyncio.CancelledError: logger.debug(f"{self}: Conversation gate task was cancelled") raise @@ -288,7 +286,6 @@ class ClassificationProcessor(FrameProcessor): # Begin aggregating a new LLM response self._processing_response = True self._response_buffer = "" - logger.debug(f"{self}: Starting LLM response aggregation") elif isinstance(frame, LLMFullResponseEndFrame): # Complete response received - make classification decision @@ -300,21 +297,16 @@ class ClassificationProcessor(FrameProcessor): elif isinstance(frame, LLMTextFrame) and self._processing_response: # Accumulate text tokens from the streaming LLM response self._response_buffer += frame.text - logger.trace(f"{self}: Buffer: '{self._response_buffer}'") elif isinstance(frame, UserStartedSpeakingFrame): # User started speaking - cancel voicemail callback timer if self._voicemail_callback_task: - logger.debug(f"{self}: User started speaking, cancelling voicemail callback") await self.cancel_task(self._voicemail_callback_task) self._voicemail_callback_task = None elif isinstance(frame, UserStoppedSpeakingFrame): # User stopped speaking - restart voicemail callback timer if voicemail detected if self._voicemail_detected and not self._voicemail_callback_task: - logger.debug( - f"{self}: User stopped speaking, restarting voicemail callback timer ({self._voicemail_response_delay}s)" - ) self._voicemail_callback_task = self.create_task(self._delayed_voicemail_callback()) else: @@ -333,11 +325,10 @@ class ClassificationProcessor(FrameProcessor): full_response: The complete aggregated response text from the LLM. """ if self._decision_made: - logger.debug(f"{self}: Decision already made, ignoring response") return response = full_response.upper() - logger.info(f"{self}: Classifying response: '{full_response}'") + logger.debug(f"{self}: Classifying response: '{full_response}'") if "CONVERSATION" in response: # Human answered - continue normal conversation flow @@ -360,14 +351,11 @@ class ClassificationProcessor(FrameProcessor): # Always start the callback timer immediately # It will be cancelled and restarted if user starts/stops speaking if not self._voicemail_callback_task: - logger.debug( - f"{self}: Starting voicemail callback timer ({self._voicemail_response_delay}s)" - ) self._voicemail_callback_task = self.create_task(self._delayed_voicemail_callback()) else: - # Unexpected response - log warning but don't crash - logger.warning(f"{self}: Unexpected classification response: '{full_response}'") + # This can happen if the LLM is interrupted before completing the response + logger.debug(f"{self}: No classification found: '{full_response}'") async def _delayed_voicemail_callback(self): """Execute the voicemail callback after the configured delay. @@ -377,20 +365,16 @@ class ClassificationProcessor(FrameProcessor): based on user speech patterns to ensure proper timing. """ try: - logger.debug( - f"{self}: Waiting {self._voicemail_response_delay}s before voicemail callback" - ) await asyncio.sleep(self._voicemail_response_delay) - logger.info(f"{self}: Executing voicemail callback") if self._on_voicemail_detected: try: + logger.debug(f"{self}: Executing voicemail callback") await self._on_voicemail_detected(self) except Exception as e: logger.exception(f"{self}: Error in voicemail callback: {e}") except asyncio.CancelledError: - logger.debug(f"{self}: Voicemail callback timer was cancelled") raise finally: self._voicemail_callback_task = None @@ -589,21 +573,15 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo self, *, llm: LLMService, + on_voicemail_detected: Callable[["ClassificationProcessor"], Awaitable[None]], + voicemail_response_delay: float = 2.0, system_prompt: Optional[str] = None, - on_voicemail_detected: Optional[ - Callable[["ClassificationProcessor"], Awaitable[None]] - ] = None, - voicemail_response_delay: Optional[float] = 2.0, ): """Initialize the voicemail detector with classification and buffering components. Args: llm: LLM service used for voicemail vs conversation classification. Should be fast and reliable for real-time classification. - system_prompt: Optional custom system prompt for classification. If None, - uses the default prompt optimized for outbound calling scenarios. - Custom prompts should instruct the LLM to respond with exactly - "CONVERSATION" or "VOICEMAIL" for proper detection functionality. on_voicemail_detected: Optional callback function invoked when voicemail is detected. Receives the ClassificationProcessor instance which can be used to push frames (like custom voicemail greetings). @@ -611,6 +589,10 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo before triggering the voicemail callback. This allows voicemail responses to be played back after a short delay to ensure the response occurs during the voicemail recording. Default is 2.0 seconds. + system_prompt: Optional custom system prompt for classification. If None, + uses the default prompt optimized for outbound calling scenarios. + Custom prompts should instruct the LLM to respond with exactly + "CONVERSATION" or "VOICEMAIL" for proper detection functionality. """ self._classifier_llm = llm self._prompt = system_prompt if system_prompt is not None else self.DEFAULT_SYSTEM_PROMPT @@ -640,7 +622,7 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo # Create the processor components self._classifier_gate = ClassifierGate(self._gate_notifier) self._conversation_gate = ConversationGate(self._voicemail_notifier) - self._voicemail_processor = ClassificationProcessor( + self._classification_processor = ClassificationProcessor( gate_notifier=self._gate_notifier, conversation_notifier=self._conversation_notifier, voicemail_notifier=self._voicemail_notifier, @@ -658,7 +640,7 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo self._classifier_gate, self._context_aggregator.user(), self._classifier_llm, - self._voicemail_processor, + self._classification_processor, self._context_aggregator.assistant(), ], ) From ac30083b459928bc9feaf93fe66bd1df8e150524 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 9 Aug 2025 09:24:20 -0400 Subject: [PATCH 10/22] Add CHANGELOG entry --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8e536d09..5e808f828 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `pipecat.utils.voicemail`, a module for detecting voicemail vs. live + conversation, primarily intended for use in outbound calling scenarios. + - Added new frames to the `idle_timeout_frames` arg: `TranscriptionFrame`, `InterimTranscriptionFrame`, `UserStartedSpeakingFrame`, and `UserStoppedSpeakingFrame`. These additions serve as indicators of user From 1c1ee940749f974ad83734fa0c4853257043609a Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 9 Aug 2025 13:38:05 -0400 Subject: [PATCH 11/22] Add 44 to evals, update evals to support user speaking first --- .../foundational/44-voicemail-detection.py | 24 +++++--------- scripts/evals/eval.py | 32 ++++++++++++++++--- scripts/evals/run-release-evals.py | 25 +++++++++++++-- 3 files changed, 58 insertions(+), 23 deletions(-) diff --git a/examples/foundational/44-voicemail-detection.py b/examples/foundational/44-voicemail-detection.py index 358116aed..f2957309d 100644 --- a/examples/foundational/44-voicemail-detection.py +++ b/examples/foundational/44-voicemail-detection.py @@ -4,15 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import os from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import EndFrame, EndTaskFrame, TTSSpeakFrame -from pipecat.observers.loggers.debug_log_observer import DebugLogObserver +from pipecat.frames.frames import EndTaskFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -63,9 +61,14 @@ async def handle_voicemail(processor): # Push frames using standard Pipecat pattern await processor.push_frame( - TTSSpeakFrame("This is Mattie. Call me back when you can!"), + TTSSpeakFrame( + "Hello, this is Jamie calling about your appointment. Please call me back at 555-0123 when you get this." + ) ) - await processor.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM) + + # NOTE: A common pattern is to end pipeline after the voicemail is left. + # Uncomment the following line to end the pipeline after leaving the voicemail. + # await processor.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM) async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): @@ -114,22 +117,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): enable_usage_metrics=True, ), idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - observers=[ - DebugLogObserver( - frame_types={ - EndFrame: None, - EndTaskFrame: None, - } - ), - ], ) @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") - # # Kick off the conversation. - # messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - # await task.queue_frames([context_aggregator.user().get_context_frame()]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py index b91f27c6e..bbc170803 100644 --- a/scripts/evals/eval.py +++ b/scripts/evals/eval.py @@ -89,7 +89,13 @@ class EvalRunner: async def assert_eval_false(self): await self._queue.put(False) - async def run_eval(self, example_file: str, prompt: EvalPrompt, eval: Optional[str] = None): + async def run_eval( + self, + example_file: str, + prompt: EvalPrompt, + eval: Optional[str] = None, + user_speaks_first: bool = False, + ): if not re.match(self._pattern, example_file): return @@ -106,7 +112,9 @@ class EvalRunner: try: tasks = [ asyncio.create_task(run_example_pipeline(script_path)), - asyncio.create_task(run_eval_pipeline(self, example_file, prompt, eval)), + asyncio.create_task( + run_eval_pipeline(self, example_file, prompt, eval, user_speaks_first) + ), ] _, pending = await asyncio.wait(tasks, timeout=EVAL_TIMEOUT_SECS) if pending: @@ -196,6 +204,7 @@ async def run_eval_pipeline( example_file: str, prompt: EvalPrompt, eval: Optional[str], + user_speaks_first: bool = False, ): logger.info(f"Starting eval bot") @@ -225,7 +234,7 @@ async def run_eval_pipeline( tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + voice_id="97f4b8fb-f2fe-444b-bb9a-c109783a857a", # Nathan ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) @@ -260,12 +269,17 @@ async def run_eval_pipeline( # See if we need to include an eval prompt. eval_prompt = "" if eval: - eval_prompt = f"The answer is correct if the user says [{eval}]." + if user_speaks_first: + eval_prompt = f"After the user responds, evaluate if their response is appropriate for the context and matches: [{eval}]." + system_prompt = f"You will start the conversation by saying: '{prompt}'. {eval_prompt} Then call the eval function with your assessment." + else: + eval_prompt = f"The answer is correct if the user says [{eval}]." + system_prompt = f"You are an LLM eval, be extremly brief. Your goal is to only ask one question: {example_prompt}. Call the eval function only if the user answers the question and check if the answer is correct (words as numbers are valid). {eval_prompt}" messages = [ { "role": "system", - "content": f"You are an LLM eval, be extremly brief. Your goal is to only ask one question: {example_prompt}. Call the eval function only if the user answers the question and check if the answer is correct (words as numbers are valid). {eval_prompt}", + "content": system_prompt, }, ] @@ -313,6 +327,14 @@ async def run_eval_pipeline( ) await audio_buffer.start_recording() + # Default behavior is for the bot to speak first + # If the eval bot speaks first, we append the prompt to the messages + if user_speaks_first: + messages.append( + {"role": "user", "content": f"Start by saying this exactly: '{prompt}'"} + ) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 49acc432a..f30b71193 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -24,6 +24,8 @@ ASSETS_DIR = SCRIPT_DIR / "assets" FOUNDATIONAL_DIR = SCRIPT_DIR.parent.parent / "examples" / "foundational" +# User speaks first +USER_SPEAKS_FIRST = True # Math PROMPT_SIMPLE_MATH = "A simple math addition." @@ -46,6 +48,12 @@ EVAL_SWITCH_LANGUAGE = "Check if the user is now talking in Spanish." PROMPT_VISION = ("What do you see?", Image.open(ASSETS_DIR / "cat.jpg")) EVAL_VISION = "A cat description." +# Voicemail +PROMPT_VOICEMAIL = "Please leave a message after the beep." +EVAL_VOICEMAIL = "Assess the conversation and determine if it is a voicemail." +PROMPT_CONVERSATION = "Hello, this is Mark." +EVAL_CONVERSATION = "A start of a conversation, not a voicemail." + TESTS_07 = [ # 07 series ("07-interruptible.py", PROMPT_SIMPLE_MATH, None), @@ -157,6 +165,11 @@ TESTS_43 = [ ("43a-heygen-video-service.py", PROMPT_SIMPLE_MATH, None), ] +TESTS_44 = [ + ("44-voicemail-detection.py", PROMPT_VOICEMAIL, EVAL_VOICEMAIL, USER_SPEAKS_FIRST), + ("44-voicemail-detection.py", PROMPT_CONVERSATION, EVAL_CONVERSATION, USER_SPEAKS_FIRST), +] + TESTS = [ *TESTS_07, *TESTS_12, @@ -168,6 +181,7 @@ TESTS = [ *TESTS_27, *TESTS_40, *TESTS_43, + *TESTS_44, ] @@ -189,8 +203,15 @@ async def main(args: argparse.Namespace): log_level=log_level, ) - for test, prompt, eval in TESTS: - await runner.run_eval(test, prompt, eval) + # Parse test config: (test, prompt, eval) or (test, prompt, eval, user_speaks_first) + for test_config in TESTS: + if len(test_config) == 3: + test, prompt, eval = test_config + user_speaks_first = False + else: + test, prompt, eval, user_speaks_first = test_config + + await runner.run_eval(test, prompt, eval, user_speaks_first) runner.print_results() From 446bb5cddfbb877e9af48e11f2cc3ca261e950b7 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 12 Aug 2025 15:35:33 -0400 Subject: [PATCH 12/22] Refactor callback to event --- .../foundational/44-voicemail-detection.py | 38 +++---- .../utils/voicemail/voicemail_detector.py | 99 ++++++++++--------- 2 files changed, 69 insertions(+), 68 deletions(-) diff --git a/examples/foundational/44-voicemail-detection.py b/examples/foundational/44-voicemail-detection.py index f2957309d..84e24cf2f 100644 --- a/examples/foundational/44-voicemail-detection.py +++ b/examples/foundational/44-voicemail-detection.py @@ -50,27 +50,6 @@ transport_params = { } -async def handle_voicemail(processor): - """Called when a voicemail is detected. - - Args: - processor: The VoicemailProcessor instance. processor.push_frame() is - available to push frames. - """ - logger.info("Voicemail detected! Leaving a message...") - - # Push frames using standard Pipecat pattern - await processor.push_frame( - TTSSpeakFrame( - "Hello, this is Jamie calling about your appointment. Please call me back at 555-0123 when you get this." - ) - ) - - # NOTE: A common pattern is to end pipeline after the voicemail is left. - # Uncomment the following line to end the pipeline after leaving the voicemail. - # await processor.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM) - - async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") @@ -84,7 +63,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) classifier_llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - voicemail = VoicemailDetector(llm=classifier_llm, on_voicemail_detected=handle_voicemail) + voicemail = VoicemailDetector(llm=classifier_llm) messages = [ { @@ -128,6 +107,21 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Client disconnected") await task.cancel() + @voicemail.event_handler("on_voicemail_detected") + async def handle_voicemail(processor): + logger.info("Voicemail detected! Leaving a message...") + + # Push frames using standard Pipecat pattern + await processor.push_frame( + TTSSpeakFrame( + "Hello, this is Jamie calling about your appointment. Please call me back at 555-0123 when you get this." + ) + ) + + # NOTE: A common pattern is to end pipeline after the voicemail is left. + # Uncomment the following line to end the pipeline after leaving the voicemail. + # await processor.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM) + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) await runner.run(task) diff --git a/src/pipecat/utils/voicemail/voicemail_detector.py b/src/pipecat/utils/voicemail/voicemail_detector.py index af7692918..c8ed6efd6 100644 --- a/src/pipecat/utils/voicemail/voicemail_detector.py +++ b/src/pipecat/utils/voicemail/voicemail_detector.py @@ -13,7 +13,7 @@ a bot needs to determine if a human answered or if the call went to voicemail. """ import asyncio -from typing import Awaitable, Callable, List, Optional +from typing import List, Optional from loguru import logger @@ -208,7 +208,7 @@ class ConversationGate(FrameProcessor): class ClassificationProcessor(FrameProcessor): - """Processor that handles LLM classification responses and triggers callbacks. + """Processor that handles LLM classification responses and triggers events. This processor aggregates LLM text tokens into complete responses and analyzes them to determine if the call reached a voicemail system or a live person. @@ -218,9 +218,9 @@ class ClassificationProcessor(FrameProcessor): The processor expects responses containing either "CONVERSATION" (indicating a human answered) or "VOICEMAIL" (indicating an automated system). Once a - decision is made, it triggers the appropriate notifications and callbacks. + decision is made, it triggers the appropriate notifications and event handlers. - For voicemail detection, the callback timer starts immediately and is cancelled + For voicemail detection, the event handler timer starts immediately and is cancelled and restarted based on user speech patterns to ensure proper timing. """ @@ -230,9 +230,6 @@ class ClassificationProcessor(FrameProcessor): gate_notifier: BaseNotifier, conversation_notifier: BaseNotifier, voicemail_notifier: BaseNotifier, - on_voicemail_detected: Optional[ - Callable[["ClassificationProcessor"], Awaitable[None]] - ] = None, voicemail_response_delay: float, ): """Initialize the voicemail processor. @@ -244,20 +241,19 @@ class ClassificationProcessor(FrameProcessor): all buffered TTS frames for normal conversation flow. voicemail_notifier: Notifier to signal the TTSBuffer to clear buffered TTS frames since voicemail was detected. - on_voicemail_detected: Optional callback function called when voicemail - is detected. The callback receives this processor instance and can - use it to push custom frames (like voicemail greetings). voicemail_response_delay: Delay in seconds after user stops speaking - before triggering the voicemail callback. This ensures the voicemail + before triggering the voicemail event handler. This ensures the voicemail greeting or user message is complete before responding. """ super().__init__() self._gate_notifier = gate_notifier self._conversation_notifier = conversation_notifier self._voicemail_notifier = voicemail_notifier - self._on_voicemail_detected = on_voicemail_detected self._voicemail_response_delay = voicemail_response_delay + # Register the voicemail detected event + self._register_event_handler("on_voicemail_detected") + # Aggregation state for collecting complete LLM responses self._processing_response = False self._response_buffer = "" @@ -265,7 +261,7 @@ class ClassificationProcessor(FrameProcessor): # Voicemail timing state self._voicemail_detected = False - self._voicemail_callback_task: Optional[asyncio.Task] = None + self._voicemail_handler_task: Optional[asyncio.Task] = None async def process_frame(self, frame: Frame, direction: FrameDirection): """Process frames and handle LLM classification responses. @@ -299,15 +295,15 @@ class ClassificationProcessor(FrameProcessor): self._response_buffer += frame.text elif isinstance(frame, UserStartedSpeakingFrame): - # User started speaking - cancel voicemail callback timer - if self._voicemail_callback_task: - await self.cancel_task(self._voicemail_callback_task) - self._voicemail_callback_task = None + # User started speaking - cancel voicemail handler timer + if self._voicemail_handler_task: + await self.cancel_task(self._voicemail_handler_task) + self._voicemail_handler_task = None elif isinstance(frame, UserStoppedSpeakingFrame): - # User stopped speaking - restart voicemail callback timer if voicemail detected - if self._voicemail_detected and not self._voicemail_callback_task: - self._voicemail_callback_task = self.create_task(self._delayed_voicemail_callback()) + # User stopped speaking - restart voicemail handler timer if voicemail detected + if self._voicemail_detected and not self._voicemail_handler_task: + self._voicemail_handler_task = self.create_task(self._delayed_voicemail_handler()) else: # Pass all non-LLM frames through @@ -348,36 +344,35 @@ class ClassificationProcessor(FrameProcessor): # Interrupt the current pipeline to stop any ongoing processing await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM) - # Always start the callback timer immediately + # Always start the handler timer immediately # It will be cancelled and restarted if user starts/stops speaking - if not self._voicemail_callback_task: - self._voicemail_callback_task = self.create_task(self._delayed_voicemail_callback()) + if not self._voicemail_handler_task: + self._voicemail_handler_task = self.create_task(self._delayed_voicemail_handler()) else: # This can happen if the LLM is interrupted before completing the response logger.debug(f"{self}: No classification found: '{full_response}'") - async def _delayed_voicemail_callback(self): - """Execute the voicemail callback after the configured delay. + async def _delayed_voicemail_handler(self): + """Execute the voicemail event handler after the configured delay. This method waits for the specified delay period, then triggers the - developer's voicemail callback. The timer can be cancelled and restarted + developer's voicemail event handler. The timer can be cancelled and restarted based on user speech patterns to ensure proper timing. """ try: await asyncio.sleep(self._voicemail_response_delay) - if self._on_voicemail_detected: - try: - logger.debug(f"{self}: Executing voicemail callback") - await self._on_voicemail_detected(self) - except Exception as e: - logger.exception(f"{self}: Error in voicemail callback: {e}") + try: + logger.debug(f"{self}: Triggering voicemail detected event") + await self._call_event_handler("on_voicemail_detected") + except Exception as e: + logger.exception(f"{self}: Error in voicemail event handler: {e}") except asyncio.CancelledError: raise finally: - self._voicemail_callback_task = None + self._voicemail_handler_task = None class TTSBuffer(FrameProcessor): @@ -482,7 +477,7 @@ class TTSBuffer(FrameProcessor): When voicemail is detected, all buffered TTS frames are discarded since they were intended for human conversation and are not appropriate - for voicemail systems. The developer callback will handle voicemail- + for voicemail systems. The developer event handlers will handle voicemail- specific audio output. """ try: @@ -520,20 +515,17 @@ class VoicemailDetector(ParallelPipeline): Once a decision is made, the appropriate action is taken: - CONVERSATION: Continue normal bot dialogue - - VOICEMAIL: Trigger developer callback for custom voicemail handling + - VOICEMAIL: Trigger developer event handler for custom voicemail handling Example:: classification_llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + detector = VoicemailDetector(llm=classification_llm) + @detector.event_handler("on_voicemail_detected") async def handle_voicemail(processor): await processor.push_frame(TTSSpeakFrame("Please leave a message.")) - detector = VoicemailDetector( - llm=classification_llm, - on_voicemail_detected=handle_voicemail - ) - pipeline = Pipeline([ transport.input(), stt, @@ -545,6 +537,11 @@ class VoicemailDetector(ParallelPipeline): transport.output(), context_aggregator.assistant(), ]) + + Events: + on_voicemail_detected: Triggered when voicemail is detected after the configured + delay. The event handler receives one argument: the ClassificationProcessor + instance which can be used to push frames. """ DEFAULT_SYSTEM_PROMPT = """You are a voicemail detection classifier for an OUTBOUND calling system. A bot has called a phone number and you need to determine if a human answered or if the call went to voicemail based on the provided text. @@ -573,7 +570,6 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo self, *, llm: LLMService, - on_voicemail_detected: Callable[["ClassificationProcessor"], Awaitable[None]], voicemail_response_delay: float = 2.0, system_prompt: Optional[str] = None, ): @@ -582,11 +578,8 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo Args: llm: LLM service used for voicemail vs conversation classification. Should be fast and reliable for real-time classification. - on_voicemail_detected: Optional callback function invoked when voicemail - is detected. Receives the ClassificationProcessor instance which can be - used to push frames (like custom voicemail greetings). voicemail_response_delay: Delay in seconds after user stops speaking - before triggering the voicemail callback. This allows voicemail + before triggering the voicemail event handler. This allows voicemail responses to be played back after a short delay to ensure the response occurs during the voicemail recording. Default is 2.0 seconds. system_prompt: Optional custom system prompt for classification. If None, @@ -626,7 +619,6 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo gate_notifier=self._gate_notifier, conversation_notifier=self._conversation_notifier, voicemail_notifier=self._voicemail_notifier, - on_voicemail_detected=on_voicemail_detected, voicemail_response_delay=voicemail_response_delay, ) self._voicemail_buffer = TTSBuffer(self._conversation_notifier, self._voicemail_notifier) @@ -645,6 +637,9 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo ], ) + # Register the voicemail detected event after super().__init__() + self._register_event_handler("on_voicemail_detected") + def _validate_prompt(self, prompt: str) -> None: """Validate custom prompt contains required response format instructions. @@ -686,3 +681,15 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo The TTSBuffer processor instance. """ return self._voicemail_buffer + + def add_event_handler(self, event_name: str, handler): + """Add an event handler for voicemail detection events. + + Args: + event_name: The name of the event to handle. + handler: The function to call when the event occurs. + """ + if event_name == "on_voicemail_detected": + self._classification_processor.add_event_handler(event_name, handler) + else: + super().add_event_handler(event_name, handler) From b30af3e1557deed97131660920142598c528a0ea Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 18 Aug 2025 14:57:16 -0400 Subject: [PATCH 13/22] Tests specify USER_SPEAKS_FIRST or BOT_SPEAKS_FIRST --- scripts/evals/run-release-evals.py | 171 +++++++++++++++-------------- 1 file changed, 89 insertions(+), 82 deletions(-) diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index f30b71193..938200464 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -24,8 +24,9 @@ ASSETS_DIR = SCRIPT_DIR / "assets" FOUNDATIONAL_DIR = SCRIPT_DIR.parent.parent / "examples" / "foundational" -# User speaks first +# Speaking order constants USER_SPEAKS_FIRST = True +BOT_SPEAKS_FIRST = False # Math PROMPT_SIMPLE_MATH = "A simple math addition." @@ -56,113 +57,123 @@ EVAL_CONVERSATION = "A start of a conversation, not a voicemail." TESTS_07 = [ # 07 series - ("07-interruptible.py", PROMPT_SIMPLE_MATH, None), - ("07-interruptible-cartesia-http.py", PROMPT_SIMPLE_MATH, None), - ("07a-interruptible-speechmatics.py", PROMPT_SIMPLE_MATH, None), - ("07aa-interruptible-soniox.py", PROMPT_SIMPLE_MATH, None), - ("07ab-interruptible-inworld-http.py", PROMPT_SIMPLE_MATH, None), - ("07ac-interruptible-asyncai.py", PROMPT_SIMPLE_MATH, None), - ("07ac-interruptible-asyncai-http.py", PROMPT_SIMPLE_MATH, None), - ("07b-interruptible-langchain.py", PROMPT_SIMPLE_MATH, None), - ("07c-interruptible-deepgram.py", PROMPT_SIMPLE_MATH, None), - ("07d-interruptible-elevenlabs.py", PROMPT_SIMPLE_MATH, None), - ("07d-interruptible-elevenlabs-http.py", PROMPT_SIMPLE_MATH, None), - ("07e-interruptible-playht.py", PROMPT_SIMPLE_MATH, None), - ("07e-interruptible-playht-http.py", PROMPT_SIMPLE_MATH, None), - ("07f-interruptible-azure.py", PROMPT_SIMPLE_MATH, None), - ("07g-interruptible-openai.py", PROMPT_SIMPLE_MATH, None), - ("07h-interruptible-openpipe.py", PROMPT_SIMPLE_MATH, None), - ("07j-interruptible-gladia.py", PROMPT_SIMPLE_MATH, None), - ("07k-interruptible-lmnt.py", PROMPT_SIMPLE_MATH, None), - ("07l-interruptible-groq.py", PROMPT_SIMPLE_MATH, None), - ("07m-interruptible-aws.py", PROMPT_SIMPLE_MATH, None), - ("07n-interruptible-gemini.py", PROMPT_SIMPLE_MATH, None), - ("07n-interruptible-google.py", PROMPT_SIMPLE_MATH, None), - ("07o-interruptible-assemblyai.py", PROMPT_SIMPLE_MATH, None), - ("07q-interruptible-rime.py", PROMPT_SIMPLE_MATH, None), - ("07q-interruptible-rime-http.py", PROMPT_SIMPLE_MATH, None), - ("07r-interruptible-riva-nim.py", PROMPT_SIMPLE_MATH, None), - ("07s-interruptible-google-audio-in.py", PROMPT_SIMPLE_MATH, None), - ("07t-interruptible-fish.py", PROMPT_SIMPLE_MATH, None), - ("07v-interruptible-neuphonic.py", PROMPT_SIMPLE_MATH, None), - ("07v-interruptible-neuphonic-http.py", PROMPT_SIMPLE_MATH, None), - ("07w-interruptible-fal.py", PROMPT_SIMPLE_MATH, None), - ("07y-interruptible-minimax.py", PROMPT_SIMPLE_MATH, None), - ("07z-interruptible-sarvam.py", PROMPT_SIMPLE_MATH, None), + ("07-interruptible.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07-interruptible-cartesia-http.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07a-interruptible-speechmatics.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07aa-interruptible-soniox.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07ab-interruptible-inworld-http.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07ac-interruptible-asyncai.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07ac-interruptible-asyncai-http.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07b-interruptible-langchain.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07c-interruptible-deepgram.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07d-interruptible-elevenlabs.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07d-interruptible-elevenlabs-http.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07e-interruptible-playht.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07e-interruptible-playht-http.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07f-interruptible-azure.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07g-interruptible-openai.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07h-interruptible-openpipe.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07j-interruptible-gladia.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07k-interruptible-lmnt.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07l-interruptible-groq.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07m-interruptible-aws.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07n-interruptible-gemini.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07n-interruptible-google.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07o-interruptible-assemblyai.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07q-interruptible-rime.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07q-interruptible-rime-http.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07r-interruptible-riva-nim.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07s-interruptible-google-audio-in.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07t-interruptible-fish.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07v-interruptible-neuphonic.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07v-interruptible-neuphonic-http.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07w-interruptible-fal.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07y-interruptible-minimax.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("07z-interruptible-sarvam.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), # Needs a local XTTS docker instance running. - # ("07i-interruptible-xtts.py", PROMPT_SIMPLE_MATH, None), + # ("07i-interruptible-xtts.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), # Needs a Krisp license. - # ("07p-interruptible-krisp.py", PROMPT_SIMPLE_MATH, None), + # ("07p-interruptible-krisp.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), # Needs GPU resources. - # ("07u-interruptible-ultravox.py", PROMPT_SIMPLE_MATH, None), + # ("07u-interruptible-ultravox.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), ] TESTS_12 = [ - ("12-describe-video.py", PROMPT_VISION, EVAL_VISION), - ("12a-describe-video-gemini-flash.py", PROMPT_VISION, EVAL_VISION), - ("12b-describe-video-gpt-4o.py", PROMPT_VISION, EVAL_VISION), - ("12c-describe-video-anthropic.py", PROMPT_VISION, EVAL_VISION), + ("12-describe-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), + ("12a-describe-video-gemini-flash.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), + ("12b-describe-video-gpt-4o.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), + ("12c-describe-video-anthropic.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), ] TESTS_14 = [ - ("14-function-calling.py", PROMPT_WEATHER, EVAL_WEATHER), - ("14a-function-calling-anthropic.py", PROMPT_WEATHER, EVAL_WEATHER), - ("14b-function-calling-anthropic-video.py", PROMPT_WEATHER, EVAL_WEATHER), - ("14d-function-calling-video.py", PROMPT_WEATHER, EVAL_WEATHER), - ("14e-function-calling-google.py", PROMPT_WEATHER, EVAL_WEATHER), - ("14f-function-calling-groq.py", PROMPT_WEATHER, EVAL_WEATHER), - ("14g-function-calling-grok.py", PROMPT_WEATHER, EVAL_WEATHER), - ("14h-function-calling-azure.py", PROMPT_WEATHER, EVAL_WEATHER), - ("14i-function-calling-fireworks.py", PROMPT_WEATHER, EVAL_WEATHER), - ("14j-function-calling-nim.py", PROMPT_WEATHER, EVAL_WEATHER), - ("14m-function-calling-openrouter.py", PROMPT_WEATHER, EVAL_WEATHER), - ("14n-function-calling-perplexity.py", PROMPT_WEATHER, EVAL_WEATHER), - ("14p-function-calling-gemini-vertex-ai.py", PROMPT_WEATHER, EVAL_WEATHER), - ("14q-function-calling-qwen.py", PROMPT_WEATHER, EVAL_WEATHER), - ("14r-function-calling-aws.py", PROMPT_WEATHER, EVAL_WEATHER), - ("14v-function-calling-openai.py", PROMPT_WEATHER, EVAL_WEATHER), - ("14w-function-calling-mistral.py", PROMPT_WEATHER, EVAL_WEATHER), + ("14-function-calling.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14a-function-calling-anthropic.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14b-function-calling-anthropic-video.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14d-function-calling-video.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14e-function-calling-google.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14f-function-calling-groq.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14g-function-calling-grok.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14h-function-calling-azure.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14i-function-calling-fireworks.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14j-function-calling-nim.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14m-function-calling-openrouter.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14n-function-calling-perplexity.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14p-function-calling-gemini-vertex-ai.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14q-function-calling-qwen.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14r-function-calling-aws.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14v-function-calling-openai.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14w-function-calling-mistral.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), # Currently not working. - # ("14c-function-calling-together.py", PROMPT_WEATHER, EVAL_WEATHER), - # ("14k-function-calling-cerebras.py", PROMPT_WEATHER, EVAL_WEATHER), - # ("14l-function-calling-deepseek.py", PROMPT_WEATHER, EVAL_WEATHER), - # ("14o-function-calling-gemini-openai-format.py", PROMPT_WEATHER, EVAL_WEATHER), + # ("14c-function-calling-together.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + # ("14k-function-calling-cerebras.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + # ("14l-function-calling-deepseek.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + # ("14o-function-calling-gemini-openai-format.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ] TESTS_15 = [ - ("15a-switch-languages.py", PROMPT_SWITCH_LANGUAGE, EVAL_SWITCH_LANGUAGE), + ("15a-switch-languages.py", PROMPT_SWITCH_LANGUAGE, EVAL_SWITCH_LANGUAGE, BOT_SPEAKS_FIRST), ] TESTS_19 = [ - ("19-openai-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER), - ("19a-azure-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER), - ("19b-openai-realtime-beta-text.py", PROMPT_WEATHER, EVAL_WEATHER), + ("19-openai-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("19a-azure-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("19b-openai-realtime-beta-text.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ] TESTS_21 = [ - ("21a-tavus-video-service.py", PROMPT_SIMPLE_MATH, None), + ("21a-tavus-video-service.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), ] TESTS_26 = [ - ("26-gemini-multimodal-live.py", PROMPT_SIMPLE_MATH, None), - ("26a-gemini-multimodal-live-transcription.py", PROMPT_SIMPLE_MATH, None), - ("26b-gemini-multimodal-live-function-calling.py", PROMPT_WEATHER, EVAL_WEATHER), - ("26c-gemini-multimodal-live-video.py", PROMPT_SIMPLE_MATH, None), - ("26e-gemini-multimodal-google-search.py", PROMPT_ONLINE_SEARCH, EVAL_ONLINE_SEARCH), + ("26-gemini-multimodal-live.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ("26a-gemini-multimodal-live-transcription.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ( + "26b-gemini-multimodal-live-function-calling.py", + PROMPT_WEATHER, + EVAL_WEATHER, + BOT_SPEAKS_FIRST, + ), + ("26c-gemini-multimodal-live-video.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), + ( + "26e-gemini-multimodal-google-search.py", + PROMPT_ONLINE_SEARCH, + EVAL_ONLINE_SEARCH, + BOT_SPEAKS_FIRST, + ), # Currently not working. - # ("26d-gemini-multimodal-live-text.py", PROMPT_SIMPLE_MATH, None), + # ("26d-gemini-multimodal-live-text.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), ] TESTS_27 = [ - ("27-simli-layer.py", PROMPT_SIMPLE_MATH, None), + ("27-simli-layer.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), ] TESTS_40 = [ - ("40-aws-nova-sonic.py", PROMPT_SIMPLE_MATH, None), + ("40-aws-nova-sonic.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), ] TESTS_43 = [ - ("43a-heygen-video-service.py", PROMPT_SIMPLE_MATH, None), + ("43a-heygen-video-service.py", PROMPT_SIMPLE_MATH, None, BOT_SPEAKS_FIRST), ] TESTS_44 = [ @@ -203,13 +214,9 @@ async def main(args: argparse.Namespace): log_level=log_level, ) - # Parse test config: (test, prompt, eval) or (test, prompt, eval, user_speaks_first) + # Parse test config: (test, prompt, eval, user_speaks_first) for test_config in TESTS: - if len(test_config) == 3: - test, prompt, eval = test_config - user_speaks_first = False - else: - test, prompt, eval, user_speaks_first = test_config + test, prompt, eval, user_speaks_first = test_config await runner.run_eval(test, prompt, eval, user_speaks_first) From 40b5ef485d034dbee9777a8dca6f6fc625af041e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 18 Aug 2025 15:17:03 -0400 Subject: [PATCH 14/22] Add base NotifierGate class and ClassifierGate, ConversationGate subclasses --- .../utils/voicemail/voicemail_detector.py | 156 +++++++----------- 1 file changed, 62 insertions(+), 94 deletions(-) diff --git a/src/pipecat/utils/voicemail/voicemail_detector.py b/src/pipecat/utils/voicemail/voicemail_detector.py index c8ed6efd6..c1d3b723a 100644 --- a/src/pipecat/utils/voicemail/voicemail_detector.py +++ b/src/pipecat/utils/voicemail/voicemail_detector.py @@ -29,7 +29,9 @@ from pipecat.frames.frames import ( LLMTextFrame, StartFrame, StartInterruptionFrame, + StopFrame, StopInterruptionFrame, + StopTaskFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, @@ -45,33 +47,33 @@ from pipecat.sync.base_notifier import BaseNotifier from pipecat.sync.event_notifier import EventNotifier -class ClassifierGate(FrameProcessor): - """Gate processor that controls frame flow based on classification decisions. +class NotifierGate(FrameProcessor): + """Base gate processor that controls frame flow based on notifier signals. - The gate starts open to allow initial classification processing and closes - permanently once a classification decision is made (CONVERSATION or VOICEMAIL). - This ensures the classifier only runs until a definitive decision is reached, - preventing unnecessary LLM calls and maintaining system efficiency. + This base class provides common gate functionality for processors that need to + start open and close permanently when a notifier signals. Subclasses define + which frames are allowed through when the gate is closed. - The gate allows all frames to pass through while open, but once closed, only - allows system frames and user speaking frames to continue. Speaking frames - are needed for voicemail timing control. + The gate starts open to allow initial processing and closes permanently once + the notifier signals. This ensures controlled frame flow based on external + decisions or events. """ - def __init__(self, gate_notifier: BaseNotifier): - """Initialize the classifier gate. + def __init__(self, notifier: BaseNotifier, task_name: str = "gate"): + """Initialize the notifier gate. Args: - gate_notifier: Notifier that signals when a classification decision has - been made and the gate should close. + notifier: Notifier that signals when the gate should close. + task_name: Name for the notification waiting task (for debugging). """ super().__init__() - self._gate_notifier = gate_notifier + self._notifier = notifier + self._task_name = task_name self._gate_opened = True self._gate_task: Optional[asyncio.Task] = None async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process frames and control gate state based on classification decisions. + """Process frames and control gate state based on notifier signals. Args: frame: The frame to process. @@ -89,51 +91,78 @@ class ClassifierGate(FrameProcessor): await self.cancel_task(self._gate_task) self._gate_task = None - # Gate logic: open gate allows all frames, closed gate only allows specific system frames + # Gate logic: open gate allows all frames, closed gate filters frames if self._gate_opened: await self.push_frame(frame, direction) - elif not self._gate_opened and isinstance( + elif isinstance( frame, ( BotInterruptionFrame, + CancelFrame, + CancelTaskFrame, EndTaskFrame, EndFrame, - CancelTaskFrame, - CancelFrame, - UserStartedSpeakingFrame, - UserStoppedSpeakingFrame, StartInterruptionFrame, StopInterruptionFrame, + StopFrame, + StopTaskFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, ), ): await self.push_frame(frame, direction) async def _wait_for_notification(self): - """Wait for classification decision notification and close the gate. + """Wait for notifier signal and close the gate. - This method blocks until the ClassificationProcessor makes a classification - decision and signals through the notifier. Once notified, the gate - closes permanently to stop further classification processing. + This method blocks until the notifier signals, then closes the gate + permanently to change frame filtering behavior. """ try: - await self._gate_notifier.wait() + await self._notifier.wait() if self._gate_opened: self._gate_opened = False except asyncio.CancelledError: - logger.debug(f"{self}: Gate task was cancelled") + logger.debug(f"{self}: {self._task_name} task was cancelled") raise except Exception as e: - logger.exception(f"{self}: Error in gate task: {e}") + logger.exception(f"{self}: Error in {self._task_name} task: {e}") raise -class ConversationGate(FrameProcessor): +class ClassifierGate(NotifierGate): + """Gate processor that controls frame flow based on classification decisions. + + Inherits from NotifierGate and starts open to allow initial classification + processing. Closes permanently once a classification decision is made + (CONVERSATION or VOICEMAIL). This ensures the classifier only runs until a + definitive decision is reached, preventing unnecessary LLM calls and maintaining + system efficiency. + + When closed, only allows system frames and user speaking frames to continue. + Speaking frames are needed for voicemail timing control. + """ + + def __init__(self, gate_notifier: BaseNotifier): + """Initialize the classifier gate. + + Args: + gate_notifier: Notifier that signals when a classification decision has + been made and the gate should close. + """ + super().__init__(gate_notifier, task_name="classifier_gate") + + +class ConversationGate(NotifierGate): """Gate processor that blocks conversation flow when voicemail is detected. - This gate starts open to allow normal conversation processing but closes - permanently when voicemail is detected. This prevents the main conversation - LLM from processing additional input after voicemail classification. + Inherits from NotifierGate and starts open to allow normal conversation + processing. Closes permanently when voicemail is detected to prevent the + main conversation LLM from processing additional input after voicemail + classification. + + When closed, only allows system frames and user speaking frames to continue. """ def __init__(self, voicemail_notifier: BaseNotifier): @@ -143,68 +172,7 @@ class ConversationGate(FrameProcessor): voicemail_notifier: Notifier that signals when voicemail has been detected and the conversation should be blocked. """ - super().__init__() - self._voicemail_notifier = voicemail_notifier - self._gate_opened = True - self._voicemail_task: Optional[asyncio.Task] = None - - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process frames and control gate state based on voicemail detection. - - Args: - frame: The frame to process. - direction: The direction of frame flow in the pipeline. - """ - await super().process_frame(frame, direction) - - if isinstance(frame, StartFrame): - # Start the notification waiting task immediately - self._voicemail_task = self.create_task(self._wait_for_voicemail()) - - elif isinstance(frame, (EndFrame, CancelFrame)): - # Clean up the task when pipeline ends or is cancelled - if self._voicemail_task: - await self.cancel_task(self._voicemail_task) - self._voicemail_task = None - - # Gate logic: open gate allows all frames, closed gate blocks everything - if self._gate_opened: - await self.push_frame(frame, direction) - elif not self._gate_opened and isinstance( - frame, - ( - BotInterruptionFrame, - EndTaskFrame, - EndFrame, - CancelTaskFrame, - CancelFrame, - UserStartedSpeakingFrame, - UserStoppedSpeakingFrame, - StartInterruptionFrame, - StopInterruptionFrame, - ), - ): - # Only allow system frames and user speaking frames through when closed - await self.push_frame(frame, direction) - # When closed, don't push any frames (complete conversation blocking) - - async def _wait_for_voicemail(self): - """Wait for voicemail detection notification and close the gate. - - This method blocks until voicemail is detected, then closes the gate - permanently to prevent any further conversation processing. - """ - try: - await self._voicemail_notifier.wait() - - if self._gate_opened: - self._gate_opened = False - except asyncio.CancelledError: - logger.debug(f"{self}: Conversation gate task was cancelled") - raise - except Exception as e: - logger.exception(f"{self}: Error in conversation gate task: {e}") - raise + super().__init__(voicemail_notifier, task_name="conversation_gate") class ClassificationProcessor(FrameProcessor): From fbc907c3712fbe6c8016be8acf5bcd178c726505 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 18 Aug 2025 16:47:34 -0400 Subject: [PATCH 15/22] Change path to extensions --- CHANGELOG.md | 4 ++-- examples/foundational/44-voicemail-detection.py | 2 +- src/pipecat/{utils/voicemail => extensions}/__init__.py | 0 src/pipecat/extensions/voicemail/__init__.py | 0 .../{utils => extensions}/voicemail/voicemail_detector.py | 0 5 files changed, 3 insertions(+), 3 deletions(-) rename src/pipecat/{utils/voicemail => extensions}/__init__.py (100%) create mode 100644 src/pipecat/extensions/voicemail/__init__.py rename src/pipecat/{utils => extensions}/voicemail/voicemail_detector.py (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e808f828..25a5d887a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added `pipecat.utils.voicemail`, a module for detecting voicemail vs. live - conversation, primarily intended for use in outbound calling scenarios. +- Added `pipecat.extensions.voicemail`, a module for detecting voicemail vs. + live conversation, primarily intended for use in outbound calling scenarios. - Added new frames to the `idle_timeout_frames` arg: `TranscriptionFrame`, `InterimTranscriptionFrame`, `UserStartedSpeakingFrame`, and diff --git a/examples/foundational/44-voicemail-detection.py b/examples/foundational/44-voicemail-detection.py index 84e24cf2f..60eefa0c2 100644 --- a/examples/foundational/44-voicemail-detection.py +++ b/examples/foundational/44-voicemail-detection.py @@ -10,6 +10,7 @@ from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.extensions.voicemail.voicemail_detector import VoicemailDetector from pipecat.frames.frames import EndTaskFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -24,7 +25,6 @@ from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams from pipecat.transports.services.daily import DailyParams -from pipecat.utils.voicemail.voicemail_detector import VoicemailDetector load_dotenv(override=True) diff --git a/src/pipecat/utils/voicemail/__init__.py b/src/pipecat/extensions/__init__.py similarity index 100% rename from src/pipecat/utils/voicemail/__init__.py rename to src/pipecat/extensions/__init__.py diff --git a/src/pipecat/extensions/voicemail/__init__.py b/src/pipecat/extensions/voicemail/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/utils/voicemail/voicemail_detector.py b/src/pipecat/extensions/voicemail/voicemail_detector.py similarity index 100% rename from src/pipecat/utils/voicemail/voicemail_detector.py rename to src/pipecat/extensions/voicemail/voicemail_detector.py From f0dfab23e7bbd68c189d2e62ca1a444186588df8 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 21 Aug 2025 16:16:10 -0400 Subject: [PATCH 16/22] Cleanup --- .../foundational/44-voicemail-detection.py | 2 +- .../voicemail/voicemail_detector.py | 185 +++++++++--------- 2 files changed, 90 insertions(+), 97 deletions(-) diff --git a/examples/foundational/44-voicemail-detection.py b/examples/foundational/44-voicemail-detection.py index 60eefa0c2..f7019f937 100644 --- a/examples/foundational/44-voicemail-detection.py +++ b/examples/foundational/44-voicemail-detection.py @@ -83,7 +83,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): context_aggregator.user(), llm, tts, - voicemail.buffer(), # TTS buffering — Immediately after the TTS service + voicemail.gate(), # TTS buffering — Immediately after the TTS service transport.output(), context_aggregator.assistant(), ] diff --git a/src/pipecat/extensions/voicemail/voicemail_detector.py b/src/pipecat/extensions/voicemail/voicemail_detector.py index c1d3b723a..6c4ba221e 100644 --- a/src/pipecat/extensions/voicemail/voicemail_detector.py +++ b/src/pipecat/extensions/voicemail/voicemail_detector.py @@ -41,7 +41,7 @@ from pipecat.frames.frames import ( ) from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup from pipecat.services.llm_service import LLMService from pipecat.sync.base_notifier import BaseNotifier from pipecat.sync.event_notifier import EventNotifier @@ -72,6 +72,22 @@ class NotifierGate(FrameProcessor): self._gate_opened = True self._gate_task: Optional[asyncio.Task] = None + async def setup(self, setup: FrameProcessorSetup): + """Set up the processor with required components. + + Args: + setup: Configuration object containing setup parameters. + """ + await super().setup(setup) + self._gate_task = self.create_task(self._wait_for_notification()) + + async def cleanup(self): + """Clean up the processor resources.""" + await super().cleanup() + if self._gate_task: + await self.cancel_task(self._gate_task) + self._gate_task = None + async def process_frame(self, frame: Frame, direction: FrameDirection): """Process frames and control gate state based on notifier signals. @@ -81,16 +97,6 @@ class NotifierGate(FrameProcessor): """ await super().process_frame(frame, direction) - if isinstance(frame, StartFrame): - # Start the notification waiting task immediately - self._gate_task = self.create_task(self._wait_for_notification()) - - elif isinstance(frame, (EndFrame, CancelFrame)): - # Clean up the gate task when pipeline ends or is cancelled - if self._gate_task: - await self.cancel_task(self._gate_task) - self._gate_task = None - # Gate logic: open gate allows all frames, closed gate filters frames if self._gate_opened: await self.push_frame(frame, direction) @@ -118,17 +124,10 @@ class NotifierGate(FrameProcessor): This method blocks until the notifier signals, then closes the gate permanently to change frame filtering behavior. """ - try: - await self._notifier.wait() + await self._notifier.wait() - if self._gate_opened: - self._gate_opened = False - except asyncio.CancelledError: - logger.debug(f"{self}: {self._task_name} task was cancelled") - raise - except Exception as e: - logger.exception(f"{self}: Error in {self._task_name} task: {e}") - raise + if self._gate_opened: + self._gate_opened = False class ClassifierGate(NotifierGate): @@ -229,7 +228,25 @@ class ClassificationProcessor(FrameProcessor): # Voicemail timing state self._voicemail_detected = False - self._voicemail_handler_task: Optional[asyncio.Task] = None + self._voicemail_task: Optional[asyncio.Task] = None + self._voicemail_event = asyncio.Event() + self._voicemail_event.set() + + async def setup(self, setup: FrameProcessorSetup): + """Set up the processor with required components. + + Args: + setup: Configuration object containing setup parameters. + """ + await super().setup(setup) + self._voicemail_task = self.create_task(self._delayed_voicemail_handler()) + + async def cleanup(self): + """Clean up the processor resources.""" + await super().cleanup() + if self._voicemail_task: + await self.cancel_task(self._voicemail_task) + self._voicemail_task = None async def process_frame(self, frame: Frame, direction: FrameDirection): """Process frames and handle LLM classification responses. @@ -263,15 +280,12 @@ class ClassificationProcessor(FrameProcessor): self._response_buffer += frame.text elif isinstance(frame, UserStartedSpeakingFrame): - # User started speaking - cancel voicemail handler timer - if self._voicemail_handler_task: - await self.cancel_task(self._voicemail_handler_task) - self._voicemail_handler_task = None + # User started speaking - set the voicemail event + self._voicemail_event.set() elif isinstance(frame, UserStoppedSpeakingFrame): - # User stopped speaking - restart voicemail handler timer if voicemail detected - if self._voicemail_detected and not self._voicemail_handler_task: - self._voicemail_handler_task = self.create_task(self._delayed_voicemail_handler()) + # User stopped speaking - clear the voicemail event + self._voicemail_event.clear() else: # Pass all non-LLM frames through @@ -312,10 +326,8 @@ class ClassificationProcessor(FrameProcessor): # Interrupt the current pipeline to stop any ongoing processing await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM) - # Always start the handler timer immediately - # It will be cancelled and restarted if user starts/stops speaking - if not self._voicemail_handler_task: - self._voicemail_handler_task = self.create_task(self._delayed_voicemail_handler()) + # Set the voicemail event to trigger the voicemail handler + self._voicemail_event.clear() else: # This can happen if the LLM is interrupted before completing the response @@ -328,22 +340,18 @@ class ClassificationProcessor(FrameProcessor): developer's voicemail event handler. The timer can be cancelled and restarted based on user speech patterns to ensure proper timing. """ - try: - await asyncio.sleep(self._voicemail_response_delay) - + while True: try: - logger.debug(f"{self}: Triggering voicemail detected event") + await asyncio.wait_for( + self._voicemail_event.wait(), timeout=self._voicemail_response_delay + ) + await asyncio.sleep(0.1) + except asyncio.TimeoutError: await self._call_event_handler("on_voicemail_detected") - except Exception as e: - logger.exception(f"{self}: Error in voicemail event handler: {e}") - - except asyncio.CancelledError: - raise - finally: - self._voicemail_handler_task = None + break -class TTSBuffer(FrameProcessor): +class TTSGate(FrameProcessor): """Buffers TTS frames until voicemail classification decision is made. This processor holds TTS output frames in a buffer while the voicemail @@ -376,6 +384,27 @@ class TTSBuffer(FrameProcessor): self._conversation_task: Optional[asyncio.Task] = None self._voicemail_task: Optional[asyncio.Task] = None + async def setup(self, setup: FrameProcessorSetup): + """Set up the processor with required components. + + Args: + setup: Configuration object containing setup parameters. + """ + await super().setup(setup) + + self._conversation_task = self.create_task(self._wait_for_conversation()) + self._voicemail_task = self.create_task(self._wait_for_voicemail()) + + async def cleanup(self): + """Clean up the processor resources.""" + await super().cleanup() + if self._conversation_task: + await self.cancel_task(self._conversation_task) + self._conversation_task = None + if self._voicemail_task: + await self.cancel_task(self._voicemail_task) + self._voicemail_task = None + async def process_frame(self, frame: Frame, direction: FrameDirection): """Process frames and handle buffering logic based on frame type. @@ -389,24 +418,8 @@ class TTSBuffer(FrameProcessor): """ await super().process_frame(frame, direction) - if isinstance(frame, StartFrame): - # Start notification waiting tasks for both conversation and voicemail - self._conversation_task = self.create_task(self._wait_for_conversation()) - self._voicemail_task = self.create_task(self._wait_for_voicemail()) - await self.push_frame(frame, direction) - - elif isinstance(frame, (EndFrame, CancelFrame)): - # Clean up notification tasks when pipeline ends - if self._conversation_task: - await self.cancel_task(self._conversation_task) - self._conversation_task = None - if self._voicemail_task: - await self.cancel_task(self._voicemail_task) - self._voicemail_task = None - await self.push_frame(frame, direction) - # Core buffering logic: hold TTS frames, pass everything else through - elif self._buffering_active and isinstance( + if self._buffering_active and isinstance( frame, (TTSStartedFrame, TTSStoppedFrame, TTSTextFrame, TTSAudioRawFrame) ): # Buffer TTS frames while waiting for classification decision @@ -422,23 +435,13 @@ class TTSBuffer(FrameProcessor): in order to continue normal dialogue flow. This allows the bot to respond naturally to the human caller. """ - try: - await self._conversation_notifier.wait() + await self._conversation_notifier.wait() - # Release all buffered frames in original order - self._buffering_active = False - for frame, direction in self._frame_buffer: - await self.push_frame(frame, direction) - self._frame_buffer.clear() - - # Cancel the voicemail task since decision is final - if self._voicemail_task: - await self.cancel_task(self._voicemail_task) - self._voicemail_task = None - - except asyncio.CancelledError: - logger.debug(f"{self}: Conversation task was cancelled") - raise + # Release all buffered frames in original order + self._buffering_active = False + for frame, direction in self._frame_buffer: + await self.push_frame(frame, direction) + self._frame_buffer.clear() async def _wait_for_voicemail(self): """Wait for voicemail detection notification and clear buffered frames. @@ -448,21 +451,11 @@ class TTSBuffer(FrameProcessor): for voicemail systems. The developer event handlers will handle voicemail- specific audio output. """ - try: - await self._voicemail_notifier.wait() + await self._voicemail_notifier.wait() - # Clear buffered frames without playing them - self._buffering_active = False - self._frame_buffer.clear() - - # Cancel the conversation task since decision is final - if self._conversation_task: - await self.cancel_task(self._conversation_task) - self._conversation_task = None - - except asyncio.CancelledError: - logger.debug(f"{self}: Voicemail task was cancelled") - raise + # Clear buffered frames without playing them + self._buffering_active = False + self._frame_buffer.clear() class VoicemailDetector(ParallelPipeline): @@ -501,7 +494,7 @@ class VoicemailDetector(ParallelPipeline): context_aggregator.user(), llm, tts, - detector.buffer(), # TTS buffering + detector.gate(), # TTS buffering transport.output(), context_aggregator.assistant(), ]) @@ -589,7 +582,7 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo voicemail_notifier=self._voicemail_notifier, voicemail_response_delay=voicemail_response_delay, ) - self._voicemail_buffer = TTSBuffer(self._conversation_notifier, self._voicemail_notifier) + self._voicemail_gate = TTSGate(self._conversation_notifier, self._voicemail_notifier) # Initialize the parallel pipeline with conversation and classifier branches super().__init__( @@ -639,7 +632,7 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo """ return self - def buffer(self) -> TTSBuffer: + def gate(self) -> TTSGate: """Get the buffer processor for placement after TTS in the main pipeline. This should be placed after the TTS service and before the transport @@ -648,7 +641,7 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo Returns: The TTSBuffer processor instance. """ - return self._voicemail_buffer + return self._voicemail_gate def add_event_handler(self, event_name: str, handler): """Add an event handler for voicemail detection events. From bd401e8d6fc06c20ef19d6be31686cf73e576afc Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 21 Aug 2025 17:27:29 -0400 Subject: [PATCH 17/22] Rename TTSBuffer to TTSGate --- .../foundational/44-voicemail-detection.py | 2 +- .../voicemail/voicemail_detector.py | 67 +++++++++---------- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/examples/foundational/44-voicemail-detection.py b/examples/foundational/44-voicemail-detection.py index f7019f937..1834346ba 100644 --- a/examples/foundational/44-voicemail-detection.py +++ b/examples/foundational/44-voicemail-detection.py @@ -83,7 +83,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): context_aggregator.user(), llm, tts, - voicemail.gate(), # TTS buffering — Immediately after the TTS service + voicemail.gate(), # TTS gating — Immediately after the TTS service transport.output(), context_aggregator.assistant(), ] diff --git a/src/pipecat/extensions/voicemail/voicemail_detector.py b/src/pipecat/extensions/voicemail/voicemail_detector.py index 6c4ba221e..47a2e5c46 100644 --- a/src/pipecat/extensions/voicemail/voicemail_detector.py +++ b/src/pipecat/extensions/voicemail/voicemail_detector.py @@ -27,7 +27,6 @@ from pipecat.frames.frames import ( LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMTextFrame, - StartFrame, StartInterruptionFrame, StopFrame, StopInterruptionFrame, @@ -204,10 +203,10 @@ class ClassificationProcessor(FrameProcessor): Args: gate_notifier: Notifier to signal the ClassifierGate about classification decisions so it can close and stop processing. - conversation_notifier: Notifier to signal the TTSBuffer to release - all buffered TTS frames for normal conversation flow. - voicemail_notifier: Notifier to signal the TTSBuffer to clear - buffered TTS frames since voicemail was detected. + conversation_notifier: Notifier to signal the TTSGate to release + all gated TTS frames for normal conversation flow. + voicemail_notifier: Notifier to signal the TTSGate to clear + gated TTS frames since voicemail was detected. voicemail_response_delay: Delay in seconds after user stops speaking before triggering the voicemail event handler. This ensures the voicemail greeting or user message is complete before responding. @@ -352,35 +351,35 @@ class ClassificationProcessor(FrameProcessor): class TTSGate(FrameProcessor): - """Buffers TTS frames until voicemail classification decision is made. + """Gates TTS frames until voicemail classification decision is made. - This processor holds TTS output frames in a buffer while the voicemail + This processor holds TTS output frames in a gate while the voicemail classification is in progress. This prevents audio from being played to the caller before determining if they're human or a voicemail system. - The buffer operates in two modes based on the classification result: + The gate operates in two modes based on the classification result: - - CONVERSATION: Releases all buffered frames to continue normal dialogue - - VOICEMAIL: Clears buffered frames since they're not needed for voicemail + - CONVERSATION: Opens the gate to release all held frames for normal dialogue + - VOICEMAIL: Clears held frames since they're not needed for voicemail - The buffering only applies to TTS-related frames (TTSTextFrame, TTSAudioRawFrame). + The gating only applies to TTS-related frames (TTSTextFrame, TTSAudioRawFrame). All other frames pass through immediately to maintain proper pipeline flow. """ def __init__(self, conversation_notifier: BaseNotifier, voicemail_notifier: BaseNotifier): - """Initialize the voicemail buffer. + """Initialize the TTS gate. Args: conversation_notifier: Notifier that signals when a conversation is - detected and buffered frames should be released for playback. + detected and gated frames should be released for playback. voicemail_notifier: Notifier that signals when voicemail is detected - and buffered frames should be cleared (not played). + and gated frames should be cleared (not played). """ super().__init__() self._conversation_notifier = conversation_notifier self._voicemail_notifier = voicemail_notifier self._frame_buffer: List[tuple[Frame, FrameDirection]] = [] - self._buffering_active = True + self._gating_active = True self._conversation_task: Optional[asyncio.Task] = None self._voicemail_task: Optional[asyncio.Task] = None @@ -406,10 +405,10 @@ class TTSGate(FrameProcessor): self._voicemail_task = None async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process frames and handle buffering logic based on frame type. + """Process frames and handle gating logic based on frame type. - TTS frames are buffered while classification is active. All other frames - pass through immediately. The buffering state is controlled by the + TTS frames are gated while classification is active. All other frames + pass through immediately. The gating state is controlled by the classification notifications. Args: @@ -418,43 +417,43 @@ class TTSGate(FrameProcessor): """ await super().process_frame(frame, direction) - # Core buffering logic: hold TTS frames, pass everything else through - if self._buffering_active and isinstance( + # Core gating logic: hold TTS frames, pass everything else through + if self._gating_active and isinstance( frame, (TTSStartedFrame, TTSStoppedFrame, TTSTextFrame, TTSAudioRawFrame) ): - # Buffer TTS frames while waiting for classification decision + # Gate TTS frames while waiting for classification decision self._frame_buffer.append((frame, direction)) else: # Pass through all non-TTS frames immediately await self.push_frame(frame, direction) async def _wait_for_conversation(self): - """Wait for conversation detection notification and release buffered frames. + """Wait for conversation detection notification and release gated frames. - When a conversation is detected, all buffered TTS frames are released + When a conversation is detected, all gated TTS frames are released in order to continue normal dialogue flow. This allows the bot to respond naturally to the human caller. """ await self._conversation_notifier.wait() - # Release all buffered frames in original order - self._buffering_active = False + # Release all gated frames in original order + self._gating_active = False for frame, direction in self._frame_buffer: await self.push_frame(frame, direction) self._frame_buffer.clear() async def _wait_for_voicemail(self): - """Wait for voicemail detection notification and clear buffered frames. + """Wait for voicemail detection notification and clear gated frames. - When voicemail is detected, all buffered TTS frames are discarded + When voicemail is detected, all gated TTS frames are discarded since they were intended for human conversation and are not appropriate for voicemail systems. The developer event handlers will handle voicemail- specific audio output. """ await self._voicemail_notifier.wait() - # Clear buffered frames without playing them - self._buffering_active = False + # Clear gated frames without playing them + self._gating_active = False self._frame_buffer.clear() @@ -472,7 +471,7 @@ class VoicemailDetector(ParallelPipeline): - Classification branch: Contains the LLM classifier and decision logic The system uses a gate mechanism to control when classification runs and - a buffering system to prevent TTS output until classification is complete. + a gating system to prevent TTS output until classification is complete. Once a decision is made, the appropriate action is taken: - CONVERSATION: Continue normal bot dialogue @@ -494,7 +493,7 @@ class VoicemailDetector(ParallelPipeline): context_aggregator.user(), llm, tts, - detector.gate(), # TTS buffering + detector.gate(), # TTS gating transport.output(), context_aggregator.assistant(), ]) @@ -633,13 +632,13 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo return self def gate(self) -> TTSGate: - """Get the buffer processor for placement after TTS in the main pipeline. + """Get the gate processor for placement after TTS in the main pipeline. This should be placed after the TTS service and before the transport - output to enable TTS frame buffering during classification. + output to enable TTS frame gating during classification. Returns: - The TTSBuffer processor instance. + The TTSGate processor instance. """ return self._voicemail_gate From 87ebbab7583ee39e9afa9e2a194b4a18740e7da5 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 21 Aug 2025 19:40:57 -0400 Subject: [PATCH 18/22] Only set/clear voicemail_event when voicemail is detected --- src/pipecat/extensions/voicemail/voicemail_detector.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pipecat/extensions/voicemail/voicemail_detector.py b/src/pipecat/extensions/voicemail/voicemail_detector.py index 47a2e5c46..52b7feeab 100644 --- a/src/pipecat/extensions/voicemail/voicemail_detector.py +++ b/src/pipecat/extensions/voicemail/voicemail_detector.py @@ -280,11 +280,13 @@ class ClassificationProcessor(FrameProcessor): elif isinstance(frame, UserStartedSpeakingFrame): # User started speaking - set the voicemail event - self._voicemail_event.set() + if self._voicemail_detected: + self._voicemail_event.set() elif isinstance(frame, UserStoppedSpeakingFrame): # User stopped speaking - clear the voicemail event - self._voicemail_event.clear() + if self._voicemail_detected: + self._voicemail_event.clear() else: # Pass all non-LLM frames through From 57028255ee2ee11e9c8c36eeb99dd0a49f149260 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 21 Aug 2025 19:47:51 -0400 Subject: [PATCH 19/22] Update changelog, mention text LLMs only --- CHANGELOG.md | 1 + src/pipecat/extensions/voicemail/voicemail_detector.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25a5d887a..62cd3813e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `pipecat.extensions.voicemail`, a module for detecting voicemail vs. live conversation, primarily intended for use in outbound calling scenarios. + The voicemail module is optimized for text LLMs only. - Added new frames to the `idle_timeout_frames` arg: `TranscriptionFrame`, `InterimTranscriptionFrame`, `UserStartedSpeakingFrame`, and diff --git a/src/pipecat/extensions/voicemail/voicemail_detector.py b/src/pipecat/extensions/voicemail/voicemail_detector.py index 52b7feeab..fb17f5028 100644 --- a/src/pipecat/extensions/voicemail/voicemail_detector.py +++ b/src/pipecat/extensions/voicemail/voicemail_detector.py @@ -10,6 +10,9 @@ This module provides voicemail detection capabilities using parallel pipeline processing to classify incoming calls as either voicemail messages or live conversations. It's specifically designed for outbound calling scenarios where a bot needs to determine if a human answered or if the call went to voicemail. + +Note: + The voicemail module is optimized for text LLMs only. """ import asyncio From 4d49210a7312507eb4d4ef7911313d0180325adb Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 22 Aug 2025 11:11:14 -0400 Subject: [PATCH 20/22] Rename system_prompt to custom_system_prompt; improve dev ex for classification prompt requirements --- .../voicemail/voicemail_detector.py | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/pipecat/extensions/voicemail/voicemail_detector.py b/src/pipecat/extensions/voicemail/voicemail_detector.py index fb17f5028..19f8cf870 100644 --- a/src/pipecat/extensions/voicemail/voicemail_detector.py +++ b/src/pipecat/extensions/voicemail/voicemail_detector.py @@ -503,13 +503,23 @@ class VoicemailDetector(ParallelPipeline): context_aggregator.assistant(), ]) + # For custom prompts, append the required response instruction: + custom_prompt = "Your custom classification logic here. " + VoicemailDetector.CLASSIFIER_RESPONSE_INSTRUCTION + Events: on_voicemail_detected: Triggered when voicemail is detected after the configured delay. The event handler receives one argument: the ClassificationProcessor instance which can be used to push frames. + + Constants: + CLASSIFIER_RESPONSE_INSTRUCTION: The exact text that must be included in custom + system prompts to ensure proper classification functionality. """ - DEFAULT_SYSTEM_PROMPT = """You are a voicemail detection classifier for an OUTBOUND calling system. A bot has called a phone number and you need to determine if a human answered or if the call went to voicemail based on the provided text. + CLASSIFIER_RESPONSE_INSTRUCTION = 'Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it\'s voicemail/recording.' + + DEFAULT_SYSTEM_PROMPT = ( + """You are a voicemail detection classifier for an OUTBOUND calling system. A bot has called a phone number and you need to determine if a human answered or if the call went to voicemail based on the provided text. HUMAN ANSWERED - LIVE CONVERSATION (respond "CONVERSATION"): - Personal greetings: "Hello?", "Hi", "Yeah?", "John speaking" @@ -529,14 +539,16 @@ VOICEMAIL SYSTEM (respond "VOICEMAIL"): - Carrier system messages: "mailbox is full", "has not been set up" - Business hours messages: "our office is currently closed" -Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's voicemail/recording.""" +""" + + CLASSIFIER_RESPONSE_INSTRUCTION + ) def __init__( self, *, llm: LLMService, voicemail_response_delay: float = 2.0, - system_prompt: Optional[str] = None, + custom_system_prompt: Optional[str] = None, ): """Initialize the voicemail detector with classification and buffering components. @@ -547,18 +559,20 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo before triggering the voicemail event handler. This allows voicemail responses to be played back after a short delay to ensure the response occurs during the voicemail recording. Default is 2.0 seconds. - system_prompt: Optional custom system prompt for classification. If None, + custom_system_prompt: Optional custom system prompt for classification. If None, uses the default prompt optimized for outbound calling scenarios. Custom prompts should instruct the LLM to respond with exactly "CONVERSATION" or "VOICEMAIL" for proper detection functionality. """ self._classifier_llm = llm - self._prompt = system_prompt if system_prompt is not None else self.DEFAULT_SYSTEM_PROMPT + self._prompt = ( + custom_system_prompt if custom_system_prompt is not None else self.DEFAULT_SYSTEM_PROMPT + ) self._voicemail_response_delay = voicemail_response_delay # Validate custom prompts to ensure they work with the detection logic - if system_prompt is not None: - self._validate_prompt(system_prompt) + if custom_system_prompt is not None: + self._validate_prompt(custom_system_prompt) # Set up the LLM context with the classification prompt self._messages = [ @@ -622,7 +636,8 @@ Respond with ONLY "CONVERSATION" if a person answered, or "VOICEMAIL" if it's vo logger.warning( "Custom system prompt should instruct the LLM to respond with exactly " '"CONVERSATION" or "VOICEMAIL" for proper detection functionality. ' - 'Example: "Respond with ONLY \\"CONVERSATION\\" if a person answered, or \\"VOICEMAIL\\" if it\'s voicemail/recording."' + f"Consider appending VoicemailDetector.CLASSIFIER_RESPONSE_INSTRUCTION to your prompt: " + f'"{self.CLASSIFIER_RESPONSE_INSTRUCTION}"' ) def detector(self) -> "VoicemailDetector": From 69c6a95b8a79d31449b0d1af549f5fd474275090 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 22 Aug 2025 11:24:24 -0400 Subject: [PATCH 21/22] Simplify frames in the NotifierGate --- .../voicemail/voicemail_detector.py | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/src/pipecat/extensions/voicemail/voicemail_detector.py b/src/pipecat/extensions/voicemail/voicemail_detector.py index 19f8cf870..7f6d6eb73 100644 --- a/src/pipecat/extensions/voicemail/voicemail_detector.py +++ b/src/pipecat/extensions/voicemail/voicemail_detector.py @@ -22,18 +22,13 @@ from loguru import logger from pipecat.frames.frames import ( BotInterruptionFrame, - CancelFrame, - CancelTaskFrame, EndFrame, - EndTaskFrame, Frame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMTextFrame, - StartInterruptionFrame, StopFrame, - StopInterruptionFrame, - StopTaskFrame, + SystemFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, @@ -104,19 +99,7 @@ class NotifierGate(FrameProcessor): await self.push_frame(frame, direction) elif isinstance( frame, - ( - BotInterruptionFrame, - CancelFrame, - CancelTaskFrame, - EndTaskFrame, - EndFrame, - StartInterruptionFrame, - StopInterruptionFrame, - StopFrame, - StopTaskFrame, - UserStartedSpeakingFrame, - UserStoppedSpeakingFrame, - ), + (SystemFrame, EndFrame, StopFrame), ): await self.push_frame(frame, direction) From 402661ae03df310483cf324e76a9486c11224b5d Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 22 Aug 2025 14:07:41 -0400 Subject: [PATCH 22/22] Prevent user speaking frames from entering the classifier branch after a conversation is detected --- .../voicemail/voicemail_detector.py | 55 ++++++++++++++++++- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/src/pipecat/extensions/voicemail/voicemail_detector.py b/src/pipecat/extensions/voicemail/voicemail_detector.py index 7f6d6eb73..12c429a3f 100644 --- a/src/pipecat/extensions/voicemail/voicemail_detector.py +++ b/src/pipecat/extensions/voicemail/voicemail_detector.py @@ -125,17 +125,66 @@ class ClassifierGate(NotifierGate): system efficiency. When closed, only allows system frames and user speaking frames to continue. - Speaking frames are needed for voicemail timing control. + Speaking frames are needed for voicemail timing control, but not for conversation. """ - def __init__(self, gate_notifier: BaseNotifier): + def __init__(self, gate_notifier: BaseNotifier, conversation_notifier: BaseNotifier): """Initialize the classifier gate. Args: gate_notifier: Notifier that signals when a classification decision has been made and the gate should close. + conversation_notifier: Notifier that signals when conversation is detected. """ super().__init__(gate_notifier, task_name="classifier_gate") + self._conversation_notifier = conversation_notifier + self._conversation_detected = False + self._conversation_task: Optional[asyncio.Task] = None + + async def setup(self, setup: FrameProcessorSetup): + """Set up the processor with required components. + + Args: + setup: Configuration object containing setup parameters. + """ + await super().setup(setup) + self._conversation_task = self.create_task(self._wait_for_conversation()) + + async def cleanup(self): + """Clean up the processor resources.""" + await super().cleanup() + if self._conversation_task: + await self.cancel_task(self._conversation_task) + self._conversation_task = None + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames and control gate state based on notifier signals. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ + await FrameProcessor.process_frame(self, frame, direction) + + # Gate logic: open gate allows all frames, closed gate filters frames + if self._gate_opened: + await self.push_frame(frame, direction) + elif isinstance(frame, (UserStartedSpeakingFrame, UserStoppedSpeakingFrame)): + # Only allow speaking frames if conversation was NOT detected (i.e., voicemail case) + # This prevents the UserContextAggregator from issuing a warning about no aggregation + # to push. + if not self._conversation_detected: + await self.push_frame(frame, direction) + elif isinstance(frame, (SystemFrame, EndFrame, StopFrame)): + # Always allow system frames through + # This includes the UserStartedSpeakingFrame and UserStoppedSpeakingFrame + # which are used to detect voicemail timing. + await self.push_frame(frame, direction) + + async def _wait_for_conversation(self): + """Wait for conversation detection notification and mark conversation detected.""" + await self._conversation_notifier.wait() + self._conversation_detected = True class ConversationGate(NotifierGate): @@ -575,7 +624,7 @@ VOICEMAIL SYSTEM (respond "VOICEMAIL"): self._voicemail_notifier = EventNotifier() # Signals voicemail detected # Create the processor components - self._classifier_gate = ClassifierGate(self._gate_notifier) + self._classifier_gate = ClassifierGate(self._gate_notifier, self._conversation_notifier) self._conversation_gate = ConversationGate(self._voicemail_notifier) self._classification_processor = ClassificationProcessor( gate_notifier=self._gate_notifier,