wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add UserIdleController, deprecate UserIdleProcessor

Browse Source
This commit is contained in:
Mark Backman
2026-01-16 17:09:11 -05:00
parent 1e8516e91d
commit 2e8e574ea5
7 changed files with 524 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/pipecat/processors/aggregators/llm_response_universal.py
View File

@@ -62,6 +62,7 @@ from pipecat.processors.aggregators.llm_context import (
NotGiven,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.turns.user_idle_controller import UserIdleController
from pipecat.turns.user_mute import BaseUserMuteStrategy
from pipecat.turns.user_start import BaseUserTurnStartStrategy, UserTurnStartedParams
from pipecat.turns.user_stop import BaseUserTurnStopStrategy, UserTurnStoppedParams
@@ -80,11 +81,16 @@ class LLMUserAggregatorParams:
user_mute_strategies: List of user mute strategies.
user_turn_stop_timeout: Time in seconds to wait before considering the
user's turn finished.
user_idle_timeout: Optional timeout in seconds for detecting user idle state.
If set, the aggregator will emit an `on_user_idle` event when the user
has been idle (not speaking) for this duration. Set to None to disable
idle detection.
"""
user_turn_strategies: Optional[UserTurnStrategies] = None
user_mute_strategies: List[BaseUserMuteStrategy] = field(default_factory=list)
user_turn_stop_timeout: float = 5.0
user_idle_timeout: Optional[float] = None
@dataclass
@@ -291,11 +297,12 @@ class LLMUserAggregator(LLMContextAggregator):
- on_user_turn_started: Called when the user turn starts
- on_user_turn_stopped: Called when the user turn ends
- on_user_turn_stop_timeout: Called when no user turn stop strategy triggers
- on_user_idle: Called when the user has been idle for the configured timeout
Example::
@aggregator.event_handler("on_user_turn_started")
async def on_user_turn_started(aggregator, strategy: BaseUserTurnStartStrategy]):
async def on_user_turn_started(aggregator, strategy: BaseUserTurnStartStrategy):
...
@aggregator.event_handler("on_user_turn_stopped")
@@ -306,6 +313,10 @@ class LLMUserAggregator(LLMContextAggregator):
async def on_user_turn_stop_timeout(aggregator):
...
@aggregator.event_handler("on_user_idle")
async def on_user_idle(aggregator):
...
"""
def __init__(
@@ -328,6 +339,7 @@ class LLMUserAggregator(LLMContextAggregator):
self._register_event_handler("on_user_turn_started")
self._register_event_handler("on_user_turn_stopped")
self._register_event_handler("on_user_turn_stop_timeout")
self._register_event_handler("on_user_idle")
user_turn_strategies = self._params.user_turn_strategies or UserTurnStrategies()
@@ -350,6 +362,14 @@ class LLMUserAggregator(LLMContextAggregator):
"on_user_turn_stop_timeout", self._on_user_turn_stop_timeout
)
# Optional user idle controller
self._user_idle_controller: Optional[UserIdleController] = None
if self._params.user_idle_timeout:
self._user_idle_controller = UserIdleController(
user_idle_timeout=self._params.user_idle_timeout
)
self._user_idle_controller.add_event_handler("on_user_idle", self._on_user_idle)
async def cleanup(self):
"""Clean up processor resources."""
await super().cleanup()
@@ -405,6 +425,9 @@ class LLMUserAggregator(LLMContextAggregator):
await self._user_turn_controller.process_frame(frame)
if self._user_idle_controller:
await self._user_idle_controller.process_frame(frame)
async def push_aggregation(self) -> str:
"""Push the current aggregation."""
if len(self._aggregation) == 0:
@@ -420,6 +443,9 @@ class LLMUserAggregator(LLMContextAggregator):
async def _start(self, frame: StartFrame):
await self._user_turn_controller.setup(self.task_manager)
if self._user_idle_controller:
await self._user_idle_controller.setup(self.task_manager)
for s in self._params.user_mute_strategies:
await s.setup(self.task_manager)
@@ -432,6 +458,9 @@ class LLMUserAggregator(LLMContextAggregator):
async def _cleanup(self):
await self._user_turn_controller.cleanup()
if self._user_idle_controller:
await self._user_idle_controller.cleanup()
for s in self._params.user_mute_strategies:
await s.cleanup()
@@ -565,6 +594,9 @@ class LLMUserAggregator(LLMContextAggregator):
async def _on_user_turn_stop_timeout(self, controller):
await self._call_event_handler("on_user_turn_stop_timeout")
async def _on_user_idle(self, controller):
await self._call_event_handler("on_user_idle")
class LLMAssistantAggregator(LLMContextAggregator):
"""Assistant LLM aggregator that processes bot responses and function calls.

13
src/pipecat/processors/user_idle_processor.py
View File

@@ -8,6 +8,7 @@
import asyncio
import inspect
import warnings
from typing import Awaitable, Callable, Union
from pipecat.frames.frames import (
@@ -26,6 +27,10 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class UserIdleProcessor(FrameProcessor):
"""Monitors user inactivity and triggers callbacks after timeout periods.
.. deprecated::
UserIdleProcessor is deprecated in 0.0.100 and will be removed in a future version.
Use LLMUserAggregator with user_idle_timeout parameter instead.
This processor tracks user activity and triggers configurable callbacks when
users become idle. It starts monitoring only after the first conversation
activity and supports both basic and retry-based callback patterns.
@@ -70,6 +75,14 @@ class UserIdleProcessor(FrameProcessor):
**kwargs: Additional arguments passed to FrameProcessor.
"""
super().__init__(**kwargs)
warnings.warn(
"UserIdleProcessor is deprecated in 0.0.100 and will be removed in a "
"future version. Use LLMUserAggregator with user_idle_timeout parameter "
"instead.",
DeprecationWarning,
)
self._callback = self._wrap_callback(callback)
self._timeout = timeout
self._retry_count = 0

173
src/pipecat/turns/user_idle_controller.py Normal file
View File

@@ -0,0 +1,173 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""This module defines a controller for managing user idle detection."""
import asyncio
from typing import Optional
from pipecat.frames.frames import (
BotSpeakingFrame,
Frame,
FunctionCallResultFrame,
FunctionCallsStartedFrame,
UserSpeakingFrame,
UserStartedSpeakingFrame,
)
from pipecat.utils.asyncio.task_manager import BaseTaskManager
from pipecat.utils.base_object import BaseObject
class UserIdleController(BaseObject):
"""Controller for managing user idle detection.
This class monitors user activity and triggers an event when the user has been
idle (not speaking) for a configured timeout period. It only starts monitoring
after the first conversation activity and does not trigger while the bot is
speaking or function calls are in progress.
The controller tracks activity using continuous frames (UserSpeakingFrame and
BotSpeakingFrame) which are emitted repeatedly while speaking is happening, and
state-based tracking for function calls (FunctionCallsStartedFrame and
FunctionCallResultFrame) which are only sent at start and end.
Event handlers available:
- on_user_idle: Emitted when the user has been idle for the timeout period.
Example::
@controller.event_handler("on_user_idle")
async def on_user_idle(controller):
# Handle user idle - send reminder, prompt, etc.
...
"""
def __init__(
self,
*,
user_idle_timeout: float,
):
"""Initialize the user idle controller.
Args:
user_idle_timeout: Timeout in seconds before considering the user idle.
"""
super().__init__()
self._user_idle_timeout = user_idle_timeout
self._task_manager: Optional[BaseTaskManager] = None
self._conversation_started = False
self._function_call_in_progress = False
self.user_idle_event = asyncio.Event()
self.user_idle_task: Optional[asyncio.Task] = None
self._register_event_handler("on_user_idle", sync=True)
@property
def task_manager(self) -> BaseTaskManager:
"""Returns the configured task manager."""
if not self._task_manager:
raise RuntimeError(f"{self} user idle controller was not properly setup")
return self._task_manager
async def setup(self, task_manager: BaseTaskManager):
"""Initialize the controller with the given task manager.
Args:
task_manager: The task manager to be associated with this instance.
"""
self._task_manager = task_manager
if not self.user_idle_task:
self.user_idle_task = self.task_manager.create_task(
self.user_idle_task_handler(),
f"{self}::user_idle_task_handler",
)
async def cleanup(self):
"""Cleanup the controller."""
await super().cleanup()
if self.user_idle_task:
await self.task_manager.cancel_task(self.user_idle_task)
self.user_idle_task = None
async def process_frame(self, frame: Frame):
"""Process an incoming frame to track user activity state.
Args:
frame: The frame to be processed.
"""
# Start monitoring on first conversation activity
if not self._conversation_started:
if isinstance(frame, (UserStartedSpeakingFrame, BotSpeakingFrame)):
self._conversation_started = True
self.user_idle_event.set()
else:
return
# Reset idle timer on continuous activity frames
if isinstance(frame, (UserSpeakingFrame, BotSpeakingFrame)):
await self._handle_activity(frame)
# Track function call state (start/end frames, not continuous)
elif isinstance(frame, FunctionCallsStartedFrame):
await self._handle_function_calls_started(frame)
elif isinstance(frame, FunctionCallResultFrame):
await self._handle_function_call_result(frame)
async def _handle_activity(self, _: UserSpeakingFrame | BotSpeakingFrame):
"""Handle continuous activity frames that should reset the idle timer.
These frames are emitted continuously while the user or bot is speaking,
so we simply reset the timer whenever we receive them.
Args:
frame: The activity frame to process.
"""
self.user_idle_event.set()
async def _handle_function_calls_started(self, _: FunctionCallsStartedFrame):
"""Handle function calls started event.
Function calls can take longer than the timeout, so we track their state
to prevent idle callbacks while they're in progress.
Args:
frame: The FunctionCallsStartedFrame to process.
"""
self._function_call_in_progress = True
self.user_idle_event.set()
async def _handle_function_call_result(self, _: FunctionCallResultFrame):
"""Handle function call result event.
Args:
frame: The FunctionCallResultFrame to process.
"""
self._function_call_in_progress = False
self.user_idle_event.set()
async def user_idle_task_handler(self):
"""Monitors for idle timeout and triggers events.
Runs in a loop until cancelled. The idle timer is reset whenever activity
frames are received (UserSpeakingFrame or BotSpeakingFrame). Function calls
are tracked via state since they only send start/end frames. If no activity
is detected for the configured timeout period and no function call is in
progress, the on_user_idle event is triggered.
"""
while True:
try:
await asyncio.wait_for(self.user_idle_event.wait(), timeout=self._user_idle_timeout)
self.user_idle_event.clear()
except asyncio.TimeoutError:
# Only trigger if conversation has started and no function call is in progress
if self._conversation_started and not self._function_call_in_progress:
await self._call_event_handler("on_user_idle")

30
src/pipecat/turns/user_turn_processor.py
View File

@@ -19,6 +19,7 @@ from pipecat.frames.frames import (
UserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.turns.user_idle_controller import UserIdleController
from pipecat.turns.user_start import BaseUserTurnStartStrategy, UserTurnStartedParams
from pipecat.turns.user_stop import BaseUserTurnStopStrategy, UserTurnStoppedParams
from pipecat.turns.user_turn_controller import UserTurnController
@@ -38,6 +39,7 @@ class UserTurnProcessor(FrameProcessor):
- on_user_turn_started: Emitted when a user turn starts.
- on_user_turn_stopped: Emitted when a user turn stops.
- on_user_turn_stop_timeout: Emitted if no stop strategy triggers before timeout.
- on_user_idle: Emitted when the user has been idle for the configured timeout.
Example::
@@ -53,6 +55,10 @@ class UserTurnProcessor(FrameProcessor):
async def on_user_turn_stop_timeout(processor):
...
@processor.event_handler("on_user_idle")
async def on_user_idle(processor):
...
"""
def __init__(
@@ -60,6 +66,7 @@ class UserTurnProcessor(FrameProcessor):
*,
user_turn_strategies: Optional[UserTurnStrategies] = None,
user_turn_stop_timeout: float = 5.0,
user_idle_timeout: Optional[float] = None,
**kwargs,
):
"""Initialize the user turn processor.
@@ -68,6 +75,10 @@ class UserTurnProcessor(FrameProcessor):
user_turn_strategies: Configured strategies for starting and stopping user turns.
user_turn_stop_timeout: Timeout in seconds to automatically stop a user turn
if no activity is detected.
user_idle_timeout: Optional timeout in seconds for detecting user idle state.
If set, the processor will emit an `on_user_idle` event when the user
has been idle (not speaking) for this duration. Set to None to disable
idle detection.
**kwargs: Additional keyword arguments.
"""
super().__init__(**kwargs)
@@ -75,6 +86,7 @@ class UserTurnProcessor(FrameProcessor):
self._register_event_handler("on_user_turn_started")
self._register_event_handler("on_user_turn_stopped")
self._register_event_handler("on_user_turn_stop_timeout")
self._register_event_handler("on_user_idle")
self._user_turn_controller = UserTurnController(
user_turn_strategies=user_turn_strategies or UserTurnStrategies(),
@@ -92,6 +104,12 @@ class UserTurnProcessor(FrameProcessor):
"on_user_turn_stop_timeout", self._on_user_turn_stop_timeout
)
# Optional user idle controller
self._user_idle_controller: Optional[UserIdleController] = None
if user_idle_timeout:
self._user_idle_controller = UserIdleController(user_idle_timeout=user_idle_timeout)
self._user_idle_controller.add_event_handler("on_user_idle", self._on_user_idle)
async def cleanup(self):
"""Clean up processor resources."""
await super().cleanup()
@@ -129,9 +147,15 @@ class UserTurnProcessor(FrameProcessor):
await self._user_turn_controller.process_frame(frame)
if self._user_idle_controller:
await self._user_idle_controller.process_frame(frame)
async def _start(self, frame: StartFrame):
await self._user_turn_controller.setup(self.task_manager)
if self._user_idle_controller:
await self._user_idle_controller.setup(self.task_manager)
async def _stop(self, frame: EndFrame):
await self._cleanup()
@@ -141,6 +165,9 @@ class UserTurnProcessor(FrameProcessor):
async def _cleanup(self):
await self._user_turn_controller.cleanup()
if self._user_idle_controller:
await self._user_idle_controller.cleanup()
async def _on_push_frame(
self, controller, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM
):
@@ -180,3 +207,6 @@ class UserTurnProcessor(FrameProcessor):
async def _on_user_turn_stop_timeout(self, controller):
await self._call_event_handler("on_user_turn_stop_timeout")
async def _on_user_idle(self, controller):
await self._call_event_handler("on_user_idle")