diff --git a/engine/pipeline.py b/engine/pipeline.py
index cb178a8..f0af44c 100644
--- a/engine/pipeline.py
+++ b/engine/pipeline.py
@@ -10,6 +10,7 @@ from pipecat.frames.frames import (
     LLMRunFrame,
     OutputTransportMessageUrgentFrame,
     TTSSpeakFrame,
+    UserStartedSpeakingFrame,
 )
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.runner import PipelineRunner
@@ -189,6 +190,7 @@ async def run_pipeline_with_serializer(
         ),
         idle_timeout_secs=config.session.inactivity_timeout_sec,
     )
+    task.set_reached_upstream_filter((UserStartedSpeakingFrame,))
     idle_prompt_count = 0
 
     @transport.event_handler("on_client_connected")
@@ -218,6 +220,11 @@ async def run_pipeline_with_serializer(
         logger.info(f"{client_label} websocket session timed out")
         await task.cancel()
 
+    @task.event_handler("on_frame_reached_upstream")
+    async def on_frame_reached_upstream(_task, _frame: UserStartedSpeakingFrame):
+        nonlocal idle_prompt_count
+        idle_prompt_count = 0
+
     @user_aggregator.event_handler("on_user_turn_started")
     async def on_user_turn_started(_aggregator, _strategy):
         nonlocal idle_prompt_count