Preserve full user transcript across multiple inferences in one turn

When a stop-strategy chain splits inference-triggered from
finalization (e.g. `LLMTurnCompletionUserTurnStopStrategy` gating a
deferred detector), more than one inference can fire inside a single
user turn — each adds the new transcription segment to the context.
Previously each inference overwrote `_pending_user_turn_aggregation`,
so the eventual `on_user_turn_stopped` event surfaced only the
segment from the last inference, dropping anything the user said
before it.

Concatenate each segment into `_full_user_turn_aggregation` instead
of overwriting, and combine that running buffer with any post-final-
inference segment when emitting the public event.
This commit is contained in:
Aleix Conchillo Flaqué
2026-05-05 16:31:56 -07:00
parent d1c8162b0c
commit e3e90d38aa
2 changed files with 103 additions and 24 deletions

View File

@@ -588,10 +588,14 @@ class LLMUserAggregator(LLMContextAggregator):
self._user_is_muted = False self._user_is_muted = False
self._user_turn_start_timestamp = "" self._user_turn_start_timestamp = ""
# Aggregation captured at inference-trigger time and surfaced again # Full transcript across the user turn. Each
# in `on_user_turn_stopped`. Set to None when no inference-triggered # `_on_user_turn_inference_triggered` push captures only the
# event has fired since the last finalization. # new segment since the previous push (push_aggregation resets
self._pending_user_turn_aggregation: str | None = None # `_aggregation` after writing to context); we accumulate those
# segments here so the eventual `on_user_turn_stopped` event
# surfaces the full turn transcript even when several
# inferences fire before finalization.
self._full_user_turn_aggregation: str | None = None
self._user_turn_controller = UserTurnController( self._user_turn_controller = UserTurnController(
user_turn_strategies=user_turn_strategies, user_turn_strategies=user_turn_strategies,
@@ -862,7 +866,7 @@ class LLMUserAggregator(LLMContextAggregator):
logger.debug(f"{self}: User started speaking (strategy: {strategy})") logger.debug(f"{self}: User started speaking (strategy: {strategy})")
self._user_turn_start_timestamp = time_now_iso8601() self._user_turn_start_timestamp = time_now_iso8601()
self._pending_user_turn_aggregation = None self._full_user_turn_aggregation = None
if params.enable_user_speaking_frames: if params.enable_user_speaking_frames:
await self.broadcast_frame(UserStartedSpeakingFrame) await self.broadcast_frame(UserStartedSpeakingFrame)
@@ -881,12 +885,20 @@ class LLMUserAggregator(LLMContextAggregator):
): ):
logger.debug(f"{self}: User turn inference triggered (strategy: {strategy})") logger.debug(f"{self}: User turn inference triggered (strategy: {strategy})")
# Push aggregation now: this writes the user message to the context # Push aggregation now: this writes the user message segment to
# and pushes LLMContextFrame, which is what kicks LLM inference. # the context and emits LLMContextFrame, which kicks LLM
# `on_user_turn_stopped` later fires when the turn is semantically # inference. Concatenate the segment into
# final and surfaces the aggregated message via the public event. # `_full_user_turn_aggregation` so multiple inferences in the
aggregation = await self.push_aggregation() # same turn don't lose earlier segments from the eventual
self._pending_user_turn_aggregation = aggregation # `on_user_turn_stopped` event.
segment = await self.push_aggregation()
if segment:
if self._full_user_turn_aggregation:
self._full_user_turn_aggregation = (
f"{self._full_user_turn_aggregation} {segment}".strip()
)
else:
self._full_user_turn_aggregation = segment
await self._call_event_handler("on_user_turn_inference_triggered", strategy) await self._call_event_handler("on_user_turn_inference_triggered", strategy)
@@ -924,27 +936,25 @@ class LLMUserAggregator(LLMContextAggregator):
): ):
"""Maybe emit user turn stopped event. """Maybe emit user turn stopped event.
The aggregation has typically already been pushed at Earlier inference triggers in the same turn have already pushed
inference-trigger time and is cached in their segments to the context and accumulated them into
``self._pending_user_turn_aggregation``. Any aggregation that has ``self._full_user_turn_aggregation``. Any aggregation that
accumulated since the last inference-trigger (e.g. transcriptions arrived after the last inference trigger is flushed here so
that arrived between inference trigger and finalization) is flushed end-of-turn content is never lost from the public event.
here so end-of-turn content is never lost.
Args: Args:
strategy: The strategy that triggered the turn stop. strategy: The strategy that triggered the turn stop.
on_session_end: If True, only emit if there's unemitted content on_session_end: If True, only emit if there's unemitted content
(avoids duplicate events when session ends). (avoids duplicate events when session ends).
""" """
aggregation = await self.push_aggregation() segment = await self.push_aggregation()
previous_aggregation = self._pending_user_turn_aggregation full_aggregation = self._full_user_turn_aggregation
self._pending_user_turn_aggregation = None self._full_user_turn_aggregation = None
content = None if segment and full_aggregation:
if aggregation and previous_aggregation: content = f"{full_aggregation} {segment}".strip()
content = f"{previous_aggregation} {aggregation}".strip()
else: else:
content = previous_aggregation or aggregation content = full_aggregation or segment
if not on_session_end or content: if not on_session_end or content:
message = UserTurnStoppedMessage( message = UserTurnStoppedMessage(

View File

@@ -697,6 +697,75 @@ class TestLLMUserAggregator(unittest.IsolatedAsyncioTestCase):
self.assertEqual(events, ["inference_triggered", "stopped"]) self.assertEqual(events, ["inference_triggered", "stopped"])
async def test_multiple_inferences_in_one_turn_preserve_aggregation(self):
"""Two inference triggers before finalization should preserve the full user transcript.
When the LLM marks the first inference incomplete (○ / ◐) and the
user keeps speaking, the deferred upstream strategy fires a
second inference. Both the public ``on_user_turn_stopped`` event
and the conversation context should reflect the full user
utterance, not just the segment from the last inference.
"""
from pipecat.frames.frames import UserTurnCompletedFrame
from pipecat.turns.user_stop import LLMTurnCompletionUserTurnStopStrategy, deferred
gating = LLMTurnCompletionUserTurnStopStrategy()
upstream = deferred(
SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=TRANSCRIPTION_TIMEOUT)
)
context = LLMContext()
user_aggregator = LLMUserAggregator(
context,
params=LLMUserAggregatorParams(
user_turn_strategies=UserTurnStrategies(stop=[upstream, gating]),
),
)
inference_count = 0
stop_message = None
@user_aggregator.event_handler("on_user_turn_inference_triggered")
async def on_inference_triggered(aggregator, strategy):
nonlocal inference_count
inference_count += 1
@user_aggregator.event_handler("on_user_turn_stopped")
async def on_stopped(aggregator, strategy, message):
nonlocal stop_message
stop_message = message
pipeline = Pipeline([user_aggregator])
frames_to_send = [
VADUserStartedSpeakingFrame(),
TranscriptionFrame(text="I'm thinking", user_id="", timestamp="now"),
SleepFrame(),
VADUserStoppedSpeakingFrame(),
SleepFrame(sleep=TRANSCRIPTION_TIMEOUT + 0.1),
# First inference fired here. Imagine the LLM returned ○;
# the turn is not yet finalized, so the user keeps talking.
VADUserStartedSpeakingFrame(),
TranscriptionFrame(text="about pizza", user_id="", timestamp="now"),
SleepFrame(),
VADUserStoppedSpeakingFrame(),
SleepFrame(sleep=TRANSCRIPTION_TIMEOUT + 0.1),
# Second inference fired here. Now the LLM returns ✓ and the
# turn finalizes via UserTurnCompletedFrame.
UserTurnCompletedFrame(),
SleepFrame(),
]
await run_test(pipeline, frames_to_send=frames_to_send)
self.assertEqual(inference_count, 2)
self.assertIsNotNone(stop_message)
# The public event should report the full transcript, even
# though each inference push only writes its own segment to
# the context.
self.assertEqual(stop_message.content, "I'm thinking about pizza")
user_messages = [m for m in context.get_messages() if m.get("role") == "user"]
self.assertEqual([m["content"] for m in user_messages], ["I'm thinking", "about pizza"])
class TestLLMAssistantAggregator(unittest.IsolatedAsyncioTestCase): class TestLLMAssistantAggregator(unittest.IsolatedAsyncioTestCase):
async def test_empty(self): async def test_empty(self):