Add VADTurnAnalyzerUserTurnStopStrategy for speech-to-speech pipelines

For speech-to-speech models like Gemini Live, audio goes directly to the
LLM and transcriptions arrive too late to be useful for turn decisions.
The existing TurnAnalyzerUserTurnStopStrategy waits for STT transcripts
before triggering end-of-turn, adding unnecessary latency.

This adds VADTurnAnalyzerUserTurnStopStrategy which triggers immediately
on turn analyzer COMPLETE without waiting for any STT transcript. Also
fixes the Gemini Live local VAD example to use UserStartedSpeakingFrame/
UserStoppedSpeakingFrame instead of VAD variants, since with local turn
management these are the frames that flow through the pipeline.
This commit is contained in:
Mark Backman
2026-03-30 10:43:07 -04:00
parent e8c3f73968
commit 677ca04a18
4 changed files with 359 additions and 1 deletions

View File

@@ -10,6 +10,7 @@ import os
from dotenv import load_dotenv
from loguru import logger
from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import LLMRunFrame
from pipecat.pipeline.pipeline import Pipeline
@@ -28,6 +29,9 @@ from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService, Gemini
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
from pipecat.turns.user_start import VADUserTurnStartStrategy
from pipecat.turns.user_stop import VADTurnAnalyzerUserTurnStopStrategy
from pipecat.turns.user_turn_strategies import UserTurnStrategies
load_dotenv(override=True)
@@ -73,6 +77,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
context,
user_params=LLMUserAggregatorParams(
user_turn_strategies=UserTurnStrategies(
start=[VADUserTurnStartStrategy()],
stop=[
VADTurnAnalyzerUserTurnStopStrategy(turn_analyzer=LocalSmartTurnAnalyzerV3())
],
),
vad_analyzer=SileroVADAnalyzer(),
),
)