Add VADTurnAnalyzerUserTurnStopStrategy for speech-to-speech pipelines
For speech-to-speech models like Gemini Live, audio goes directly to the LLM and transcriptions arrive too late to be useful for turn decisions. The existing TurnAnalyzerUserTurnStopStrategy waits for STT transcripts before triggering end-of-turn, adding unnecessary latency. This adds VADTurnAnalyzerUserTurnStopStrategy which triggers immediately on turn analyzer COMPLETE without waiting for any STT transcript. Also fixes the Gemini Live local VAD example to use UserStartedSpeakingFrame/ UserStoppedSpeakingFrame instead of VAD variants, since with local turn management these are the frames that flow through the pipeline.
This commit is contained in:
@@ -10,6 +10,7 @@ import os
|
||||
from dotenv import load_dotenv
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
|
||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||
from pipecat.frames.frames import LLMRunFrame
|
||||
from pipecat.pipeline.pipeline import Pipeline
|
||||
@@ -28,6 +29,9 @@ from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService, Gemini
|
||||
from pipecat.transports.base_transport import BaseTransport, TransportParams
|
||||
from pipecat.transports.daily.transport import DailyParams
|
||||
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
|
||||
from pipecat.turns.user_start import VADUserTurnStartStrategy
|
||||
from pipecat.turns.user_stop import VADTurnAnalyzerUserTurnStopStrategy
|
||||
from pipecat.turns.user_turn_strategies import UserTurnStrategies
|
||||
|
||||
load_dotenv(override=True)
|
||||
|
||||
@@ -73,6 +77,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
start=[VADUserTurnStartStrategy()],
|
||||
stop=[
|
||||
VADTurnAnalyzerUserTurnStopStrategy(turn_analyzer=LocalSmartTurnAnalyzerV3())
|
||||
],
|
||||
),
|
||||
vad_analyzer=SileroVADAnalyzer(),
|
||||
),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user