677ca04a18
For speech-to-speech models like Gemini Live, audio goes directly to the LLM and transcriptions arrive too late to be useful for turn decisions. The existing TurnAnalyzerUserTurnStopStrategy waits for STT transcripts before triggering end-of-turn, adding unnecessary latency. This adds VADTurnAnalyzerUserTurnStopStrategy which triggers immediately on turn analyzer COMPLETE without waiting for any STT transcript. Also fixes the Gemini Live local VAD example to use UserStartedSpeakingFrame/ UserStoppedSpeakingFrame instead of VAD variants, since with local turn management these are the frames that flow through the pipeline.
147 lines
4.7 KiB
Python
147 lines
4.7 KiB
Python
#
|
|
# Copyright (c) 2024-2026, Daily
|
|
#
|
|
# SPDX-License-Identifier: BSD 2-Clause License
|
|
#
|
|
|
|
|
|
import os
|
|
|
|
from dotenv import load_dotenv
|
|
from loguru import logger
|
|
|
|
from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
|
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
|
from pipecat.frames.frames import LLMRunFrame
|
|
from pipecat.pipeline.pipeline import Pipeline
|
|
from pipecat.pipeline.runner import PipelineRunner
|
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|
from pipecat.processors.aggregators.llm_context import LLMContext
|
|
from pipecat.processors.aggregators.llm_response_universal import (
|
|
AssistantTurnStoppedMessage,
|
|
LLMContextAggregatorPair,
|
|
LLMUserAggregatorParams,
|
|
UserTurnStoppedMessage,
|
|
)
|
|
from pipecat.runner.types import RunnerArguments
|
|
from pipecat.runner.utils import create_transport
|
|
from pipecat.services.google.gemini_live.llm import GeminiLiveLLMService, GeminiVADParams
|
|
from pipecat.transports.base_transport import BaseTransport, TransportParams
|
|
from pipecat.transports.daily.transport import DailyParams
|
|
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
|
|
from pipecat.turns.user_start import VADUserTurnStartStrategy
|
|
from pipecat.turns.user_stop import VADTurnAnalyzerUserTurnStopStrategy
|
|
from pipecat.turns.user_turn_strategies import UserTurnStrategies
|
|
|
|
load_dotenv(override=True)
|
|
|
|
|
|
# We use lambdas to defer transport parameter creation until the transport
|
|
# type is selected at runtime.
|
|
transport_params = {
|
|
"daily": lambda: DailyParams(
|
|
audio_in_enabled=True,
|
|
audio_out_enabled=True,
|
|
),
|
|
"twilio": lambda: FastAPIWebsocketParams(
|
|
audio_in_enabled=True,
|
|
audio_out_enabled=True,
|
|
),
|
|
"webrtc": lambda: TransportParams(
|
|
audio_in_enabled=True,
|
|
audio_out_enabled=True,
|
|
),
|
|
}
|
|
|
|
|
|
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
|
logger.info(f"Starting bot")
|
|
|
|
llm = GeminiLiveLLMService(
|
|
api_key=os.getenv("GOOGLE_API_KEY"),
|
|
settings=GeminiLiveLLMService.Settings(
|
|
voice="Aoede", # Puck, Charon, Kore, Fenrir, Aoede
|
|
vad=GeminiVADParams(disabled=True),
|
|
),
|
|
# inference_on_context_initialization=False,
|
|
)
|
|
|
|
context = LLMContext(
|
|
[
|
|
{
|
|
"role": "user",
|
|
"content": "Say hello. Then ask if I want to hear a joke.",
|
|
},
|
|
],
|
|
)
|
|
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
|
context,
|
|
user_params=LLMUserAggregatorParams(
|
|
user_turn_strategies=UserTurnStrategies(
|
|
start=[VADUserTurnStartStrategy()],
|
|
stop=[
|
|
VADTurnAnalyzerUserTurnStopStrategy(turn_analyzer=LocalSmartTurnAnalyzerV3())
|
|
],
|
|
),
|
|
vad_analyzer=SileroVADAnalyzer(),
|
|
),
|
|
)
|
|
|
|
pipeline = Pipeline(
|
|
[
|
|
transport.input(),
|
|
user_aggregator,
|
|
llm,
|
|
transport.output(),
|
|
assistant_aggregator,
|
|
]
|
|
)
|
|
|
|
task = PipelineTask(
|
|
pipeline,
|
|
params=PipelineParams(
|
|
enable_metrics=True,
|
|
enable_usage_metrics=True,
|
|
),
|
|
idle_timeout_secs=runner_args.pipeline_idle_timeout_secs,
|
|
)
|
|
|
|
@transport.event_handler("on_client_connected")
|
|
async def on_client_connected(transport, client):
|
|
logger.info(f"Client connected")
|
|
# Kick off the conversation.
|
|
await task.queue_frames([LLMRunFrame()])
|
|
|
|
@transport.event_handler("on_client_disconnected")
|
|
async def on_client_disconnected(transport, client):
|
|
logger.info(f"Client disconnected")
|
|
await task.cancel()
|
|
|
|
@user_aggregator.event_handler("on_user_turn_stopped")
|
|
async def on_user_turn_stopped(aggregator, strategy, message: UserTurnStoppedMessage):
|
|
timestamp = f"[{message.timestamp}] " if message.timestamp else ""
|
|
line = f"{timestamp}user: {message.content}"
|
|
logger.info(f"Transcript: {line}")
|
|
|
|
@assistant_aggregator.event_handler("on_assistant_turn_stopped")
|
|
async def on_assistant_turn_stopped(aggregator, message: AssistantTurnStoppedMessage):
|
|
timestamp = f"[{message.timestamp}] " if message.timestamp else ""
|
|
line = f"{timestamp}assistant: {message.content}"
|
|
logger.info(f"Transcript: {line}")
|
|
|
|
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
|
|
|
await runner.run(task)
|
|
|
|
|
|
async def bot(runner_args: RunnerArguments):
|
|
"""Main bot entry point compatible with Pipecat Cloud."""
|
|
transport = await create_transport(runner_args, transport_params)
|
|
await run_bot(transport, runner_args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
from pipecat.runner.run import main
|
|
|
|
main()
|