Files
pipecat/examples/word-wrangler-gemini-live/server/bot_phone_local.py

754 lines
29 KiB
Python

#
# Copyright (c) 2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Word Wrangler: A voice-based word guessing game.
To run this demo:
1. Set up environment variables:
- GOOGLE_API_KEY: API key for Google services
- GOOGLE_TEST_CREDENTIALS_FILE: Path to Google credentials JSON file
2. Install requirements:
pip install -r requirements.txt
3. Run in local development mode:
LOCAL_RUN=1 python word_wrangler.py
"""
import asyncio
import os
import re
import sys
from typing import Any, Mapping, Optional
import aiohttp
from dotenv import load_dotenv
from loguru import logger
from pipecatcloud.agent import DailySessionArguments
from word_list import generate_game_words
from pipecat.audio.resamplers.soxr_resampler import SOXRAudioResampler
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import (
BotStoppedSpeakingFrame,
CancelFrame,
EndFrame,
Frame,
InputAudioRawFrame,
LLMFullResponseEndFrame,
LLMTextFrame,
StartFrame,
TTSAudioRawFrame,
TTSSpeakFrame,
)
from pipecat.pipeline.parallel_pipeline import ParallelPipeline
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
)
from pipecat.processors.consumer_processor import ConsumerProcessor
from pipecat.processors.filters.stt_mute_filter import STTMuteConfig, STTMuteFilter, STTMuteStrategy
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.producer_processor import ProducerProcessor
from pipecat.services.gemini_multimodal_live.gemini import (
GeminiMultimodalLiveLLMService,
GeminiMultimodalModalities,
InputParams,
)
from pipecat.services.google.tts import GoogleTTSService
from pipecat.sync.base_notifier import BaseNotifier
from pipecat.sync.event_notifier import EventNotifier
from pipecat.transports.services.daily import DailyParams, DailyTransport
from pipecat.utils.text.base_text_filter import BaseTextFilter
load_dotenv(override=True)
# Check if we're in local development mode
LOCAL_RUN = os.getenv("LOCAL_RUN")
if LOCAL_RUN:
import webbrowser
try:
from runner import configure
except ImportError:
logger.error("Could not import local_runner module. Local development mode may not work.")
logger.add(sys.stderr, level="DEBUG")
GAME_DURATION_SECONDS = 120
NUM_WORDS_PER_GAME = 20
HOST_VOICE_ID = "en-US-Chirp3-HD-Charon"
PLAYER_VOICE_ID = "Kore"
# Define conversation modes with their respective prompt templates
game_player_prompt = """You are a player for a game of Word Wrangler.
GAME RULES:
1. The user will be given a word or phrase that they must describe to you
2. The user CANNOT say any part of the word/phrase directly
3. You must try to guess the word/phrase based on the user's description
4. Once you guess correctly, the user will move on to their next word
5. The user is trying to get through as many words as possible in 60 seconds
6. The external application will handle timing and keeping score
YOUR ROLE:
1. Listen carefully to the user's descriptions
2. Make intelligent guesses based on what they say
3. When you think you know the answer, state it clearly: "Is it [your guess]?"
4. If you're struggling, ask for more specific clues
5. Keep the game moving quickly - make guesses promptly
6. Be enthusiastic and encouraging
IMPORTANT:
- Keep all responses brief - the game is timed!
- Make multiple guesses if needed
- Use your common knowledge to make educated guesses
- If the user indicates you got it right, just say "Got it!" and prepare for the next word
- If you've made several wrong guesses, simply ask for "Another clue please?"
Start by guessing once you hear the user describe the word or phrase."""
game_host_prompt = """You are the AI host for a game of Word Wrangler. There are two players in the game: the human describer and the AI guesser.
GAME RULES:
1. You, the host, will give the human describer a word or phrase that they must describe
2. The describer CANNOT say any part of the word/phrase directly
3. The AI guesser will try to guess the word/phrase based on the describer's description
4. Once the guesser guesses correctly, move on to the next word
5. The describer is trying to get through as many words as possible in 60 seconds
6. The describer can say "skip" or "pass" to get a new word if they find a word too difficult
7. The describer can ask you to repeat the current word if they didn't hear it clearly
8. You'll keep track of the score (1 point for each correct guess)
9. The external application will handle timing
YOUR ROLE:
1. Start with this exact brief introduction: "Welcome to Word Wrangler! I'll give you words to describe, and the A.I. player will try to guess them. Remember, don't say any part of the word itself. Here's your first word: [word]."
2. Provide words to the describer. Choose 1 or 2 word phrases that cover a variety of topics, including animals, objects, places, and actions.
3. IMPORTANT: You will hear DIFFERENT types of input:
a. DESCRIPTIONS from the human (which you should IGNORE)
b. AFFIRMATIONS from the human (like "correct", "that's right", "you got it") which you should IGNORE
c. GUESSES from the AI player (which will be in the form of "Is it [word]?" or similar question format)
d. SKIP REQUESTS from the human (if they say "skip", "pass", or "next word please")
e. REPEAT REQUESTS from the human (if they say "repeat", "what was that?", "say again", etc.)
4. HOW TO RESPOND:
- If you hear a DESCRIPTION or AFFIRMATION from the human, respond with exactly "IGNORE" (no other text)
- If you hear a GUESS (in question form) and it's INCORRECT, respond with exactly "NO" (no other text)
- If you hear a GUESS (in question form) and it's CORRECT, respond with "Correct! That's [N] points. Your next word is [new word]" where N is the current score
- If you hear a SKIP REQUEST, respond with "The new word is [new word]" (don't change the score)
- If you hear a REPEAT REQUEST, respond with "Your word is [current word]" (don't change the score)
5. SCORING:
- Start with a score of 0
- Add 1 point for each correct guess by the AI player
- Do NOT add points for skipped words
- Announce the current score after every correct guess
RESPONSE EXAMPLES:
- Human says: "This is something you use to write" → You respond: "IGNORE"
- Human says: "That's right!" or "You got it!" → You respond: "IGNORE"
- Human says: "Wait, what was my word again?" → You respond: "Your word is [current word]"
- Human says: "Can you repeat that?" → You respond: "Your word is [current word]"
- AI says: "Is it a pen?" → If correct and it's the first point, you respond: "Correct! That's 1 point. Your next word is [new word]"
- AI says: "Is it a pencil?" → If correct and it's the third point, you respond: "Correct! That's 3 points. Your next word is [new word]"
- AI says: "Is it a marker?" → If incorrect, you respond: "NO"
- Human says: "Skip this one" or "Pass" → You respond: "The new word is [new word]"
IMPORTANT GUIDELINES:
- Choose words that range from easy to moderately difficult
- Keep all responses brief - the game is timed!
- Your "NO" and "IGNORE" responses won't be verbalized, but will be visible in the chat
- Always keep track of the CURRENT word so you can repeat it when asked
- Always keep track of the CURRENT SCORE and announce it after every correct guess
- Make sure your word choices are appropriate for all audiences
- If the human asks to skip, always provide a new word immediately without changing the score
- If the human asks you to repeat the word, say ONLY "Your word is [current word]" - don't add additional text
- CRUCIAL: Never interpret the human saying "correct", "that's right", "good job", or similar affirmations as a correct guess. These are just the human giving feedback to the AI player.
Start with the exact introduction specified above and give the first word."""
class HostResponseTextFilter(BaseTextFilter):
"""Custom text filter for Word Wrangler game.
This filter removes "NO" and "IGNORE" responses from the host so they don't get verbalized,
allowing for silent incorrect guess handling and ignoring descriptions.
"""
def __init__(self):
self._interrupted = False
def update_settings(self, settings: Mapping[str, Any]):
# No settings to update for this filter
pass
def filter(self, text: str) -> str:
# Remove case and whitespace for comparison
clean_text = text.strip().upper()
# If the text is exactly "NO" or "IGNORE", return empty string
if clean_text == "NO" or clean_text == "IGNORE":
return ""
return text
def handle_interruption(self):
self._interrupted = True
def reset_interruption(self):
self._interrupted = False
class BotStoppedSpeakingNotifier(FrameProcessor):
"""A processor that notifies whenever a BotStoppedSpeakingFrame is detected."""
def __init__(self, notifier: BaseNotifier):
super().__init__()
self._notifier = notifier
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# Check if this is a BotStoppedSpeakingFrame
if isinstance(frame, BotStoppedSpeakingFrame):
logger.debug(f"{self}: Host bot stopped speaking, notifying listeners")
await self._notifier.notify()
# Always push the frame through
await self.push_frame(frame, direction)
class StartFrameGate(FrameProcessor):
"""A gate that blocks only StartFrame until notified by a notifier.
Once opened, all frames pass through normally.
"""
def __init__(self, notifier: BaseNotifier):
super().__init__()
self._notifier = notifier
self._blocked_start_frame: Optional[Frame] = None
self._gate_opened = False
self._gate_task: Optional[asyncio.Task] = None
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if self._gate_opened:
# Once the gate is open, let everything through
await self.push_frame(frame, direction)
elif isinstance(frame, StartFrame):
# Store the StartFrame and wait for notification
logger.debug(f"{self}: Blocking StartFrame until host bot stops speaking")
self._blocked_start_frame = frame
# Start the gate task if not already running
if not self._gate_task:
self._gate_task = self.create_task(self._wait_for_notification())
async def _wait_for_notification(self):
try:
# Wait for the notifier
await self._notifier.wait()
# Gate is now open - only run this code once
if not self._gate_opened:
self._gate_opened = True
logger.debug(f"{self}: Gate opened, passing through blocked StartFrame")
# Push the blocked StartFrame if we have one
if self._blocked_start_frame:
await self.push_frame(self._blocked_start_frame)
self._blocked_start_frame = None
except asyncio.CancelledError:
logger.debug(f"{self}: Gate task was cancelled")
raise
except Exception as e:
logger.exception(f"{self}: Error in gate task: {e}")
raise
class GameStateTracker(FrameProcessor):
"""Tracks game state including new words and score by monitoring host responses."""
def __init__(self, new_word_notifier: BaseNotifier):
super().__init__()
self._new_word_notifier = new_word_notifier
self._text_buffer = ""
self._current_score = 0
# Words/phrases that indicate a new word being provided
self._key_phrases = ["your word is", "new word is", "next word is"]
# Pattern to extract score from responses
self._score_pattern = re.compile(r"that's (\d+) point", re.IGNORECASE)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# Collect text from LLMTextFrames
if isinstance(frame, LLMTextFrame):
text = frame.text
# Skip responses that are "NO" or "IGNORE"
if text.strip() in ["NO", "IGNORE"]:
logger.debug(f"Skipping NO/IGNORE response")
await self.push_frame(frame, direction)
return
# Add the new text to our buffer
self._text_buffer += text
# Process complete responses when we get an end frame
elif isinstance(frame, LLMFullResponseEndFrame):
if self._text_buffer:
buffer_lower = self._text_buffer.lower()
# 1. Check for new word announcements
new_word_detected = False
for phrase in self._key_phrases:
if phrase in buffer_lower:
await self._new_word_notifier.notify()
new_word_detected = True
break
if not new_word_detected:
logger.debug(f"No new word phrases detected")
# 2. Check for score updates
score_match = self._score_pattern.search(buffer_lower)
if score_match:
try:
score = int(score_match.group(1))
# Only update if the new score is higher
if score > self._current_score:
logger.debug(f"Score updated from {self._current_score} to {score}")
self._current_score = score
else:
logger.debug(
f"Ignoring score {score} <= current score {self._current_score}"
)
except ValueError as e:
logger.warning(f"Error parsing score: {e}")
else:
logger.debug(f"No score pattern match in: '{buffer_lower}'")
# Reset the buffer after processing the complete response
self._text_buffer = ""
# Always push the frame through
await self.push_frame(frame, direction)
@property
def current_score(self) -> int:
"""Get the current score."""
return self._current_score
class GameTimer:
"""Manages the game timer and triggers end-game events."""
def __init__(
self,
task: PipelineTask,
game_state_tracker: GameStateTracker,
game_duration_seconds: int = 120,
):
self._task = task
self._game_state_tracker = game_state_tracker
self._game_duration = game_duration_seconds
self._timer_task = None
self._start_time = None
def start(self):
"""Start the game timer."""
if self._timer_task is None:
self._start_time = asyncio.get_event_loop().time()
self._timer_task = asyncio.create_task(self._run_timer())
logger.info(f"Game timer started: {self._game_duration} seconds")
def stop(self):
"""Stop the game timer."""
if self._timer_task:
self._timer_task.cancel()
self._timer_task = None
logger.info("Game timer stopped")
def get_remaining_time(self) -> int:
"""Get the remaining time in seconds."""
if self._start_time is None:
return self._game_duration
elapsed = asyncio.get_event_loop().time() - self._start_time
remaining = max(0, self._game_duration - int(elapsed))
return remaining
async def _run_timer(self):
"""Run the timer and end the game when time is up."""
try:
# Wait for the game duration
await asyncio.sleep(self._game_duration)
# Game time is up, get the final score
final_score = self._game_state_tracker.current_score
# Create end game message
end_message = f"Time's up! Thank you for playing Word Wrangler. Your final score is {final_score} point"
if final_score != 1:
end_message += "s"
end_message += ". Great job!"
# Send end game message as TTSSpeakFrame
logger.info(f"Game over! Final score: {final_score}")
await self._task.queue_frames([TTSSpeakFrame(text=end_message)])
# End the game
await self._task.queue_frames([EndFrame()])
except asyncio.CancelledError:
logger.debug("Game timer task cancelled")
except Exception as e:
logger.exception(f"Error in game timer: {e}")
class ResettablePlayerLLM(GeminiMultimodalLiveLLMService):
"""A specialized LLM service that can reset its context when notified about a new word.
This LLM intelligently waits for the host to finish speaking before reconnecting.
"""
def __init__(
self,
api_key: str,
system_instruction: str,
new_word_notifier: BaseNotifier,
host_stopped_speaking_notifier: BaseNotifier,
voice_id: str = PLAYER_VOICE_ID,
**kwargs,
):
super().__init__(
api_key=api_key, voice_id=voice_id, system_instruction=system_instruction, **kwargs
)
self._new_word_notifier = new_word_notifier
self._host_stopped_speaking_notifier = host_stopped_speaking_notifier
self._base_system_instruction = system_instruction
self._reset_task: Optional[asyncio.Task] = None
self._pending_reset: bool = False
async def start(self, frame: StartFrame):
await super().start(frame)
# Start the notifier listener task
if not self._reset_task or self._reset_task.done():
self._reset_task = self.create_task(self._listen_for_notifications())
async def stop(self, frame: EndFrame):
# Cancel the reset task if it exists
if self._reset_task and not self._reset_task.done():
await self.cancel_task(self._reset_task)
self._reset_task = None
await super().stop(frame)
async def cancel(self, frame: CancelFrame):
# Cancel the reset task if it exists
if self._reset_task and not self._reset_task.done():
await self.cancel_task(self._reset_task)
self._reset_task = None
await super().cancel(frame)
async def _listen_for_notifications(self):
"""Listen for new word and host stopped speaking notifications."""
try:
# Create tasks for both notifiers
new_word_task = self.create_task(self._listen_for_new_word())
host_stopped_task = self.create_task(self._listen_for_host_stopped())
# Wait for both tasks to complete (which should never happen)
await asyncio.gather(new_word_task, host_stopped_task)
except asyncio.CancelledError:
logger.debug(f"{self}: Notification listener tasks cancelled")
raise
except Exception as e:
logger.exception(f"{self}: Error in notification listeners: {e}")
raise
async def _listen_for_new_word(self):
"""Listen for new word notifications and flag a reset is needed."""
while True:
# Wait for a new word notification
await self._new_word_notifier.wait()
logger.info(
f"{self}: Received new word notification, disconnecting and waiting for host to finish"
)
# Disconnect immediately to stop processing
await self._disconnect()
# Reset the system instruction
self._system_instruction = self._base_system_instruction
# Flag that we need to reconnect when the host stops speaking
self._pending_reset = True
async def _listen_for_host_stopped(self):
"""Listen for host stopped speaking and reconnect if a reset is pending."""
while True:
# Wait for host stopped speaking notification
await self._host_stopped_speaking_notifier.wait()
# If we have a pending reset, reconnect now
if self._pending_reset:
logger.info(f"{self}: Host finished speaking, completing the LLM reset")
# Reconnect
await self._connect()
# Reset the flag
self._pending_reset = False
logger.info(f"{self}: LLM reset complete")
async def tts_audio_raw_frame_filter(frame: Frame):
"""Filter to check if the frame is a TTSAudioRawFrame."""
return isinstance(frame, TTSAudioRawFrame)
# Create a resampler instance once
resampler = SOXRAudioResampler()
async def tts_to_input_audio_transformer(frame: Frame):
"""Transform TTS audio frames to InputAudioRawFrame with resampling.
Converts 24kHz TTS output to 16kHz input audio required by the player LLM.
Args:
frame (Frame): The frame to transform (expected to be TTSAudioRawFrame)
Returns:
InputAudioRawFrame: The transformed and resampled input audio frame
"""
if isinstance(frame, TTSAudioRawFrame):
# Resample the audio from 24kHz to 16kHz
resampled_audio = await resampler.resample(
frame.audio,
frame.sample_rate, # Source rate (24kHz)
16000, # Target rate (16kHz)
)
# Create a new InputAudioRawFrame with the resampled audio
input_frame = InputAudioRawFrame(
audio=resampled_audio,
sample_rate=16000, # New sample rate
num_channels=frame.num_channels,
)
return input_frame
async def main(room_url: str, token: str):
# Use the provided session logger if available, otherwise use the default logger
logger.debug("Starting bot in room: {}", room_url)
game_words = generate_game_words(NUM_WORDS_PER_GAME)
words_string = ", ".join(f'"{word}"' for word in game_words)
logger.debug(f"Game words: {words_string}")
transport = DailyTransport(
room_url,
token,
"Word Wrangler Bot",
DailyParams(
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
)
player_instruction = f"""{game_player_prompt}
Important guidelines:
1. Your responses will be converted to speech, so keep them concise and conversational.
2. Don't use special characters or formatting that wouldn't be natural in speech.
3. Encourage the user to elaborate when appropriate."""
host_instruction = f"""{game_host_prompt}
GAME WORDS:
Use ONLY these words for the game (in any order): {words_string}
Important guidelines:
1. Your responses will be converted to speech, so keep them concise and conversational.
2. Don't use special characters or formatting that wouldn't be natural in speech.
3. ONLY use words from the provided list above when giving words to the player."""
intro_message = """Start with this exact brief introduction: "Welcome to Word Wrangler! I'll give you words to describe, and the A.I. player will try to guess them. Remember, don't say any part of the word itself. Here's your first word: [word]." """
# Create the STT mute filter if we have strategies to apply
stt_mute_filter = STTMuteFilter(
config=STTMuteConfig(strategies={STTMuteStrategy.MUTE_UNTIL_FIRST_BOT_COMPLETE})
)
host_llm = GeminiMultimodalLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
system_instruction=host_instruction,
params=InputParams(modalities=GeminiMultimodalModalities.TEXT),
)
host_tts = GoogleTTSService(
voice_id=HOST_VOICE_ID,
credentials_path=os.getenv("GOOGLE_TEST_CREDENTIALS_FILE"),
text_filters=[HostResponseTextFilter()],
)
producer = ProducerProcessor(
filter=tts_audio_raw_frame_filter,
transformer=tts_to_input_audio_transformer,
passthrough=True,
)
consumer = ConsumerProcessor(producer=producer)
# Create the notifiers
bot_speaking_notifier = EventNotifier()
new_word_notifier = EventNotifier()
# Create BotStoppedSpeakingNotifier to detect when host bot stops speaking
bot_stopped_speaking_detector = BotStoppedSpeakingNotifier(bot_speaking_notifier)
# Create StartFrameGate to block Player LLM until host has stopped speaking
start_frame_gate = StartFrameGate(bot_speaking_notifier)
# Create GameStateTracker to handle new words and score tracking
game_state_tracker = GameStateTracker(new_word_notifier)
# Create a resettable player LLM that coordinates between notifiers
player_llm = ResettablePlayerLLM(
api_key=os.getenv("GOOGLE_API_KEY"),
system_instruction=player_instruction,
new_word_notifier=new_word_notifier,
host_stopped_speaking_notifier=bot_speaking_notifier,
voice_id=PLAYER_VOICE_ID,
)
# Set up the initial context for the conversation
messages = [
{
"role": "user",
"content": intro_message,
},
]
# This sets up the LLM context by providing messages and tools
context = OpenAILLMContext(messages)
context_aggregator = host_llm.create_context_aggregator(context)
pipeline = Pipeline(
[
transport.input(), # Receive audio/video from Daily call
stt_mute_filter, # Filter out speech during the bot's initial turn
ParallelPipeline(
# Host branch: manages the game and provides words
[
consumer, # Receives audio from the player branch
host_llm, # AI host that provides words and tracks score
game_state_tracker, # Tracks words and score from host responses
host_tts, # Converts host text to speech
bot_stopped_speaking_detector, # Notifies when host stops speaking
],
# Player branch: guesses words based on human descriptions
[
start_frame_gate, # Gates the player until host finishes intro
player_llm, # AI player that makes guesses
producer, # Collects audio frames to be passed to the consumer
],
),
transport.output(), # Send audio/video back to Daily call
]
)
task = PipelineTask(
pipeline,
params=PipelineParams(
allow_interruptions=False,
enable_metrics=True,
enable_usage_metrics=True,
),
)
# Create the game timer
game_timer = GameTimer(task, game_state_tracker, game_duration_seconds=GAME_DURATION_SECONDS)
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
logger.info("First participant joined: {}", participant["id"])
# Capture the participant's transcription
await transport.capture_participant_transcription(participant["id"])
# Kick off the conversation
await task.queue_frames([context_aggregator.user().get_context_frame()])
# Start the game timer
game_timer.start()
@transport.event_handler("on_participant_left")
async def on_participant_left(transport, participant, reason):
logger.info("Participant left: {}", participant)
# Stop the timer
game_timer.stop()
# Cancel the pipeline task
await task.cancel()
runner = PipelineRunner(handle_sigint=False, force_gc=True)
await runner.run(task)
async def bot(args: DailySessionArguments):
"""Main bot entry point compatible with the FastAPI route handler.
Args:
room_url: The Daily room URL
token: The Daily room token
body: The configuration object from the request body
session_id: The session ID for logging
"""
logger.info(f"Bot process initialized {args.room_url} {args.token}")
try:
await main(args.room_url, args.token)
logger.info("Bot process completed")
except Exception as e:
logger.exception(f"Error in bot process: {str(e)}")
raise
# Local development functions
async def local_main():
"""Function for local development testing."""
try:
async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session)
logger.warning("_")
logger.warning("_")
logger.warning(f"Talk to your voice agent here: {room_url}")
logger.warning("_")
logger.warning("_")
webbrowser.open(room_url)
await main(room_url, token)
except Exception as e:
logger.exception(f"Error in local development mode: {e}")
# Local development entry point
if LOCAL_RUN and __name__ == "__main__":
try:
asyncio.run(local_main())
except Exception as e:
logger.exception(f"Failed to run in local mode: {e}")