From 72934bd8ae23de1d4c34e75350e168faa8fe54a6 Mon Sep 17 00:00:00 2001 From: zack Date: Thu, 26 Feb 2026 21:35:47 -0500 Subject: [PATCH 01/23] Add u3-rt-pro support and improvements to AssemblyAI STT service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix speaker diarization: Add field alias for speaker_label → speaker mapping in TurnMessage model - Add warning for non-optimal min_end_of_turn_silence_when_confident values (recommends 100ms for best latency) - Improve max_turn_silence override warning message clarity - Update custom prompt warning (remove 88% accuracy claim) - Add comprehensive logging for debugging: - Log final connection params after modifications - Log WebSocket URL and parsed parameters - Log speaker field in transcripts - Log text sent to LLM with speaker formatting - Support dynamic configuration updates via STTUpdateSettingsFrame: - keyterms_prompt (when AssemblyAI API supports it) - prompt - max_turn_silence - min_end_of_turn_silence_when_confident --- src/pipecat/services/assemblyai/models.py | 45 ++- src/pipecat/services/assemblyai/stt.py | 441 ++++++++++++++++++---- 2 files changed, 408 insertions(+), 78 deletions(-) diff --git a/src/pipecat/services/assemblyai/models.py b/src/pipecat/services/assemblyai/models.py index ca58cb848..fb883ac99 100644 --- a/src/pipecat/services/assemblyai/models.py +++ b/src/pipecat/services/assemblyai/models.py @@ -12,7 +12,7 @@ transcription WebSocket messages and connection configuration. from typing import List, Literal, Optional -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field class Word(BaseModel): @@ -68,8 +68,16 @@ class TurnMessage(BaseMessage): transcript: The transcribed text for this turn. end_of_turn_confidence: Confidence score for end-of-turn detection. words: List of individual words with timing and confidence data. + language_code: Detected language code (e.g., "es", "fr"). Only present with + complete utterances or when end_of_turn is True. + language_confidence: Confidence score (0-1) for language detection. Only present + with complete utterances or when end_of_turn is True. + speaker: Speaker label (e.g., "A", "B"). Only present when speaker_labels is + enabled and end_of_turn is True. Maps to 'speaker_label' in JSON response. """ + model_config = ConfigDict(populate_by_name=True) + type: Literal["Turn"] = "Turn" turn_order: int turn_is_formatted: bool @@ -77,6 +85,21 @@ class TurnMessage(BaseMessage): transcript: str end_of_turn_confidence: float words: List[Word] + language_code: Optional[str] = None + language_confidence: Optional[float] = None + speaker: Optional[str] = Field(default=None, alias="speaker_label") + + +class SpeechStartedMessage(BaseMessage): + """Message sent when speech is first detected in the audio stream. + + Parameters: + type: Always "SpeechStarted" for this message type. + timestamp: Audio timestamp in milliseconds when speech was detected. + """ + + type: Literal["SpeechStarted"] = "SpeechStarted" + timestamp: int class TerminationMessage(BaseMessage): @@ -94,7 +117,7 @@ class TerminationMessage(BaseMessage): # Union type for all possible message types -AnyMessage = BeginMessage | TurnMessage | TerminationMessage +AnyMessage = BeginMessage | TurnMessage | SpeechStartedMessage | TerminationMessage class AssemblyAIConnectionParams(BaseModel): @@ -109,7 +132,15 @@ class AssemblyAIConnectionParams(BaseModel): min_end_of_turn_silence_when_confident: Minimum silence duration when confident about end-of-turn. max_turn_silence: Maximum silence duration before forcing end-of-turn. keyterms_prompt: List of key terms to guide transcription. Will be JSON serialized before sending. - speech_model: Select between English and multilingual models. Defaults to "universal-streaming-english". + prompt: Optional text prompt to guide the transcription. Only used when speech_model is "u3-rt-pro". + speech_model: Select between English, multilingual, and u3-rt-pro models. Defaults to "u3-rt-pro". + language_detection: Enable automatic language detection. Only applicable to + universal-streaming-multilingual. When enabled, Turn messages include + language_code and language_confidence fields. Defaults to None (not sent). + format_turns: Whether to format transcript turns. Defaults to True. + speaker_labels: Enable speaker diarization. When enabled, final transcripts + (end_of_turn=True) include a speaker field identifying the speaker + (e.g., "Speaker A", "Speaker B"). Defaults to None (not sent). """ sample_rate: int = 16000 @@ -120,6 +151,10 @@ class AssemblyAIConnectionParams(BaseModel): min_end_of_turn_silence_when_confident: Optional[int] = None max_turn_silence: Optional[int] = None keyterms_prompt: Optional[List[str]] = None - speech_model: Literal["universal-streaming-english", "universal-streaming-multilingual"] = ( - "universal-streaming-english" + prompt: Optional[str] = None + speech_model: Literal["universal-streaming-english", "universal-streaming-multilingual", "u3-rt-pro"] = ( + "u3-rt-pro" ) + language_detection: Optional[bool] = None + format_turns: bool = True + speaker_labels: Optional[bool] = None diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index a89f5fe52..edd5ac7b7 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -13,7 +13,7 @@ WebSocket API for streaming audio transcription. import asyncio import json from dataclasses import dataclass, field -from typing import Any, AsyncGenerator, Dict, Optional +from typing import Any, AsyncGenerator, Dict, Mapping, Optional from urllib.parse import urlencode from loguru import logger @@ -41,6 +41,7 @@ from .models import ( AssemblyAIConnectionParams, BaseMessage, BeginMessage, + SpeechStartedMessage, TerminationMessage, TurnMessage, ) @@ -54,6 +55,26 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +def map_language_from_assemblyai(language_code: str) -> Language: + """Map AssemblyAI language codes to Pipecat Language enum. + + AssemblyAI returns simple language codes like "es", "fr", etc. + This function maps them to the corresponding Language enum values. + + Args: + language_code: AssemblyAI language code (e.g., "es", "fr", "de") + + Returns: + Corresponding Language enum value, defaulting to Language.EN if not found. + """ + try: + # Try to match the language code directly + return Language(language_code.lower()) + except ValueError: + logger.warning(f"Unknown language code from AssemblyAI: {language_code}, defaulting to English") + return Language.EN + + @dataclass class AssemblyAISTTSettings(STTSettings): """Settings for the AssemblyAI STT service. @@ -87,6 +108,8 @@ class AssemblyAISTTService(WebsocketSTTService): api_endpoint_base_url: str = "wss://streaming.assemblyai.com/v3/ws", connection_params: AssemblyAIConnectionParams = AssemblyAIConnectionParams(), vad_force_turn_endpoint: bool = True, + should_interrupt: bool = True, + speaker_format: Optional[str] = None, ttfs_p99_latency: Optional[float] = ASSEMBLYAI_TTFS_P99, **kwargs, ): @@ -97,18 +120,64 @@ class AssemblyAISTTService(WebsocketSTTService): language: Language code for transcription. Defaults to English (Language.EN). api_endpoint_base_url: WebSocket endpoint URL. Defaults to AssemblyAI's streaming endpoint. connection_params: Connection configuration parameters. Defaults to AssemblyAIConnectionParams(). - vad_force_turn_endpoint: Whether to force turn endpoint on VAD stop. When True, - disables AssemblyAI's model-based turn detection and relies on external VAD - to trigger turn endpoints. Automatically sets end_of_turn_confidence_threshold=1.0 - and max_turn_silence=2000 unless explicitly overridden. Defaults to True. + vad_force_turn_endpoint: Controls turn detection mode. + When True (Pipecat mode, default): Forces AssemblyAI to return finals ASAP + so Pipecat's turn detection (e.g., Smart Turn) decides when the user is done. + - min_end_of_turn_silence_when_confident defaults to 100ms (user can override) + - max_turn_silence is ALWAYS set equal to min_end_of_turn_silence_when_confident + - VAD stop sends ForceEndpoint as ceiling + - No UserStarted/StoppedSpeakingFrame emitted from STT + When False (STT mode, u3-rt-pro only): AssemblyAI's model controls turn endings. + - Uses AssemblyAI API defaults for all parameters (unless user explicitly sets them) + - Respects all user-provided connection_params as-is + - Emits UserStarted/StoppedSpeakingFrame from STT + - No ForceEndpoint on VAD stop + should_interrupt: Whether to interrupt the bot when the user starts speaking + in STT mode (vad_force_turn_endpoint=False). Only applies to STT mode. + Defaults to True. + speaker_format: Optional format string for speaker labels when diarization is enabled. + Use {speaker} for speaker label and {text} for transcript text. + Example: "<{speaker}>{text}" or "{speaker}: {text}" + If None, transcript text is not modified. Defaults to None. ttfs_p99_latency: P99 latency from speech end to final transcript in seconds. Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark **kwargs: Additional arguments passed to parent STTService class. """ - # When vad_force_turn_endpoint is enabled, configure connection params for manual - # turn detection mode (disable model-based turn detection) + # STT turn detection (vad_force_turn_endpoint=False) requires the + # SpeechStarted event for reliable barge-in. Only u3-rt-pro supports + # this. Other models must use Pipecat turn detection. + is_u3_pro = connection_params.speech_model == "u3-rt-pro" + if not vad_force_turn_endpoint and not is_u3_pro: + raise ValueError( + f"STT turn detection (vad_force_turn_endpoint=False) requires " + f"u3-rt-pro for SpeechStarted support. Either set " + f"vad_force_turn_endpoint=True for {connection_params.speech_model}, " + f"or use speech_model='u3-rt-pro'." + ) + + # Validate that prompt and keyterms_prompt are not both set + if connection_params.prompt is not None and connection_params.keyterms_prompt is not None: + raise ValueError( + "The prompt and keyterms_prompt parameters cannot be used in the same request. " + "Please choose either one or the other based on your use case. When you use " + "keyterms_prompt, your boosted words are appended to the default prompt automatically. " + "Or to boost within prompt: + Make sure to boost the words in the audio. " + "For more info go to: https://www.assemblyai.com/docs/streaming/universal-3-pro" + ) + + # Warn if user sets a custom prompt (recommend testing without one first) + if connection_params.prompt is not None: + logger.warning( + "Custom prompt detected. We recommend testing with no prompt first, as this " + "will use our optimized default prompt for voice agents. Bad prompts may lead " + "to bad results. If you'd like to create your own prompt, check out our " + "prompting guide at: https://www.assemblyai.com/docs/streaming/prompting" + ) + + # When vad_force_turn_endpoint is enabled, configure connection params + # for Pipecat turn detection mode (fast finals for smart turn analyzer) if vad_force_turn_endpoint: - connection_params = self._configure_manual_turn_mode(connection_params) + connection_params = self._configure_pipecat_turn_mode(connection_params, is_u3_pro) super().__init__( sample_rate=connection_params.sample_rate, @@ -124,6 +193,8 @@ class AssemblyAISTTService(WebsocketSTTService): self._api_key = api_key self._api_endpoint_base_url = api_endpoint_base_url self._vad_force_turn_endpoint = vad_force_turn_endpoint + self._should_interrupt = should_interrupt + self._speaker_format = speaker_format self._termination_event = asyncio.Event() self._received_termination = False @@ -135,45 +206,77 @@ class AssemblyAISTTService(WebsocketSTTService): self._chunk_size_ms = 50 self._chunk_size_bytes = 0 - def _configure_manual_turn_mode( - self, connection_params: AssemblyAIConnectionParams - ) -> AssemblyAIConnectionParams: - """Configure connection params for manual turn detection mode. + self._user_speaking = False + self._vad_speaking = False - When vad_force_turn_endpoint is enabled, we want to disable AssemblyAI's - model-based turn detection and rely on external VAD. This requires: - - end_of_turn_confidence_threshold=1.0 (disable semantic turn detection) - - max_turn_silence=2000 (high value since VAD handles turn endings) + # Log final connection params after any modifications + logger.info(f"{self} Final connection params being sent to AssemblyAI:") + logger.info(f" min_end_of_turn_silence_when_confident: {self._settings.connection_params.min_end_of_turn_silence_when_confident}") + logger.info(f" max_turn_silence: {self._settings.connection_params.max_turn_silence}") + + # Warn if min_end_of_turn_silence_when_confident is not 100ms + if self._settings.connection_params.min_end_of_turn_silence_when_confident != 100: + logger.warning( + f"For best latency, set min_end_of_turn_silence_when_confident to 100ms. " + f"Current value: {self._settings.connection_params.min_end_of_turn_silence_when_confident}ms" + ) + + def _configure_pipecat_turn_mode( + self, connection_params: AssemblyAIConnectionParams, is_u3_pro: bool + ) -> AssemblyAIConnectionParams: + """Configure connection params for Pipecat turn detection mode. + + When vad_force_turn_endpoint is enabled, force AssemblyAI to return + finals as fast as possible so Pipecat's smart turn analyzer can decide + when the user is done speaking. VAD stop is the absolute ceiling. + + u3-rt-pro: + - min_end_of_turn_silence_when_confident defaults to 100ms (user can override) + - max_turn_silence is ALWAYS set equal to min_end_of_turn_silence_when_confident + to avoid double turn detection (AssemblyAI + Pipecat both analyzing) + - If user sets max_turn_silence, it's ignored with a warning + - end_of_turn_confidence_threshold: not set (API default) + + universal-streaming-*: + - end_of_turn_confidence_threshold=0.0 (disable semantic turn detection) + - min_end_of_turn_silence_when_confident=160 + - max_turn_silence: not set (API default) Args: connection_params: The user-provided connection parameters. + is_u3_pro: Whether using u3-rt-pro model. Returns: - Updated connection parameters configured for manual turn mode. + Updated connection parameters configured for Pipecat turn mode. """ updates = {} - # Check end_of_turn_confidence_threshold - if connection_params.end_of_turn_confidence_threshold is None: - updates["end_of_turn_confidence_threshold"] = 1.0 - elif connection_params.end_of_turn_confidence_threshold != 1.0: - logger.warning( - f"vad_force_turn_endpoint is enabled but end_of_turn_confidence_threshold " - f"is set to {connection_params.end_of_turn_confidence_threshold}. " - f"For manual turn detection mode, this should be 1.0 to disable " - f"model-based turn detection. The current value will be used." - ) + if is_u3_pro: + # u3-rt-pro: Synchronize max_turn_silence with min_end_of_turn_silence_when_confident + min_silence = connection_params.min_end_of_turn_silence_when_confident + if min_silence is None: + min_silence = 100 - # Check max_turn_silence - if connection_params.max_turn_silence is None: - updates["max_turn_silence"] = 2000 - elif connection_params.max_turn_silence < 1000: - logger.warning( - f"vad_force_turn_endpoint is enabled but max_turn_silence is set to " - f"{connection_params.max_turn_silence}ms. With manual turn detection, " - f"a higher value (e.g., 2000ms) is recommended to avoid premature " - f"turn endings. The current value will be used." - ) + # Warn if user set max_turn_silence (will be overridden) + if connection_params.max_turn_silence is not None: + logger.warning( + f"Your max_turn_silence value ({connection_params.max_turn_silence}ms) will be " + f"OVERRIDDEN in Pipecat mode (vad_force_turn_endpoint=True). It will be set to " + f"{min_silence}ms (matching min_end_of_turn_silence_when_confident) and SENT to " + f"AssemblyAI to avoid double turn detection. To use your max_turn_silence as-is, " + f"switch to STT mode (vad_force_turn_endpoint=False)." + ) + + updates = { + "min_end_of_turn_silence_when_confident": min_silence, + "max_turn_silence": min_silence, + } + else: + # universal-streaming: Different configuration (works differently) + updates = { + "end_of_turn_confidence_threshold": 0.0, + "min_end_of_turn_silence_when_confident": 160, + } # Apply updates if any if updates: @@ -190,9 +293,14 @@ class AssemblyAISTTService(WebsocketSTTService): return True async def _update_settings(self, delta: STTSettings) -> dict[str, Any]: - """Apply a settings delta. + """Apply a settings delta and send UpdateConfiguration if connected. - Settings are stored but not applied to the active connection. + Stores settings changes and sends UpdateConfiguration message to AssemblyAI + without reconnecting. Supports updating: + - keyterms_prompt: List of terms to boost (can be empty array to clear) + - prompt: Custom prompt text (u3-rt-pro only) + - max_turn_silence: Maximum silence before forcing turn end + - min_end_of_turn_silence_when_confident: Silence before EOT check Args: delta: A :class:`STTSettings` (or ``AssemblyAISTTSettings``) delta. @@ -205,18 +313,63 @@ class AssemblyAISTTService(WebsocketSTTService): if not changed: return changed - # TODO: someday we could reconnect here to apply updated settings. - # Code might look something like the below: - # # Re-apply manual turn mode config if vad_force_turn_endpoint is active - # # and connection_params were updated. - # if self._vad_force_turn_endpoint and "connection_params" in changed: - # self._settings.connection_params = self._configure_manual_turn_mode( - # self._settings.connection_params - # ) - # await self._disconnect() - # await self._connect() + # If websocket is connected, send UpdateConfiguration for supported params + if self._websocket and self._websocket.state is State.OPEN and "connection_params" in changed: + # Build UpdateConfiguration message + update_config = {"type": "UpdateConfiguration"} + conn_params = self._settings.connection_params - self._warn_unhandled_updated_settings(changed) + # Get the old connection_params to see what changed + old_conn_params = changed.get("connection_params") + + # Check each potentially changed parameter + if hasattr(conn_params, "keyterms_prompt"): + if old_conn_params is None or conn_params.keyterms_prompt != old_conn_params.keyterms_prompt: + if conn_params.keyterms_prompt is not None: + update_config["keyterms_prompt"] = conn_params.keyterms_prompt + logger.info(f"Updating keyterms_prompt to: {conn_params.keyterms_prompt}") + + if hasattr(conn_params, "prompt"): + if old_conn_params is None or conn_params.prompt != old_conn_params.prompt: + if conn_params.prompt is not None: + if conn_params.speech_model != "u3-rt-pro": + logger.warning( + f"prompt parameter is only supported with u3-rt-pro model, " + f"current model is {conn_params.speech_model}" + ) + else: + update_config["prompt"] = conn_params.prompt + logger.info(f"Updating prompt") + + if hasattr(conn_params, "max_turn_silence"): + if old_conn_params is None or conn_params.max_turn_silence != old_conn_params.max_turn_silence: + if conn_params.max_turn_silence is not None: + update_config["max_turn_silence"] = conn_params.max_turn_silence + logger.info(f"Updating max_turn_silence to: {conn_params.max_turn_silence}ms") + + if hasattr(conn_params, "min_end_of_turn_silence_when_confident"): + if old_conn_params is None or conn_params.min_end_of_turn_silence_when_confident != old_conn_params.min_end_of_turn_silence_when_confident: + if conn_params.min_end_of_turn_silence_when_confident is not None: + update_config["min_end_of_turn_silence_when_confident"] = conn_params.min_end_of_turn_silence_when_confident + logger.info(f"Updating min_end_of_turn_silence_when_confident to: {conn_params.min_end_of_turn_silence_when_confident}ms") + + # Send update if we have parameters to update + if len(update_config) > 1: # More than just "type" + try: + await self._websocket.send(json.dumps(update_config)) + logger.info(f"Sent UpdateConfiguration: {update_config}") + except Exception as e: + logger.error(f"Failed to send UpdateConfiguration: {e}") + elif "connection_params" in changed: + logger.warning( + "Connection params changed but WebSocket not connected. " + "Settings will be applied on next connection." + ) + + # Warn about other settings that can't be changed dynamically + other_changes = {k: v for k, v in changed.items() if k not in ["connection_params"]} + if other_changes: + self._warn_unhandled_updated_settings(other_changes) return changed @@ -305,7 +458,13 @@ class AssemblyAISTTService(WebsocketSTTService): if params: query_string = urlencode(params) - return f"{self._api_endpoint_base_url}?{query_string}" + full_url = f"{self._api_endpoint_base_url}?{query_string}" + logger.info(f"{self} WebSocket URL being sent to AssemblyAI:") + logger.info(f" {full_url}") + logger.info(f" Parsed params:") + for k, v in params.items(): + logger.info(f" {k}: {v}") + return full_url return self._api_endpoint_base_url async def _connect(self): @@ -421,6 +580,9 @@ class AssemblyAISTTService(WebsocketSTTService): async for message in self._get_websocket(): try: data = json.loads(message) + # Log raw JSON for Turn messages to debug speaker_label + if data.get("type") == "Turn": + logger.debug(f"{self} RAW JSON from AssemblyAI: {json.dumps(data, indent=2)}") await self._handle_message(data) except json.JSONDecodeError: logger.warning(f"Received non-JSON message: {message}") @@ -433,6 +595,8 @@ class AssemblyAISTTService(WebsocketSTTService): return BeginMessage.model_validate(message) elif msg_type == "Turn": return TurnMessage.model_validate(message) + elif msg_type == "SpeechStarted": + return SpeechStartedMessage.model_validate(message) elif msg_type == "Termination": return TerminationMessage.model_validate(message) else: @@ -449,11 +613,37 @@ class AssemblyAISTTService(WebsocketSTTService): ) elif isinstance(parsed_message, TurnMessage): await self._handle_transcription(parsed_message) + elif isinstance(parsed_message, SpeechStartedMessage): + await self._handle_speech_started(parsed_message) elif isinstance(parsed_message, TerminationMessage): await self._handle_termination(parsed_message) except Exception as e: await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) + async def _handle_speech_started(self, message: SpeechStartedMessage): + """Handle SpeechStarted event — fast barge-in for Mode 2. + + Broadcasts UserStartedSpeakingFrame to signal the start of user + speech, then pushes an interruption to cancel any bot audio. + SpeechStarted fires before any transcript arrives, so the turn + is cleanly started before any transcription frames are pushed. + + Only applies to Mode 2 (STT turn detection). In Mode 1, VAD + + smart turn analyzer handle interruptions via the aggregator. + """ + logger.debug(f"{self} SpeechStarted received (vad_force_turn_endpoint={self._vad_force_turn_endpoint})") + if self._vad_force_turn_endpoint: + logger.debug(f"{self} SpeechStarted ignored in Pipecat mode") + return # Mode 1: handled by aggregator + + logger.debug(f"{self} Processing SpeechStarted in STT mode") + await self.start_processing_metrics() + await self.broadcast_frame(UserStartedSpeakingFrame) + if self._should_interrupt: + await self.push_interruption_task_frame_and_wait() + self._user_speaking = True + logger.debug(f"{self} _user_speaking set to True") + async def _handle_termination(self, message: TerminationMessage): """Handle termination message.""" self._received_termination = True @@ -466,30 +656,135 @@ class AssemblyAISTTService(WebsocketSTTService): await self.push_frame(EndFrame()) async def _handle_transcription(self, message: TurnMessage): - """Handle transcription results.""" + """Handle transcription results with two-mode turn detection. + + Mode 1 (vad_force_turn_endpoint=True, Pipecat turn detection): + - No UserStarted/StoppedSpeakingFrame from STT + - end_of_turn → TranscriptionFrame (finalized set by base class + if this is a ForceEndpoint response) + - else → InterimTranscriptionFrame + + Mode 2 (vad_force_turn_endpoint=False, STT turn detection): + - UserStartedSpeakingFrame on first transcript + - end_of_turn → TranscriptionFrame + UserStoppedSpeakingFrame + - else → InterimTranscriptionFrame + """ + # Log transcript details + logger.info(f"{self} ===== TRANSCRIPT RECEIVED =====") + logger.info(f" Text: \"{message.transcript}\"") + logger.info(f" end_of_turn: {message.end_of_turn}") + logger.info(f" turn_is_formatted: {message.turn_is_formatted}") + logger.info(f" turn_order: {message.turn_order}") + if message.end_of_turn_confidence is not None: + logger.info(f" end_of_turn_confidence: {message.end_of_turn_confidence}") + logger.info(f" speaker: {message.speaker}") + logger.info(f"===============================") + if not message.transcript: return - if message.end_of_turn and ( - not self._settings.connection_params.formatted_finals or message.turn_is_formatted - ): - await self.push_frame( - TranscriptionFrame( - message.transcript, - self._user_id, - time_now_iso8601(), - self._settings.language, - message, + + # Use detected language if available with sufficient confidence + language = Language.EN + if message.language_code and message.language_confidence: + if message.language_confidence >= 0.7: + language = map_language_from_assemblyai(message.language_code) + else: + logger.warning( + f"Low language detection confidence ({message.language_confidence:.2f}) " + f"for language '{message.language_code}', falling back to English" ) - ) - await self._trace_transcription(message.transcript, True, self._settings.language) - await self.stop_processing_metrics() + + # Handle speaker diarization + speaker_id = self._user_id + transcript_text = message.transcript + + if message.speaker: + speaker_id = message.speaker + # Format transcript with speaker labels if format string provided + if self._speaker_format: + transcript_text = self._speaker_format.format( + speaker=message.speaker, + text=message.transcript + ) + logger.info(f"{self} 🤖 TEXT SENT TO LLM (with speaker format): \"{transcript_text}\"") + else: + logger.info(f"{self} 🤖 TEXT SENT TO LLM (speaker {message.speaker}): \"{transcript_text}\"") else: - await self.push_frame( - InterimTranscriptionFrame( - message.transcript, - self._user_id, - time_now_iso8601(), - self._settings.language, - message, + logger.info(f"{self} 🤖 TEXT SENT TO LLM: \"{transcript_text}\"") + + # Determine if this is a final turn from AssemblyAI + is_final_turn = message.end_of_turn and ( + not self._settings.connection_params.format_turns or message.turn_is_formatted + ) + + if self._vad_force_turn_endpoint: + # --- Mode 1: Pipecat turn detection --- + # No UserStarted/StoppedSpeakingFrame — VAD + smart turn analyzer handle this + if is_final_turn: + finalize_confirmed = bool(message.turn_is_formatted) + if finalize_confirmed: + self.confirm_finalize() + logger.debug(f"{self} Final transcript: \"{transcript_text}\"") + await self.push_frame( + TranscriptionFrame( + transcript_text, + speaker_id, + time_now_iso8601(), + language, + message, + ) + ) + await self._trace_transcription(transcript_text, True, language) + await self.stop_processing_metrics() + else: + logger.debug(f"{self} Interim transcript: \"{transcript_text}\"") + await self.push_frame( + InterimTranscriptionFrame( + transcript_text, + speaker_id, + time_now_iso8601(), + language, + message, + ) + ) + else: + # --- Mode 2: STT turn detection --- + # SpeechStarted handles UserStartedSpeakingFrame + interruption. + # If SpeechStarted hasn't fired yet (shouldn't happen, but guard), + # broadcast here as fallback. + logger.debug(f"{self} Transcript received in STT mode (_user_speaking={self._user_speaking})") + if not self._user_speaking: + logger.warning(f"{self} Transcript arrived before SpeechStarted, broadcasting fallback UserStartedSpeakingFrame") + await self.broadcast_frame(UserStartedSpeakingFrame) + self._user_speaking = True + + if is_final_turn: + if message.turn_is_formatted: + self.confirm_finalize() + await self.push_frame( + TranscriptionFrame( + transcript_text, + speaker_id, + time_now_iso8601(), + language, + message, + finalized=True, + ) + ) + await self._trace_transcription(transcript_text, True, language) + await self.stop_processing_metrics() + # AAI is authoritative — emit UserStoppedSpeakingFrame immediately. + # broadcast_frame pushes downstream (same queue as TranscriptionFrame + # above, so ordering is preserved) and upstream. + await self.broadcast_frame(UserStoppedSpeakingFrame) + self._user_speaking = False + else: + await self.push_frame( + InterimTranscriptionFrame( + transcript_text, + speaker_id, + time_now_iso8601(), + language, + message, + ) ) - ) From cd07937c5dcc413000ede5bbc8a232f477a6cd25 Mon Sep 17 00:00:00 2001 From: zack Date: Thu, 26 Feb 2026 22:18:02 -0500 Subject: [PATCH 02/23] Fix missing imports: Add UserStartedSpeakingFrame and UserStoppedSpeakingFrame --- src/pipecat/services/assemblyai/stt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index edd5ac7b7..9881c145f 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -26,6 +26,8 @@ from pipecat.frames.frames import ( InterimTranscriptionFrame, StartFrame, TranscriptionFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, VADUserStartedSpeakingFrame, VADUserStoppedSpeakingFrame, ) From aa7e9a17d5ecdb3d934833e65dae56570bd8c0ca Mon Sep 17 00:00:00 2001 From: zack Date: Fri, 27 Feb 2026 14:55:22 -0500 Subject: [PATCH 03/23] Fix finalization pattern: Use request/confirm in Pipecat mode, finalized flag in STT mode - Add request_finalize() before sending ForceEndpoint in Pipecat mode - Keep confirm_finalize() when receiving formatted finals in Pipecat mode - Remove confirm_finalize() from STT mode (use finalized=True instead) This follows Pipecat's two-step finalization pattern where request_finalize() is called when sending a finalize request to the STT service, and confirm_finalize() is called when receiving confirmation back. --- src/pipecat/services/assemblyai/stt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index 9881c145f..90e292ac0 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -438,6 +438,7 @@ class AssemblyAISTTService(WebsocketSTTService): and self._websocket and self._websocket.state is State.OPEN ): + await self.request_finalize() await self._websocket.send(json.dumps({"type": "ForceEndpoint"})) await self.start_processing_metrics() @@ -761,8 +762,7 @@ class AssemblyAISTTService(WebsocketSTTService): self._user_speaking = True if is_final_turn: - if message.turn_is_formatted: - self.confirm_finalize() + # STT mode: AssemblyAI controls finalization, just mark as finalized await self.push_frame( TranscriptionFrame( transcript_text, From 6ba9f780b08eddbfb6fa2cabe3cce6fe3adc457d Mon Sep 17 00:00:00 2001 From: zack Date: Fri, 27 Feb 2026 15:00:38 -0500 Subject: [PATCH 04/23] Remove unnecessary SpeechStarted fallback in STT mode u3-rt-pro guarantees SpeechStarted is always sent before transcripts, so the fallback UserStartedSpeakingFrame broadcast is never needed. This ensures clean pairing of UserStarted/StoppedSpeakingFrame: - Start: Always from _handle_speech_started - Stop: Always from _handle_transcription on final turn --- src/pipecat/services/assemblyai/stt.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index 90e292ac0..0618a29fe 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -752,14 +752,9 @@ class AssemblyAISTTService(WebsocketSTTService): ) else: # --- Mode 2: STT turn detection --- - # SpeechStarted handles UserStartedSpeakingFrame + interruption. - # If SpeechStarted hasn't fired yet (shouldn't happen, but guard), - # broadcast here as fallback. + # SpeechStarted always arrives before transcripts with u3-rt-pro, + # so UserStartedSpeakingFrame is guaranteed to be broadcast first. logger.debug(f"{self} Transcript received in STT mode (_user_speaking={self._user_speaking})") - if not self._user_speaking: - logger.warning(f"{self} Transcript arrived before SpeechStarted, broadcasting fallback UserStartedSpeakingFrame") - await self.broadcast_frame(UserStartedSpeakingFrame) - self._user_speaking = True if is_final_turn: # STT mode: AssemblyAI controls finalization, just mark as finalized From 45532a947881817bd1150bd8695a7125b407df04 Mon Sep 17 00:00:00 2001 From: zack Date: Fri, 27 Feb 2026 16:15:49 -0500 Subject: [PATCH 05/23] Remove info logs and unused import per PR feedback - Remove unused Mapping import - Remove info logs at initialization (connection params) - Remove info logs in _handle_transcription (transcript details, text sent to LLM) - Remove info logs in _build_ws_url (WebSocket URL and params) - Keep debug logs (less verbose, appropriate for development) --- src/pipecat/services/assemblyai/stt.py | 38 ++------------------------ 1 file changed, 2 insertions(+), 36 deletions(-) diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index 0618a29fe..445ab7b73 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -13,7 +13,7 @@ WebSocket API for streaming audio transcription. import asyncio import json from dataclasses import dataclass, field -from typing import Any, AsyncGenerator, Dict, Mapping, Optional +from typing import Any, AsyncGenerator, Dict, Optional from urllib.parse import urlencode from loguru import logger @@ -211,18 +211,6 @@ class AssemblyAISTTService(WebsocketSTTService): self._user_speaking = False self._vad_speaking = False - # Log final connection params after any modifications - logger.info(f"{self} Final connection params being sent to AssemblyAI:") - logger.info(f" min_end_of_turn_silence_when_confident: {self._settings.connection_params.min_end_of_turn_silence_when_confident}") - logger.info(f" max_turn_silence: {self._settings.connection_params.max_turn_silence}") - - # Warn if min_end_of_turn_silence_when_confident is not 100ms - if self._settings.connection_params.min_end_of_turn_silence_when_confident != 100: - logger.warning( - f"For best latency, set min_end_of_turn_silence_when_confident to 100ms. " - f"Current value: {self._settings.connection_params.min_end_of_turn_silence_when_confident}ms" - ) - def _configure_pipecat_turn_mode( self, connection_params: AssemblyAIConnectionParams, is_u3_pro: bool ) -> AssemblyAIConnectionParams: @@ -461,13 +449,7 @@ class AssemblyAISTTService(WebsocketSTTService): if params: query_string = urlencode(params) - full_url = f"{self._api_endpoint_base_url}?{query_string}" - logger.info(f"{self} WebSocket URL being sent to AssemblyAI:") - logger.info(f" {full_url}") - logger.info(f" Parsed params:") - for k, v in params.items(): - logger.info(f" {k}: {v}") - return full_url + return f"{self._api_endpoint_base_url}?{query_string}" return self._api_endpoint_base_url async def _connect(self): @@ -672,17 +654,6 @@ class AssemblyAISTTService(WebsocketSTTService): - end_of_turn → TranscriptionFrame + UserStoppedSpeakingFrame - else → InterimTranscriptionFrame """ - # Log transcript details - logger.info(f"{self} ===== TRANSCRIPT RECEIVED =====") - logger.info(f" Text: \"{message.transcript}\"") - logger.info(f" end_of_turn: {message.end_of_turn}") - logger.info(f" turn_is_formatted: {message.turn_is_formatted}") - logger.info(f" turn_order: {message.turn_order}") - if message.end_of_turn_confidence is not None: - logger.info(f" end_of_turn_confidence: {message.end_of_turn_confidence}") - logger.info(f" speaker: {message.speaker}") - logger.info(f"===============================") - if not message.transcript: return @@ -709,11 +680,6 @@ class AssemblyAISTTService(WebsocketSTTService): speaker=message.speaker, text=message.transcript ) - logger.info(f"{self} 🤖 TEXT SENT TO LLM (with speaker format): \"{transcript_text}\"") - else: - logger.info(f"{self} 🤖 TEXT SENT TO LLM (speaker {message.speaker}): \"{transcript_text}\"") - else: - logger.info(f"{self} 🤖 TEXT SENT TO LLM: \"{transcript_text}\"") # Determine if this is a final turn from AssemblyAI is_final_turn = message.end_of_turn and ( From ef00f27d53a1733e5f94ca31b442edbee102d6f3 Mon Sep 17 00:00:00 2001 From: zack Date: Fri, 27 Feb 2026 17:58:05 -0500 Subject: [PATCH 06/23] Fix incorrect await on synchronous request_finalize() method The request_finalize() method in STTService is synchronous (sets a flag), but was being called with await in the VAD turn endpoint handling code. This caused "object NoneType can't be used in 'await' expression" errors. Also includes automatic formatting improvements from ruff. --- src/pipecat/services/assemblyai/models.py | 6 +-- src/pipecat/services/assemblyai/stt.py | 55 ++++++++++++++++------- 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/src/pipecat/services/assemblyai/models.py b/src/pipecat/services/assemblyai/models.py index fb883ac99..949f2e00e 100644 --- a/src/pipecat/services/assemblyai/models.py +++ b/src/pipecat/services/assemblyai/models.py @@ -152,9 +152,9 @@ class AssemblyAIConnectionParams(BaseModel): max_turn_silence: Optional[int] = None keyterms_prompt: Optional[List[str]] = None prompt: Optional[str] = None - speech_model: Literal["universal-streaming-english", "universal-streaming-multilingual", "u3-rt-pro"] = ( - "u3-rt-pro" - ) + speech_model: Literal[ + "universal-streaming-english", "universal-streaming-multilingual", "u3-rt-pro" + ] = "u3-rt-pro" language_detection: Optional[bool] = None format_turns: bool = True speaker_labels: Optional[bool] = None diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index 445ab7b73..b0254d410 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -73,7 +73,9 @@ def map_language_from_assemblyai(language_code: str) -> Language: # Try to match the language code directly return Language(language_code.lower()) except ValueError: - logger.warning(f"Unknown language code from AssemblyAI: {language_code}, defaulting to English") + logger.warning( + f"Unknown language code from AssemblyAI: {language_code}, defaulting to English" + ) return Language.EN @@ -304,7 +306,11 @@ class AssemblyAISTTService(WebsocketSTTService): return changed # If websocket is connected, send UpdateConfiguration for supported params - if self._websocket and self._websocket.state is State.OPEN and "connection_params" in changed: + if ( + self._websocket + and self._websocket.state is State.OPEN + and "connection_params" in changed + ): # Build UpdateConfiguration message update_config = {"type": "UpdateConfiguration"} conn_params = self._settings.connection_params @@ -314,7 +320,10 @@ class AssemblyAISTTService(WebsocketSTTService): # Check each potentially changed parameter if hasattr(conn_params, "keyterms_prompt"): - if old_conn_params is None or conn_params.keyterms_prompt != old_conn_params.keyterms_prompt: + if ( + old_conn_params is None + or conn_params.keyterms_prompt != old_conn_params.keyterms_prompt + ): if conn_params.keyterms_prompt is not None: update_config["keyterms_prompt"] = conn_params.keyterms_prompt logger.info(f"Updating keyterms_prompt to: {conn_params.keyterms_prompt}") @@ -332,16 +341,29 @@ class AssemblyAISTTService(WebsocketSTTService): logger.info(f"Updating prompt") if hasattr(conn_params, "max_turn_silence"): - if old_conn_params is None or conn_params.max_turn_silence != old_conn_params.max_turn_silence: + if ( + old_conn_params is None + or conn_params.max_turn_silence != old_conn_params.max_turn_silence + ): if conn_params.max_turn_silence is not None: update_config["max_turn_silence"] = conn_params.max_turn_silence - logger.info(f"Updating max_turn_silence to: {conn_params.max_turn_silence}ms") + logger.info( + f"Updating max_turn_silence to: {conn_params.max_turn_silence}ms" + ) if hasattr(conn_params, "min_end_of_turn_silence_when_confident"): - if old_conn_params is None or conn_params.min_end_of_turn_silence_when_confident != old_conn_params.min_end_of_turn_silence_when_confident: + if ( + old_conn_params is None + or conn_params.min_end_of_turn_silence_when_confident + != old_conn_params.min_end_of_turn_silence_when_confident + ): if conn_params.min_end_of_turn_silence_when_confident is not None: - update_config["min_end_of_turn_silence_when_confident"] = conn_params.min_end_of_turn_silence_when_confident - logger.info(f"Updating min_end_of_turn_silence_when_confident to: {conn_params.min_end_of_turn_silence_when_confident}ms") + update_config["min_end_of_turn_silence_when_confident"] = ( + conn_params.min_end_of_turn_silence_when_confident + ) + logger.info( + f"Updating min_end_of_turn_silence_when_confident to: {conn_params.min_end_of_turn_silence_when_confident}ms" + ) # Send update if we have parameters to update if len(update_config) > 1: # More than just "type" @@ -426,7 +448,7 @@ class AssemblyAISTTService(WebsocketSTTService): and self._websocket and self._websocket.state is State.OPEN ): - await self.request_finalize() + self.request_finalize() await self._websocket.send(json.dumps({"type": "ForceEndpoint"})) await self.start_processing_metrics() @@ -616,7 +638,9 @@ class AssemblyAISTTService(WebsocketSTTService): Only applies to Mode 2 (STT turn detection). In Mode 1, VAD + smart turn analyzer handle interruptions via the aggregator. """ - logger.debug(f"{self} SpeechStarted received (vad_force_turn_endpoint={self._vad_force_turn_endpoint})") + logger.debug( + f"{self} SpeechStarted received (vad_force_turn_endpoint={self._vad_force_turn_endpoint})" + ) if self._vad_force_turn_endpoint: logger.debug(f"{self} SpeechStarted ignored in Pipecat mode") return # Mode 1: handled by aggregator @@ -677,8 +701,7 @@ class AssemblyAISTTService(WebsocketSTTService): # Format transcript with speaker labels if format string provided if self._speaker_format: transcript_text = self._speaker_format.format( - speaker=message.speaker, - text=message.transcript + speaker=message.speaker, text=message.transcript ) # Determine if this is a final turn from AssemblyAI @@ -693,7 +716,7 @@ class AssemblyAISTTService(WebsocketSTTService): finalize_confirmed = bool(message.turn_is_formatted) if finalize_confirmed: self.confirm_finalize() - logger.debug(f"{self} Final transcript: \"{transcript_text}\"") + logger.debug(f'{self} Final transcript: "{transcript_text}"') await self.push_frame( TranscriptionFrame( transcript_text, @@ -706,7 +729,7 @@ class AssemblyAISTTService(WebsocketSTTService): await self._trace_transcription(transcript_text, True, language) await self.stop_processing_metrics() else: - logger.debug(f"{self} Interim transcript: \"{transcript_text}\"") + logger.debug(f'{self} Interim transcript: "{transcript_text}"') await self.push_frame( InterimTranscriptionFrame( transcript_text, @@ -720,7 +743,9 @@ class AssemblyAISTTService(WebsocketSTTService): # --- Mode 2: STT turn detection --- # SpeechStarted always arrives before transcripts with u3-rt-pro, # so UserStartedSpeakingFrame is guaranteed to be broadcast first. - logger.debug(f"{self} Transcript received in STT mode (_user_speaking={self._user_speaking})") + logger.debug( + f"{self} Transcript received in STT mode (_user_speaking={self._user_speaking})" + ) if is_final_turn: # STT mode: AssemblyAI controls finalization, just mark as finalized From d7ce1eedd956ea320d7fb980172494cb1aa8c3e2 Mon Sep 17 00:00:00 2001 From: zack Date: Fri, 27 Feb 2026 17:58:18 -0500 Subject: [PATCH 07/23] Add foundational examples for AssemblyAI u3-rt-pro - 07o-interruptible-assemblyai.py: Basic example using Pipecat VAD mode - 07o-interruptible-assemblyai-stt.py: Advanced example using STT-controlled turn detection with comprehensive documentation on u3-rt-pro features (turn detection tuning, prompt-based enhancement, speaker diarization) --- .../07o-interruptible-assemblyai-stt.py | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 examples/foundational/07o-interruptible-assemblyai-stt.py diff --git a/examples/foundational/07o-interruptible-assemblyai-stt.py b/examples/foundational/07o-interruptible-assemblyai-stt.py new file mode 100644 index 000000000..5deaa82a1 --- /dev/null +++ b/examples/foundational/07o-interruptible-assemblyai-stt.py @@ -0,0 +1,175 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.assemblyai.models import AssemblyAIConnectionParams +from pipecat.services.assemblyai.stt import AssemblyAISTTService +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams +from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams +from pipecat.turns.user_turn_strategies import ExternalUserTurnStrategies + +load_dotenv(override=True) + + +# We use lambdas to defer transport parameter creation until the transport +# type is selected at runtime. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + """AssemblyAI u3-rt-pro STT Example with STT-Controlled Turn Detection + + This example demonstrates using AssemblyAI's u3-rt-pro Speech-to-Text model + with STT-controlled turn detection for more natural conversation flow. + + Key features: + + 1. STT-Controlled Turn Detection + - Set `vad_force_turn_endpoint=False` to enable STT mode + - AssemblyAI's model determines when user starts/stops speaking + - Uses `ExternalUserTurnStrategies` instead of Pipecat's VAD + - More natural turn detection based on speech patterns and pauses + + 2. Advanced Turn Detection Tuning (STT Mode) + - `min_end_of_turn_silence_when_confident`: Minimum silence (ms) when confident + about end-of-turn. Lower values = faster responses. Default: 200ms + - `max_turn_silence`: Maximum silence (ms) before forcing end-of-turn. + Prevents long pauses. Default: 1000ms + + 3. Prompt-Based Transcription Enhancement + - Use `prompt` parameter to improve accuracy for specific names/terms + - Particularly useful for proper nouns, technical terms, domain vocabulary + - Example: "Names: Xiomara, Saoirse, Krzystof. Technical terms: API, OAuth." + + 4. Speaker Diarization (Optional) + - Enable with `speaker_labels=True` + - Automatically identifies different speakers in multi-party conversations + - TranscriptionFrame includes speaker_id field (e.g., "Speaker A", "Speaker B") + + 5. Language Detection (Optional, multilingual model only) + - Enable with `language_detection=True` + - Automatically detects spoken language + - Available with universal-streaming-multilingual model + + For more information: https://www.assemblyai.com/docs/speech-to-text/streaming + """ + logger.info(f"Starting bot") + + stt = AssemblyAISTTService( + api_key=os.getenv("ASSEMBLYAI_API_KEY"), + vad_force_turn_endpoint=False, # Enable STT-controlled turn detection + connection_params=AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + # Optional: Tune turn detection timing (defaults shown below) + # min_end_of_turn_silence_when_confident=100, # Default + # max_turn_silence=1000, # Default + # Optional: Boost accuracy for specific names/terms + # prompt="Names: Xiomara, Saoirse, Krzystof. Technical terms: API, OAuth.", + # Optional: Enable speaker diarization + # speaker_labels=True, + ), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = LLMContext(messages) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(user_turn_strategies=ExternalUserTurnStrategies()), + ) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + user_aggregator, # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + assistant_aggregator, # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() From 21a409e447fd23b984e4c4da7e1171c510d0d86f Mon Sep 17 00:00:00 2001 From: zack Date: Sun, 1 Mar 2026 11:17:39 -0500 Subject: [PATCH 08/23] Update prompt warning and rename min_end_of_turn_silence_when_confident to min_turn_silence - Add "beta feature" note to custom prompt warning - Rename min_end_of_turn_silence_when_confident parameter to min_turn_silence across all AssemblyAI code - Update documentation, examples, and test files to use new parameter name --- TESTING_CHECKLIST.md | 273 +++++++ TESTING_SETUP.md | 310 ++++++++ .../07o-interruptible-assemblyai-stt.py | 4 +- src/pipecat/services/assemblyai/models.py | 4 +- src/pipecat/services/assemblyai/stt.py | 45 +- test_assemblyai_custom.py | 256 ++++++ test_assemblyai_interactive.py | 750 ++++++++++++++++++ test_assemblyai_u3pro.py | 589 ++++++++++++++ 8 files changed, 2205 insertions(+), 26 deletions(-) create mode 100644 TESTING_CHECKLIST.md create mode 100644 TESTING_SETUP.md create mode 100755 test_assemblyai_custom.py create mode 100755 test_assemblyai_interactive.py create mode 100644 test_assemblyai_u3pro.py diff --git a/TESTING_CHECKLIST.md b/TESTING_CHECKLIST.md new file mode 100644 index 000000000..8d2d147d4 --- /dev/null +++ b/TESTING_CHECKLIST.md @@ -0,0 +1,273 @@ +# AssemblyAI u3-rt-pro Testing Checklist + +## Test Environment Setup +- [ ] Install dependencies: `uv sync --group dev --all-extras` +- [ ] Set up `.env` file with API keys +- [ ] Verify LiveKit connection +- [ ] Run basic voice agent test + +--- + +## Feature Testing Checklist + +### ✅ Basic Configuration Tests + +#### Test 1: Default u3-rt-pro Configuration +- [ ] **Setup:** Create service with default params +- [ ] **Expected:** No errors, uses u3-rt-pro model with 100ms min/max +- [ ] **Verify:** Check logs for connection confirmation + +#### Test 2: Custom min_turn_silence +- [ ] **Setup:** Set `min_turn_silence=200` +- [ ] **Expected:** Both min and max set to 200ms +- [ ] **Verify:** Speak short phrases, observe turn detection timing + +#### Test 3: User sets max_turn_silence (Warning Test) +- [ ] **Setup:** Set `max_turn_silence=500` in connection params +- [ ] **Expected:** Warning logged, value overridden to match min +- [ ] **Verify:** Check logs for warning message + +--- + +### ✅ Prompting Tests + +#### Test 4: No Prompt (Default - Recommended) +- [ ] **Setup:** Don't set prompt parameter +- [ ] **Expected:** Uses default prompt, 88% accuracy, no warnings +- [ ] **Verify:** Transcription quality is good + +#### Test 5: Custom Prompt (Warning Test) +- [ ] **Setup:** Set custom prompt in connection params +- [ ] **Expected:** Warning logged about testing without prompt first +- [ ] **Verify:** Check logs for prompt warning + +#### Test 6: Prompt + Keyterms Conflict (Error Test) +- [ ] **Setup:** Set both `prompt` and `keyterms_prompt` at init +- [ ] **Expected:** ValueError raised with helpful error message +- [ ] **Verify:** Service fails to initialize with clear error + +--- + +### ✅ Keyterms Prompting Tests + +#### Test 7: Basic Keyterms at Init +- [ ] **Setup:** Set `keyterms_prompt=["Pipecat", "AssemblyAI", "Universal-3"]` +- [ ] **Expected:** Terms are boosted in recognition +- [ ] **Verify:** Say the boosted terms, check accuracy + +#### Test 8: Empty Keyterms (No Boosting) +- [ ] **Setup:** Set `keyterms_prompt=[]` +- [ ] **Expected:** No boosting, default behavior +- [ ] **Verify:** Normal transcription + +--- + +### ✅ Diarization Tests + +#### Test 9: Diarization Disabled (Default) +- [ ] **Setup:** Don't set `speaker_labels` parameter +- [ ] **Expected:** No speaker info in transcripts +- [ ] **Verify:** TranscriptionFrame.user_id is default user_id + +#### Test 10: Diarization Enabled (No Formatting) +- [ ] **Setup:** Set `speaker_labels=True` +- [ ] **Expected:** Speaker ID in user_id field, plain text +- [ ] **Verify:** Multiple speakers show different IDs (Speaker A, Speaker B) + +#### Test 11: Diarization with XML Formatting +- [ ] **Setup:** Set `speaker_labels=True`, `speaker_format="<{speaker}>{text}"` +- [ ] **Expected:** Text includes speaker tags: `Hello` +- [ ] **Verify:** Formatted text in transcript, speaker ID in user_id + +#### Test 12: Diarization with Colon Prefix +- [ ] **Setup:** Set `speaker_labels=True`, `speaker_format="{speaker}: {text}"` +- [ ] **Expected:** Text includes prefix: `Speaker A: Hello` +- [ ] **Verify:** Formatted text, multiple speakers distinguishable + +--- + +### ✅ Dynamic Updates Tests + +#### Test 13: Dynamic Keyterms Update (Stage 1 → Stage 2) +- [ ] **Setup:** Start with empty keyterms, update mid-conversation +- [ ] **Expected:** New keyterms take effect immediately +- [ ] **Test Steps:** + 1. Start conversation with no keyterms + 2. Send update frame with `keyterms_prompt=["cardiology", "Dr. Smith"]` + 3. Say the new terms +- [ ] **Verify:** Improved recognition after update + +#### Test 14: Clear Keyterms (Reset Context) +- [ ] **Setup:** Start with keyterms, clear them mid-stream +- [ ] **Expected:** Context biasing removed +- [ ] **Test Steps:** + 1. Start with `keyterms_prompt=["test", "words"]` + 2. Send update frame with `keyterms_prompt=[]` +- [ ] **Verify:** No more boosting after clear + +#### Test 15: Dynamic Silence Parameters +- [ ] **Setup:** Update `max_turn_silence` mid-stream +- [ ] **Expected:** Turn detection timing changes +- [ ] **Test Steps:** + 1. Start with default (1200ms) + 2. Update to `max_turn_silence=5000` (for reading numbers) + 3. Pause longer between words + 4. Update back to `max_turn_silence=1200` +- [ ] **Verify:** Longer pauses tolerated when increased + +#### Test 16: Dynamic Prompt Update +- [ ] **Setup:** Update prompt mid-stream +- [ ] **Expected:** New instructions take effect +- [ ] **Test Steps:** + 1. Start with default prompt + 2. Send update with custom prompt +- [ ] **Verify:** Behavior changes according to new prompt + +#### Test 17: Multiple Parameters at Once +- [ ] **Setup:** Update keyterms, max_turn_silence, and min_end_of_turn together +- [ ] **Expected:** All parameters updated in single WebSocket message +- [ ] **Verify:** Check logs for single UpdateConfiguration message + +#### Test 18: Dynamic Update - Prompt + Keyterms Conflict (Error) +- [ ] **Setup:** Try to update both prompt and keyterms_prompt in same update +- [ ] **Expected:** ValueError raised +- [ ] **Verify:** Update fails with clear error message + +--- + +### ✅ Turn Detection Mode Tests + +#### Test 19: Pipecat Mode (vad_force_turn_endpoint=True) - Default +- [ ] **Setup:** Use default settings (Pipecat mode) +- [ ] **Expected:** + - ForceEndpoint sent on VAD stop + - Smart Turn Analyzer makes decisions + - min=max=100ms for u3-rt-pro +- [ ] **Verify:** Fast finals, Smart Turn handles completeness + +#### Test 20: STT Mode (vad_force_turn_endpoint=False) - u3-rt-pro only +- [ ] **Setup:** Set `vad_force_turn_endpoint=False` with u3-rt-pro +- [ ] **Expected:** + - AssemblyAI controls turn endings + - SpeechStarted message triggers interruptions + - UserStarted/StoppedSpeakingFrame emitted +- [ ] **Verify:** Turn detection from AssemblyAI model + +#### Test 21: STT Mode with universal-streaming (Error Test) +- [ ] **Setup:** Set `vad_force_turn_endpoint=False` with universal-streaming +- [ ] **Expected:** ValueError raised (requires u3-rt-pro) +- [ ] **Verify:** Service fails with clear error + +--- + +### ✅ Language Detection Tests (If Multilingual Model) + +#### Test 22: Language Detection Enabled +- [ ] **Setup:** Use `universal-streaming-multilingual` with `language_detection=True` +- [ ] **Expected:** Language codes in transcripts +- [ ] **Verify:** Speak different languages, check language_code field + +#### Test 23: Language Confidence Threshold +- [ ] **Setup:** Enable language detection +- [ ] **Expected:** High confidence (≥0.7) → detected language, Low → fallback to English +- [ ] **Verify:** Check logs for confidence warnings + +--- + +### ✅ Edge Cases & Error Handling + +#### Test 24: WebSocket Disconnect During Update +- [ ] **Setup:** Simulate disconnect, try update +- [ ] **Expected:** Error logged, update queued for reconnection +- [ ] **Verify:** Graceful handling, no crash + +#### Test 25: Invalid Parameter Types +- [ ] **Setup:** Send update with wrong type (e.g., keyterms_prompt as string) +- [ ] **Expected:** Warning logged, parameter skipped +- [ ] **Verify:** Service continues, invalid param ignored + +#### Test 26: Unknown Parameter in Update +- [ ] **Setup:** Send update with unsupported parameter (e.g., `language`) +- [ ] **Expected:** Warning logged about parameter +- [ ] **Verify:** Other valid params still updated + +--- + +### ✅ Integration Tests + +#### Test 27: Full Voice Agent Flow (Multi-Stage) +- [ ] **Setup:** Complete voice agent with stage transitions +- [ ] **Test Steps:** + 1. Greeting stage (general keyterms) + 2. Name collection stage (name keyterms) + 3. Account number stage (number keyterms, longer silence) + 4. Medical info stage (medical keyterms) + 5. Closing stage (goodbye keyterms) +- [ ] **Verify:** Each stage has appropriate keyterms and timing + +#### Test 28: Diarization + Dynamic Updates +- [ ] **Setup:** Enable diarization, update keyterms mid-stream +- [ ] **Expected:** Both features work together +- [ ] **Verify:** Speaker IDs persist, keyterms update correctly + +#### Test 29: Interruption Handling +- [ ] **Setup:** Bot speaking, user interrupts +- [ ] **Expected:** + - Pipecat mode: VAD + Smart Turn handles + - STT mode: SpeechStarted triggers interrupt +- [ ] **Verify:** Bot stops, user speech processed + +--- + +## Testing Results Template + +``` +| Test # | Feature | Status | Notes | +|--------|---------|--------|-------| +| 1 | Default Config | ✅ PASS | | +| 2 | Custom min_silence | ✅ PASS | | +| 3 | max_silence Warning | ✅ PASS | | +| ... | ... | ... | ... | +``` + +--- + +## Expected Outcomes Summary + +### ✅ Should Work (No Errors) +- Default configuration +- Custom min_turn_silence +- Keyterms prompting +- Diarization with/without formatting +- Dynamic updates (one parameter or multiple) +- Pipecat mode turn detection + +### ⚠️ Should Warn (Logs Warning, Continues) +- Custom prompt set at init +- max_turn_silence set (overridden) +- Invalid parameter types in updates +- Language update attempted +- Prompt used with universal-streaming + +### ❌ Should Error (Raises Exception, Stops) +- prompt + keyterms_prompt at init +- prompt + keyterms_prompt in same update +- vad_force_turn_endpoint=False with universal-streaming + +--- + +## Quick Test Commands + +```bash +# Run basic test +python test_assemblyai_u3pro.py --test basic + +# Run specific test +python test_assemblyai_u3pro.py --test diarization + +# Run all tests +python test_assemblyai_u3pro.py --test all + +# Interactive mode +python test_assemblyai_u3pro.py --interactive +``` diff --git a/TESTING_SETUP.md b/TESTING_SETUP.md new file mode 100644 index 000000000..fa1dca462 --- /dev/null +++ b/TESTING_SETUP.md @@ -0,0 +1,310 @@ +# AssemblyAI u3-rt-pro Testing Setup Guide + +## Quick Start + +### 1. Setup Environment + +```bash +# Copy API keys +cp .env.testing .env + +# Install dependencies +uv sync --group dev --all-extras --no-extra gstreamer --no-extra krisp + +# Make test script executable +chmod +x test_assemblyai_u3pro.py +``` + +### 2. Ensure Audio Devices + +Make sure you have: +- **Microphone** enabled and working +- **Speakers/headphones** connected +- Audio permissions granted (macOS will prompt on first run) + +### 3. Run Tests + +```bash +# Run a specific test +python test_assemblyai_u3pro.py --test basic + +# Interactive mode (choose from menu) +python test_assemblyai_u3pro.py --interactive + +# Run all tests sequentially +python test_assemblyai_u3pro.py --test all +``` + +--- + +## Available Tests + +### Basic Configuration Tests +```bash +# Test 1: Default configuration (min=max=100ms) +python test_assemblyai_u3pro.py --test basic + +# Test 2: Custom min_turn_silence +python test_assemblyai_u3pro.py --test custom_min + +# Test 3: max_turn_silence warning (should be overridden) +python test_assemblyai_u3pro.py --test max_warning +``` + +### Prompting Tests +```bash +# Test 5: Custom prompt warning +python test_assemblyai_u3pro.py --test prompt_warning + +# Test 6: Prompt + keyterms conflict (should error) +python test_assemblyai_u3pro.py --test prompt_keyterms_conflict + +# Test 7: Basic keyterms prompting +python test_assemblyai_u3pro.py --test keyterms +``` + +### Diarization Tests +```bash +# Test 10: Diarization without formatting +python test_assemblyai_u3pro.py --test diarization + +# Test 11: Diarization with XML formatting +python test_assemblyai_u3pro.py --test diarization_xml +``` + +### Dynamic Updates Tests +```bash +# Test 13: Dynamic keyterms (multi-stage) +python test_assemblyai_u3pro.py --test dynamic_keyterms + +# Test 15: Dynamic silence parameters +python test_assemblyai_u3pro.py --test dynamic_silence + +# Test 17: Multiple parameters at once +python test_assemblyai_u3pro.py --test multi_param +``` + +--- + +## Test Execution Flow + +### For Each Test: + +1. **Start the test script** + ```bash + python test_assemblyai_u3pro.py --test + ``` + +2. **Wait for "started" message** indicating the bot is ready + +3. **Speak into your microphone** to test - the bot will: + - Transcribe your speech (you'll see `📝 TRANSCRIPTION:` logs) + - Process through the LLM + - Respond with voice through your speakers + +4. **Observe logs** for: + - ✅ Success indicators + - ⚠️ Warning messages + - ❌ Error messages + - 📝 Transcription output + +5. **Verify expected behavior** against checklist + +6. **Stop test** with Ctrl+C + +--- + +## Expected Test Outcomes + +### Should Pass (✅) +- Basic configuration creates service +- Custom parameters are applied +- Keyterms boost recognition +- Diarization shows speaker IDs +- Dynamic updates work without errors + +### Should Warn (⚠️) +Check logs for warnings: +- "We recommend testing at first with no prompt" +- "max_turn_silence is not used in Pipecat mode" +- "Unknown setting for AssemblyAI STT service" + +### Should Error (❌) +Should raise ValueError and fail to start: +- Both prompt and keyterms_prompt set at init +- Both prompt and keyterms_prompt in same update +- vad_force_turn_endpoint=False with universal-streaming + +--- + +## Debugging Tips + +### Check Logs +```bash +# Run with verbose logging +LOGURU_LEVEL=DEBUG python test_assemblyai_u3pro.py --test +``` + +### Common Issues + +**Issue: "WebSocket connection failed"** +- Check ASSEMBLYAI_API_KEY is correct +- Verify network connection +- Check firewall settings + +**Issue: "No audio input/output"** +- Verify microphone permissions (System Preferences → Security & Privacy → Microphone) +- Check default audio devices in System Preferences → Sound +- Test microphone with another app first +- Make sure no other app is using the microphone + +**Issue: "No transcriptions appearing"** +- Verify microphone permissions +- Check audio levels (speak louder or move closer to mic) +- Speak clearly and wait for VAD to detect +- Check if microphone is muted + +**Issue: "Can't hear bot responses"** +- Check speaker/headphone volume +- Verify correct output device is selected +- Check terminal for TTS errors + +**Issue: "Service fails to start"** +- Check all API keys in .env +- Run `uv sync` to ensure dependencies installed +- Check Python version (3.10+) + +--- + +## Manual Testing Checklist + +After running automated tests, manually verify: + +### ✅ Audio Quality +- [ ] Transcriptions are accurate +- [ ] No distortion or dropouts +- [ ] Latency is acceptable + +### ✅ Turn Detection +- [ ] Bot waits for user to finish speaking +- [ ] No premature cutoffs +- [ ] Handles natural pauses correctly + +### ✅ Interruptions +- [ ] Can interrupt bot mid-sentence +- [ ] Interruption is smooth +- [ ] Bot stops speaking immediately + +### ✅ Diarization (if enabled) +- [ ] Multiple speakers detected correctly +- [ ] Speaker IDs consistent +- [ ] Speaker formatting works + +### ✅ Dynamic Updates +- [ ] Keyterms update without disconnection +- [ ] Turn detection timing changes work +- [ ] Updates logged correctly + +--- + +## Test Results Recording + +### Use this template: + +```markdown +## Test Run: YYYY-MM-DD + +| Test # | Test Name | Status | Notes | +|--------|-----------|--------|-------| +| 1 | basic | ✅ PASS | Transcriptions working | +| 2 | custom_min | ✅ PASS | Turn timing changed | +| 3 | max_warning | ✅ PASS | Warning logged | +| 5 | prompt_warning | ✅ PASS | Warning shown | +| 6 | prompt_keyterms_conflict | ✅ PASS | ValueError raised | +| 7 | keyterms | ✅ PASS | Terms boosted | +| 10 | diarization | ✅ PASS | Speaker IDs correct | +| 11 | diarization_xml | ✅ PASS | XML tags shown | +| 13 | dynamic_keyterms | ✅ PASS | Updates worked | +| 15 | dynamic_silence | ✅ PASS | Timing adjusted | +| 17 | multi_param | ✅ PASS | All params updated | + +### Issues Found: +- None + +### Notes: +- All tests passed successfully +- Latency is excellent (sub-300ms) +- Diarization accuracy is good +``` + +--- + +## Advanced Testing + +### Custom Test Scenarios + +Create custom tests by modifying `test_assemblyai_u3pro.py`: + +```python +async def test_my_custom_scenario(): + """My custom test scenario.""" + logger.info("Testing my specific use case") + + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + # Your custom params here + ) + + task, transport = await create_basic_voice_agent(connection_params) + + # Your test logic here + + runner = PipelineRunner() + await runner.run(task) +``` + +### Stress Testing + +Test with: +- Multiple simultaneous speakers +- Long conversations (30+ minutes) +- Rapid speech +- Heavy accents +- Background noise +- Poor network conditions + +--- + +## Reporting Issues + +When reporting issues, include: + +1. **Test name and number** +2. **Full error message and stack trace** +3. **Relevant log output** (use LOGURU_LEVEL=DEBUG) +4. **Configuration used** (connection_params) +5. **Expected vs actual behavior** +6. **Steps to reproduce** + +--- + +## Next Steps + +After testing: + +1. ✅ Mark completed tests in `TESTING_CHECKLIST.md` +2. 📝 Document any issues found +3. 🐛 Create GitHub issues for bugs +4. ✨ Suggest improvements +5. 📊 Share results with team + +--- + +## Contact + +Questions? Issues? +- Check `TESTING_CHECKLIST.md` for detailed test descriptions +- Review logs with `LOGURU_LEVEL=DEBUG` +- Reach out to the team with your findings + +Happy testing! 🎯 diff --git a/examples/foundational/07o-interruptible-assemblyai-stt.py b/examples/foundational/07o-interruptible-assemblyai-stt.py index 5deaa82a1..ee2994c0e 100644 --- a/examples/foundational/07o-interruptible-assemblyai-stt.py +++ b/examples/foundational/07o-interruptible-assemblyai-stt.py @@ -66,7 +66,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - More natural turn detection based on speech patterns and pauses 2. Advanced Turn Detection Tuning (STT Mode) - - `min_end_of_turn_silence_when_confident`: Minimum silence (ms) when confident + - `min_turn_silence`: Minimum silence (ms) when confident about end-of-turn. Lower values = faster responses. Default: 200ms - `max_turn_silence`: Maximum silence (ms) before forcing end-of-turn. Prevents long pauses. Default: 1000ms @@ -96,7 +96,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): connection_params=AssemblyAIConnectionParams( speech_model="u3-rt-pro", # Optional: Tune turn detection timing (defaults shown below) - # min_end_of_turn_silence_when_confident=100, # Default + # min_turn_silence=100, # Default # max_turn_silence=1000, # Default # Optional: Boost accuracy for specific names/terms # prompt="Names: Xiomara, Saoirse, Krzystof. Technical terms: API, OAuth.", diff --git a/src/pipecat/services/assemblyai/models.py b/src/pipecat/services/assemblyai/models.py index 949f2e00e..f92b1b8bf 100644 --- a/src/pipecat/services/assemblyai/models.py +++ b/src/pipecat/services/assemblyai/models.py @@ -129,7 +129,7 @@ class AssemblyAIConnectionParams(BaseModel): formatted_finals: Whether to enable transcript formatting. Defaults to True. word_finalization_max_wait_time: Maximum time to wait for word finalization in milliseconds. end_of_turn_confidence_threshold: Confidence threshold for end-of-turn detection. - min_end_of_turn_silence_when_confident: Minimum silence duration when confident about end-of-turn. + min_turn_silence: Minimum silence duration when confident about end-of-turn. max_turn_silence: Maximum silence duration before forcing end-of-turn. keyterms_prompt: List of key terms to guide transcription. Will be JSON serialized before sending. prompt: Optional text prompt to guide the transcription. Only used when speech_model is "u3-rt-pro". @@ -148,7 +148,7 @@ class AssemblyAIConnectionParams(BaseModel): formatted_finals: bool = True word_finalization_max_wait_time: Optional[int] = None end_of_turn_confidence_threshold: Optional[float] = None - min_end_of_turn_silence_when_confident: Optional[int] = None + min_turn_silence: Optional[int] = None max_turn_silence: Optional[int] = None keyterms_prompt: Optional[List[str]] = None prompt: Optional[str] = None diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index b0254d410..d82b38a99 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -127,8 +127,8 @@ class AssemblyAISTTService(WebsocketSTTService): vad_force_turn_endpoint: Controls turn detection mode. When True (Pipecat mode, default): Forces AssemblyAI to return finals ASAP so Pipecat's turn detection (e.g., Smart Turn) decides when the user is done. - - min_end_of_turn_silence_when_confident defaults to 100ms (user can override) - - max_turn_silence is ALWAYS set equal to min_end_of_turn_silence_when_confident + - min_turn_silence defaults to 100ms (user can override) + - max_turn_silence is ALWAYS set equal to min_turn_silence - VAD stop sends ForceEndpoint as ceiling - No UserStarted/StoppedSpeakingFrame emitted from STT When False (STT mode, u3-rt-pro only): AssemblyAI's model controls turn endings. @@ -172,10 +172,11 @@ class AssemblyAISTTService(WebsocketSTTService): # Warn if user sets a custom prompt (recommend testing without one first) if connection_params.prompt is not None: logger.warning( - "Custom prompt detected. We recommend testing with no prompt first, as this " - "will use our optimized default prompt for voice agents. Bad prompts may lead " - "to bad results. If you'd like to create your own prompt, check out our " - "prompting guide at: https://www.assemblyai.com/docs/streaming/prompting" + "Custom prompt detected. Prompting is a beta feature. We recommend testing " + "with no prompt first, as this will use our optimized default prompt for " + "voice agents. Bad prompts may lead to bad results. If you'd like to create " + "your own prompt, check out our prompting guide at: " + "https://www.assemblyai.com/docs/streaming/prompting" ) # When vad_force_turn_endpoint is enabled, configure connection params @@ -223,15 +224,15 @@ class AssemblyAISTTService(WebsocketSTTService): when the user is done speaking. VAD stop is the absolute ceiling. u3-rt-pro: - - min_end_of_turn_silence_when_confident defaults to 100ms (user can override) - - max_turn_silence is ALWAYS set equal to min_end_of_turn_silence_when_confident + - min_turn_silence defaults to 100ms (user can override) + - max_turn_silence is ALWAYS set equal to min_turn_silence to avoid double turn detection (AssemblyAI + Pipecat both analyzing) - If user sets max_turn_silence, it's ignored with a warning - end_of_turn_confidence_threshold: not set (API default) universal-streaming-*: - end_of_turn_confidence_threshold=0.0 (disable semantic turn detection) - - min_end_of_turn_silence_when_confident=160 + - min_turn_silence=160 - max_turn_silence: not set (API default) Args: @@ -244,8 +245,8 @@ class AssemblyAISTTService(WebsocketSTTService): updates = {} if is_u3_pro: - # u3-rt-pro: Synchronize max_turn_silence with min_end_of_turn_silence_when_confident - min_silence = connection_params.min_end_of_turn_silence_when_confident + # u3-rt-pro: Synchronize max_turn_silence with min_turn_silence + min_silence = connection_params.min_turn_silence if min_silence is None: min_silence = 100 @@ -254,20 +255,20 @@ class AssemblyAISTTService(WebsocketSTTService): logger.warning( f"Your max_turn_silence value ({connection_params.max_turn_silence}ms) will be " f"OVERRIDDEN in Pipecat mode (vad_force_turn_endpoint=True). It will be set to " - f"{min_silence}ms (matching min_end_of_turn_silence_when_confident) and SENT to " + f"{min_silence}ms (matching min_turn_silence) and SENT to " f"AssemblyAI to avoid double turn detection. To use your max_turn_silence as-is, " f"switch to STT mode (vad_force_turn_endpoint=False)." ) updates = { - "min_end_of_turn_silence_when_confident": min_silence, + "min_turn_silence": min_silence, "max_turn_silence": min_silence, } else: # universal-streaming: Different configuration (works differently) updates = { "end_of_turn_confidence_threshold": 0.0, - "min_end_of_turn_silence_when_confident": 160, + "min_turn_silence": 160, } # Apply updates if any @@ -292,7 +293,7 @@ class AssemblyAISTTService(WebsocketSTTService): - keyterms_prompt: List of terms to boost (can be empty array to clear) - prompt: Custom prompt text (u3-rt-pro only) - max_turn_silence: Maximum silence before forcing turn end - - min_end_of_turn_silence_when_confident: Silence before EOT check + - min_turn_silence: Silence before EOT check Args: delta: A :class:`STTSettings` (or ``AssemblyAISTTSettings``) delta. @@ -351,18 +352,18 @@ class AssemblyAISTTService(WebsocketSTTService): f"Updating max_turn_silence to: {conn_params.max_turn_silence}ms" ) - if hasattr(conn_params, "min_end_of_turn_silence_when_confident"): + if hasattr(conn_params, "min_turn_silence"): if ( old_conn_params is None - or conn_params.min_end_of_turn_silence_when_confident - != old_conn_params.min_end_of_turn_silence_when_confident + or conn_params.min_turn_silence + != old_conn_params.min_turn_silence ): - if conn_params.min_end_of_turn_silence_when_confident is not None: - update_config["min_end_of_turn_silence_when_confident"] = ( - conn_params.min_end_of_turn_silence_when_confident + if conn_params.min_turn_silence is not None: + update_config["min_turn_silence"] = ( + conn_params.min_turn_silence ) logger.info( - f"Updating min_end_of_turn_silence_when_confident to: {conn_params.min_end_of_turn_silence_when_confident}ms" + f"Updating min_turn_silence to: {conn_params.min_turn_silence}ms" ) # Send update if we have parameters to update diff --git a/test_assemblyai_custom.py b/test_assemblyai_custom.py new file mode 100755 index 000000000..c406918c0 --- /dev/null +++ b/test_assemblyai_custom.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""Custom AssemblyAI u3-rt-pro Test Script +Easy parameter tweaking for experimentation + +Edit the CONFIGURATION section below to test different settings! +""" + +import asyncio +import os +import sys + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.services.assemblyai.models import AssemblyAIConnectionParams +from pipecat.services.assemblyai.stt import AssemblyAISTTService +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.local.audio import LocalAudioTransport, LocalAudioTransportParams + +load_dotenv(override=True) + +# ============================================================================ +# CONFIGURATION +# ============================================================================ + +# Log Level: "DEBUG" for detailed logs, "INFO" for normal operation +LOG_LEVEL = "INFO" + +# ============================================================================ +# BOT IMPLEMENTATION +# ============================================================================ + + +async def main(): + """Run the custom test bot with your configured parameters.""" + # Setup logging + logger.remove(0) + logger.add(sys.stderr, level=LOG_LEVEL) + + logger.info("="*80) + logger.info("AssemblyAI u3-rt-pro Custom Test") + logger.info("="*80) + logger.info("Starting bot... Speak after you hear the greeting!") + logger.info("="*80) + + # Create local audio transport + transport = LocalAudioTransport( + LocalAudioTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ) + ) + + # ======================================================================== + # EDIT PARAMETERS HERE + # ======================================================================== + + # Build connection params + connection_params = AssemblyAIConnectionParams( + # ==================================================================== + # Model Selection + # ==================================================================== + speech_model="u3-rt-pro", + # speech_model="universal-streaming-english", + # speech_model="universal-streaming-multilingual", + + # ==================================================================== + # Turn Detection Timing + # ==================================================================== + + # Minimum silence when confident about end of turn (milliseconds) + # Default: 100ms | Higher = more patient | Lower = faster responses + # Only used in Pipecat mode (vad_force_turn_endpoint=True) + min_turn_silence=100000, + # min_turn_silence=200, + # min_turn_silence=300, + + # Maximum turn silence (milliseconds) + # WARNING: In Pipecat mode (vad_force_turn_endpoint=True), this is + # automatically set equal to min_turn_silence + # to avoid double turn detection. Only used as-is in STT mode. + max_turn_silence=500, + + # End of turn confidence threshold (0.0 to 1.0) + # Higher = requires more confidence before ending turn + # end_of_turn_confidence_threshold=0.8, + + # ==================================================================== + # Prompting & Boosting + # ==================================================================== + + # Custom Prompt (WARNING: test carefully, default is optimized!) + # None = Use AssemblyAI's optimized default (recommended for 88% accuracy) + prompt=None, + # prompt="Transcribe speech with focus on technical terms.", + # prompt="Context: Medical conversation. Transcribe accurately.", + + # Keyterms Prompting (boosts recognition for specific words) + # NOTE: Cannot use both prompt and keyterms_prompt! + keyterms_prompt=None, + # keyterms_prompt=["Pipecat", "AssemblyAI", "OpenAI", "Cartesia"], + # keyterms_prompt=["Python", "JavaScript", "TypeScript", "API"], + + # ==================================================================== + # Diarization (Speaker Identification) + # ==================================================================== + + # Enable speaker labels (identifies different speakers) + speaker_labels=None, # None or True + # speaker_labels=True, + + # ==================================================================== + # Audio Configuration + # ==================================================================== + + # Audio sample rate (Hz) + # sample_rate=16000, + # sample_rate=8000, + + # Audio encoding format + # encoding="pcm_s16le", # Default: 16-bit PCM + # encoding="pcm_mulaw", # μ-law encoding (telephony) + + # ==================================================================== + # Other Options + # ==================================================================== + + # Format transcript turns (applies formatting rules) + # format_turns=True, # Default + # format_turns=False, + + # Language detection (only for universal-streaming-multilingual) + # language_detection=True, + ) + + # Log connection parameters for debugging + logger.info("="*80) + logger.info("CONNECTION PARAMETERS:") + logger.info(f" speech_model: {connection_params.speech_model}") + logger.info(f" min_turn_silence: {connection_params.min_turn_silence}") + logger.info(f" max_turn_silence: {connection_params.max_turn_silence}") + logger.info(f" sample_rate: {connection_params.sample_rate}") + logger.info(f" encoding: {connection_params.encoding}") + logger.info(f" prompt: {connection_params.prompt}") + logger.info(f" keyterms_prompt: {connection_params.keyterms_prompt}") + logger.info(f" speaker_labels: {connection_params.speaker_labels}") + logger.info(f" format_turns: {connection_params.format_turns}") + logger.info(f" end_of_turn_confidence_threshold: {connection_params.end_of_turn_confidence_threshold}") + logger.info(f" language_detection: {connection_params.language_detection}") + logger.info("="*80) + + # AssemblyAI Speech-to-Text Service + stt = AssemblyAISTTService( + api_key=os.getenv("ASSEMBLYAI_API_KEY"), + connection_params=connection_params, + + # Turn Detection Mode + # True = Pipecat mode (VAD + Smart Turn controls turns) + # False = STT mode (u3-rt-pro model controls turns) + vad_force_turn_endpoint=True, + + # Speaker Formatting (only used if speaker_labels=True) + # None = Just log speaker IDs, don't modify transcript + speaker_format=None, + # speaker_format="{text}", + # speaker_format="{speaker}: {text}", + # speaker_format="[{speaker}] {text}", + + # Additional available parameters (uncomment to use): + # should_interrupt=True, # Only for STT mode + ) + + # ======================================================================== + + # Text-to-Speech + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="a0e99841-438c-4a64-b679-ae501e7d6091", # Conversational English + ) + + # LLM + llm = OpenAILLMService( + api_key=os.getenv("OPENAI_API_KEY"), + model="gpt-4", + ) + + # Conversation context + messages = [ + { + "role": "system", + "content": ( + "You are a helpful voice assistant testing the AssemblyAI u3-rt-pro model. " + "Keep responses very brief (1-2 sentences). " + "Start by introducing yourself briefly and asking the user to speak." + ), + }, + ] + + context = LLMContext(messages) + + # Configure aggregator based on mode + # In STT mode, don't use VAD (model handles turn detection) + # In Pipecat mode, use VAD + Smart Turn + vad_force_turn_endpoint = True # Must match the value in stt configuration above + user_params = None + if vad_force_turn_endpoint: + user_params = LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()) + + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=user_params, + ) + + # Pipeline + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + # Task + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + # Start the conversation + await task.queue_frames([LLMRunFrame()]) + + # Run + runner = PipelineRunner() + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/test_assemblyai_interactive.py b/test_assemblyai_interactive.py new file mode 100755 index 000000000..ce468ab3d --- /dev/null +++ b/test_assemblyai_interactive.py @@ -0,0 +1,750 @@ +#!/usr/bin/env python3 +"""Interactive AssemblyAI u3-rt-pro Comprehensive Test Suite + +Tests all features with detailed scenarios: +- Basic configuration variations +- Prompting and keyterms with difficult names +- Diarization +- Dynamic parameter updates (single and multiple) +- Mode comparisons +- STT mode timing experiments (testing silence parameters) +- Edge cases + +Usage: + python test_assemblyai_interactive.py +""" + +import asyncio +import os +import sys +from typing import Optional + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.services.assemblyai.models import AssemblyAIConnectionParams +from pipecat.services.assemblyai.stt import AssemblyAISTTService, AssemblyAISTTSettings +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.local.audio import LocalAudioTransport, LocalAudioTransportParams + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="INFO") + + +async def run_bot( + connection_params: AssemblyAIConnectionParams, + test_name: str, + vad_force_turn_endpoint: bool = True, + speaker_format: Optional[str] = None, + test_dynamic_updates: Optional[callable] = None, +): + """Run the voice bot with specified configuration.""" + logger.info("="*80) + logger.info(f"TEST: {test_name}") + logger.info("="*80) + logger.info("Starting bot... Speak into your microphone after you hear the greeting!") + logger.info("="*80) + + # Create local audio transport + transport = LocalAudioTransport( + LocalAudioTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ) + ) + + # AssemblyAI Speech-to-Text + stt = AssemblyAISTTService( + api_key=os.getenv("ASSEMBLYAI_API_KEY"), + connection_params=connection_params, + vad_force_turn_endpoint=vad_force_turn_endpoint, + speaker_format=speaker_format, + ) + + # Text-to-Speech + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="a0e99841-438c-4a64-b679-ae501e7d6091", + ) + + # LLM + llm = OpenAILLMService( + api_key=os.getenv("OPENAI_API_KEY"), + model="gpt-4", + ) + + # Conversation context + messages = [ + { + "role": "system", + "content": ( + "You are a helpful voice assistant testing the AssemblyAI u3-rt-pro model. " + "Keep responses very brief (1-2 sentences). " + "Start by introducing yourself briefly and asking the user to speak." + ), + }, + ] + + context = LLMContext(messages) + + # Configure aggregator based on mode + user_params = None + if vad_force_turn_endpoint: + user_params = LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()) + + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=user_params, + ) + + # Pipeline + pipeline = Pipeline( + [ + transport.input(), + stt, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + # Task + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + # Handle dynamic updates if provided + if test_dynamic_updates: + asyncio.create_task(test_dynamic_updates(task)) + + # Start the conversation + await task.queue_frames([LLMRunFrame()]) + + # Run + runner = PipelineRunner() + await runner.run(task) + + +# ============================================================================ +# Test Configurations +# ============================================================================ + +# === BASIC CONFIGURATION (1-3) === + +async def test_01_basic_100ms(): + """Test 1: Basic default configuration (100ms).""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + min_turn_silence=100, + ) + await run_bot(connection_params, "Basic Default Configuration (100ms)") + + +async def test_02_custom_200ms(): + """Test 2: Custom min_end_of_turn_silence (200ms).""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + min_turn_silence=200, + ) + await run_bot(connection_params, "Custom Turn Silence (200ms)") + + +async def test_03_custom_500ms(): + """Test 3: Longer silence threshold (500ms).""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + min_turn_silence=500, + ) + await run_bot(connection_params, "Longer Turn Silence (500ms)") + + +# === PROMPTING & WARNINGS (4-7) === + +async def test_04_max_warning(): + """Test 4: max_turn_silence warning (should be overridden).""" + logger.warning("⚠️ EXPECT WARNING: max_turn_silence will be overridden") + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + max_turn_silence=500, + ) + await run_bot(connection_params, "max_turn_silence Override Warning") + + +async def test_05_prompt_warning(): + """Test 5: Custom prompt warning.""" + logger.warning("⚠️ EXPECT WARNING: Custom prompts should be tested carefully") + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + prompt="Transcribe speech accurately with proper punctuation.", + ) + await run_bot(connection_params, "Custom Prompt Warning Test") + + +async def test_06_prompt_keyterms_conflict(): + """Test 6: Prompt + keyterms conflict (should error).""" + logger.error("❌ EXPECT ERROR: Cannot use both prompt and keyterms_prompt") + try: + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + prompt="Custom prompt", + keyterms_prompt=["test"], + ) + await run_bot(connection_params, "Prompt + Keyterms Conflict (ERROR)") + except ValueError as e: + logger.error(f"✅ EXPECTED ERROR: {e}") + input("\nPress Enter to continue...") + return + + +async def test_07_keyterms_difficult(): + """Test 7: Keyterms with difficult/unusual names.""" + # Use names that STT wouldn't normally get right + keyterms = ["Xiomara", "Saoirse", "Krzystof", "Nguyen", "Pipecat", "AssemblyAI"] + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + keyterms_prompt=keyterms, + ) + logger.info("🎯 Boosted terms: Xiomara, Saoirse, Krzystof, Nguyen, Pipecat, AssemblyAI") + logger.info(" Try saying these difficult names to test boosting!") + await run_bot(connection_params, "Keyterms with Difficult Names") + + +# === DIARIZATION (8-9) === + +async def test_08_diarization_basic(): + """Test 8: Basic diarization (speaker IDs logged).""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + speaker_labels=True, + ) + logger.info("🎤 Diarization enabled - speaker IDs will be logged") + logger.info(" Try having multiple people speak!") + await run_bot(connection_params, "Diarization - Basic") + + +async def test_09_diarization_xml(): + """Test 9: Diarization with XML formatting.""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + speaker_labels=True, + ) + logger.info("🎤 Diarization with XML tags") + logger.info(" Transcripts will include text") + await run_bot( + connection_params, + "Diarization - XML Formatting", + speaker_format="{text}", + ) + + +# === DYNAMIC UPDATES - SINGLE PARAMETER (10-13) === + +async def test_10_dynamic_keyterms(): + """Test 10: Dynamic keyterms update with difficult names.""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + ) + + async def dynamic_update(task): + logger.info("\n" + "="*80) + logger.info("PHASE 1: No keyterms boosting") + logger.info(" Try saying: Xiomara, Saoirse, Krzystof") + logger.info(" (May not transcribe correctly)") + logger.info("="*80) + await asyncio.sleep(15) + + logger.info("\n" + "="*80) + logger.info("🔄 UPDATING: Adding keyterms boost") + logger.info("="*80) + await task.queue_frame( + STTUpdateSettingsFrame( + delta=AssemblyAISTTSettings( + connection_params=AssemblyAIConnectionParams( + keyterms_prompt=["Xiomara", "Saoirse", "Krzystof", "Nguyen"] + ) + ) + ) + ) + logger.info("\n" + "="*80) + logger.info("PHASE 2: Keyterms NOW boosted") + logger.info(" Say the same names again: Xiomara, Saoirse, Krzystof") + logger.info(" (Should transcribe better now!)") + logger.info("="*80) + + logger.info("🔄 This test has 2 phases:") + logger.info(" Phase 1 (15s): No boosting - names may be wrong") + logger.info(" Phase 2: Keyterms added - names should improve") + await run_bot( + connection_params, + "Dynamic Keyterms Update (Before/After)", + test_dynamic_updates=dynamic_update, + ) + + +async def test_11_dynamic_silence(): + """Test 11: Dynamic silence parameter update (dramatic change).""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + min_turn_silence=100, + ) + + async def dynamic_update(task): + logger.info("\n" + "="*80) + logger.info("PHASE 1: Quick responses (100ms silence threshold)") + logger.info(" Speak normally - bot responds quickly") + logger.info("="*80) + await asyncio.sleep(10) + + logger.info("\n" + "="*80) + logger.info("🔄 UPDATING: Changing silence from 100ms → 3000ms (3 seconds!)") + logger.info("="*80) + await task.queue_frame( + STTUpdateSettingsFrame( + delta=AssemblyAISTTSettings( + connection_params=AssemblyAIConnectionParams( + min_turn_silence=3000 + ) + ) + ) + ) + logger.info("\n" + "="*80) + logger.info("PHASE 2: Patient responses (3 second silence threshold)") + logger.info(" Bot will wait 3 full seconds before responding") + logger.info(" Try pausing mid-sentence - bot should NOT interrupt") + logger.info("="*80) + + logger.info("🔄 Dramatic change: 100ms → 3000ms after 10 seconds") + await run_bot( + connection_params, + "Dynamic Silence Update (100ms → 3s)", + test_dynamic_updates=dynamic_update, + ) + + +async def test_12_dynamic_prompt(): + """Test 12: Dynamic prompt update with keyterms in prompt.""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + ) + + async def dynamic_update(task): + logger.info("\n" + "="*80) + logger.info("PHASE 1: Default prompt (no keyterms)") + logger.info(" Try saying: Xiomara, Saoirse, Krzystof") + logger.info(" (May not transcribe correctly)") + logger.info("="*80) + await asyncio.sleep(15) + + logger.info("\n" + "="*80) + logger.info("🔄 UPDATING: Adding custom prompt with keyterms") + logger.info("="*80) + custom_prompt = """Transcribe verbatim. Rules: +1) Always include punctuation in output. +2) Use period/question mark ONLY for complete sentences. +3) Use comma for mid-sentence pauses. +4) Use no punctuation for incomplete trailing speech. +5) Filler words (um, uh, so, like) indicate speaker will continue. + +Pay special attention to these names and transcribe them exactly: Xiomara, Saoirse, Krzystof, Nguyen.""" + await task.queue_frame( + STTUpdateSettingsFrame( + delta=AssemblyAISTTSettings( + connection_params=AssemblyAIConnectionParams( + prompt=custom_prompt + ) + ) + ) + ) + logger.info("\n" + "="*80) + logger.info("PHASE 2: Prompt with keyterms NOW active") + logger.info(" Say the same names again: Xiomara, Saoirse, Krzystof") + logger.info(" (Should transcribe better now!)") + logger.info("="*80) + + logger.info("🔄 This test has 2 phases:") + logger.info(" Phase 1 (15s): Default prompt - names may be wrong") + logger.info(" Phase 2: Custom prompt with keyterms - names should improve") + await run_bot( + connection_params, + "Dynamic Prompt Update (with keyterms)", + test_dynamic_updates=dynamic_update, + ) + + +async def test_13_dynamic_clear_keyterms(): + """Test 13: Clear keyterms dynamically.""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + keyterms_prompt=["Pipecat", "AssemblyAI"], + ) + + async def dynamic_update(task): + await asyncio.sleep(10) + logger.info("🔄 UPDATING: Clearing keyterms (empty array)") + await task.queue_frame( + STTUpdateSettingsFrame( + delta=AssemblyAISTTSettings( + connection_params=AssemblyAIConnectionParams( + keyterms_prompt=[] + ) + ) + ) + ) + + logger.info("🎯 Initial: Pipecat, AssemblyAI boosted") + logger.info("🔄 After 10s: Keyterms will be cleared") + await run_bot( + connection_params, + "Dynamic Clear Keyterms", + test_dynamic_updates=dynamic_update, + ) + + +# === DYNAMIC UPDATES - MULTIPLE PARAMETERS (14-15) === + +async def test_14_multi_param_update(): + """Test 14: Update multiple parameters at once.""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + min_turn_silence=100, + ) + + async def dynamic_update(task): + await asyncio.sleep(10) + logger.info("🔄 UPDATING MULTIPLE: keyterms + silence") + await task.queue_frame( + STTUpdateSettingsFrame( + delta=AssemblyAISTTSettings( + connection_params=AssemblyAIConnectionParams( + keyterms_prompt=["Xiomara", "Pipecat"], + min_turn_silence=250, + ) + ) + ) + ) + + logger.info("🔄 After 10s: Will update BOTH keyterms AND silence threshold") + await run_bot( + connection_params, + "Multiple Parameter Update", + test_dynamic_updates=dynamic_update, + ) + + +async def test_15_complex_sequence(): + """Test 15: Complex multi-stage update sequence.""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + ) + + async def dynamic_update(task): + logger.info("Stage 1: Initial (10s)") + await asyncio.sleep(10) + + logger.info("🔄 Stage 2: Add keyterms") + await task.queue_frame( + STTUpdateSettingsFrame( + delta=AssemblyAISTTSettings( + connection_params=AssemblyAIConnectionParams( + keyterms_prompt=["Pipecat"] + ) + ) + ) + ) + await asyncio.sleep(10) + + logger.info("🔄 Stage 3: Change silence") + await task.queue_frame( + STTUpdateSettingsFrame( + delta=AssemblyAISTTSettings( + connection_params=AssemblyAIConnectionParams( + min_turn_silence=200 + ) + ) + ) + ) + await asyncio.sleep(10) + + logger.info("🔄 Stage 4: Update both") + await task.queue_frame( + STTUpdateSettingsFrame( + delta=AssemblyAISTTSettings( + connection_params=AssemblyAIConnectionParams( + keyterms_prompt=["AssemblyAI", "OpenAI"], + min_turn_silence=150, + ) + ) + ) + ) + + logger.info("🔄 Multi-stage: 4 configuration changes over 30 seconds") + await run_bot( + connection_params, + "Complex Update Sequence (4 stages)", + test_dynamic_updates=dynamic_update, + ) + + +# === MODE COMPARISON (16-17) === + +async def test_16_pipecat_mode(): + """Test 16: Pipecat mode (VAD + Smart Turn controls turns).""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + min_turn_silence=100, + ) + logger.info("🎯 Pipecat Mode: VAD + Smart Turn control turn detection") + logger.info(" Your min_end_of_turn_silence is sent but ForceEndpoint overrides it") + await run_bot( + connection_params, + "Pipecat Mode (VAD + Smart Turn)", + vad_force_turn_endpoint=True, + ) + + +async def test_17_stt_mode(): + """Test 17: STT mode (model controls turns).""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + min_turn_silence=100, + ) + logger.info("🎯 STT Mode: u3-rt-pro model controls turn detection") + logger.info(" No ForceEndpoint - parameters are respected") + await run_bot( + connection_params, + "STT Mode (Model Turn Detection)", + vad_force_turn_endpoint=False, + ) + + +# === STT MODE TIMING EXPERIMENTS (18-20) === + +async def test_18_stt_long_max_short_min(): + """Test 18: STT mode - Long max_turn_silence + Short min (5000ms + 100ms).""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + min_turn_silence=100, # Short - quick confident turns + max_turn_silence=5000, # Long - allows pauses up to 5 seconds + ) + logger.info("🎯 STT Mode: Testing max/min parameter interaction") + logger.info(" min_turn_silence: 100ms (quick when confident)") + logger.info(" max_turn_silence: 5000ms (allows up to 5 second pauses)") + logger.info(" Try: Quick sentences (should respond fast) + Long pauses mid-thought") + await run_bot( + connection_params, + "STT: Long Max (5s) + Short Min (100ms)", + vad_force_turn_endpoint=False, + ) + + +async def test_19_stt_long_min(): + """Test 19: STT mode - Long min_turn_silence (3000ms).""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + min_turn_silence=3000, # 3 seconds + max_turn_silence=5000, # 5 seconds + ) + logger.info("🎯 STT Mode: Testing long minimum silence requirement") + logger.info(" min_turn_silence: 3000ms") + logger.info(" max_turn_silence: 5000ms") + logger.info(" Bot will wait 3 full seconds of silence before responding!") + logger.info(" Try: Speaking with short pauses - bot should NOT interrupt") + await run_bot( + connection_params, + "STT: Long Min (3s)", + vad_force_turn_endpoint=False, + ) + + +async def test_20_stt_both_short(): + """Test 20: STT mode - Both short (max=300ms, min=100ms).""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + min_turn_silence=100, # 100ms + max_turn_silence=300, # 300ms + ) + logger.info("🎯 STT Mode: Testing aggressive/quick response timing") + logger.info(" min_turn_silence: 100ms") + logger.info(" max_turn_silence: 300ms") + logger.info(" Bot will respond VERY quickly to any pause!") + logger.info(" Try: Speaking with natural pauses - expect quick responses") + await run_bot( + connection_params, + "STT: Both Short (300ms/100ms)", + vad_force_turn_endpoint=False, + ) + + +# === EDGE CASES (21-23) === + +async def test_21_very_long_silence(): + """Test 21: Very long silence threshold (STT mode only).""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + min_turn_silence=10000, # 10 seconds + ) + logger.warning("⚠️ STT Mode with 10 second silence threshold") + logger.info(" Bot will wait 10 seconds of silence before responding!") + await run_bot( + connection_params, + "Very Long Silence (10s) - STT Mode", + vad_force_turn_endpoint=False, + ) + + +async def test_22_very_short_silence(): + """Test 22: Very short silence threshold (50ms).""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + min_turn_silence=50, + ) + logger.info("⚡ Very short silence threshold (50ms)") + logger.info(" Bot will respond very quickly!") + await run_bot(connection_params, "Very Short Silence (50ms)") + + +async def test_23_keyterms_plus_diarization(): + """Test 23: Keyterms + Diarization combined.""" + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + keyterms_prompt=["Xiomara", "Saoirse", "Pipecat"], + speaker_labels=True, + ) + logger.info("🎯 Keyterms + 🎤 Diarization both enabled") + logger.info(" Try multiple speakers saying difficult names!") + await run_bot( + connection_params, + "Keyterms + Diarization Combined", + speaker_format="[{speaker}] {text}", + ) + + +# ============================================================================ +# Interactive Menu +# ============================================================================ + + +def show_menu(): + """Display the comprehensive test menu.""" + print("\n" + "="*80) + print("AssemblyAI u3-rt-pro Comprehensive Test Suite") + print("="*80) + print("\n📋 BASIC CONFIGURATION (1-3)") + print(" 1. Basic Default (100ms)") + print(" 2. Custom Silence (200ms)") + print(" 3. Longer Silence (500ms)") + + print("\n⚠️ PROMPTING & WARNINGS (4-7)") + print(" 4. max_turn_silence Warning") + print(" 5. Custom Prompt Warning") + print(" 6. Prompt + Keyterms Conflict (ERROR)") + print(" 7. Keyterms with Difficult Names") + + print("\n🎤 DIARIZATION (8-9)") + print(" 8. Diarization - Basic") + print(" 9. Diarization - XML Formatting") + + print("\n🔄 DYNAMIC UPDATES - SINGLE (10-13)") + print(" 10. Dynamic Keyterms (Before/After with difficult names)") + print(" 11. Dynamic Silence (100ms → 3s DRAMATIC)") + print(" 12. Dynamic Prompt with Keyterms (Before/After)") + print(" 13. Dynamic Clear Keyterms") + + print("\n🔄 DYNAMIC UPDATES - MULTIPLE (14-15)") + print(" 14. Multiple Parameters at Once") + print(" 15. Complex Update Sequence (4 stages)") + + print("\n⚖️ MODE COMPARISON (16-17)") + print(" 16. Pipecat Mode (VAD + Smart Turn)") + print(" 17. STT Mode (Model Turn Detection)") + + print("\n⏱️ STT MODE TIMING EXPERIMENTS (18-20)") + print(" 18. STT: Long Max (5s) + Short Min (100ms)") + print(" 19. STT: Long Min (3s)") + print(" 20. STT: Both Short (300ms/100ms)") + + print("\n🎯 EDGE CASES (21-23)") + print(" 21. Very Long Silence (10s - STT Mode)") + print(" 22. Very Short Silence (50ms)") + print(" 23. Keyterms + Diarization Combined") + + print("\n 0. Exit") + print("\n" + "="*80) + + +async def main(): + """Main interactive menu.""" + tests = { + "1": test_01_basic_100ms, + "2": test_02_custom_200ms, + "3": test_03_custom_500ms, + "4": test_04_max_warning, + "5": test_05_prompt_warning, + "6": test_06_prompt_keyterms_conflict, + "7": test_07_keyterms_difficult, + "8": test_08_diarization_basic, + "9": test_09_diarization_xml, + "10": test_10_dynamic_keyterms, + "11": test_11_dynamic_silence, + "12": test_12_dynamic_prompt, + "13": test_13_dynamic_clear_keyterms, + "14": test_14_multi_param_update, + "15": test_15_complex_sequence, + "16": test_16_pipecat_mode, + "17": test_17_stt_mode, + "18": test_18_stt_long_max_short_min, + "19": test_19_stt_long_min, + "20": test_20_stt_both_short, + "21": test_21_very_long_silence, + "22": test_22_very_short_silence, + "23": test_23_keyterms_plus_diarization, + } + + while True: + show_menu() + choice = input("Enter test number (or 0 to exit): ").strip() + + if choice == "0": + print("\n👋 Goodbye!") + break + + if choice in tests: + try: + await tests[choice]() + except KeyboardInterrupt: + print("\n\n⚠️ Test interrupted by user") + except Exception as e: + logger.error(f"Test failed with error: {e}") + import traceback + traceback.print_exc() + + input("\n\nPress Enter to return to menu...") + else: + print(f"\n❌ Invalid choice: {choice}") + input("Press Enter to continue...") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\n\n👋 Goodbye!") diff --git a/test_assemblyai_u3pro.py b/test_assemblyai_u3pro.py new file mode 100644 index 000000000..ace700753 --- /dev/null +++ b/test_assemblyai_u3pro.py @@ -0,0 +1,589 @@ +#!/usr/bin/env python3 +"""AssemblyAI u3-rt-pro Comprehensive Test Script + +Tests all features: +- Basic configuration +- Prompting and keyterms +- Diarization +- Dynamic updates +- Turn detection modes + +Usage: + python test_assemblyai_u3pro.py --test + python test_assemblyai_u3pro.py --interactive +""" + +import argparse +import asyncio +import os +import sys +from typing import List + +from dotenv import load_dotenv +from loguru import logger + +# Add src to path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src")) + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import ( + EndFrame, + Frame, + LLMRunFrame, + STTUpdateSettingsFrame, + TranscriptionFrame, +) +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.services.assemblyai.models import AssemblyAIConnectionParams +from pipecat.services.assemblyai.stt import AssemblyAISTTService +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.local.audio import LocalAudioTransport, LocalAudioTransportParams + +load_dotenv() + +# Test configuration +class TestConfig: + """Centralized test configuration.""" + + ASSEMBLYAI_API_KEY = os.getenv("ASSEMBLYAI_API_KEY") + OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") + CARTESIA_API_KEY = os.getenv("CARTESIA_API_KEY") + + @classmethod + def validate(cls): + """Validate all required API keys are set.""" + missing = [] + if not cls.ASSEMBLYAI_API_KEY: + missing.append("ASSEMBLYAI_API_KEY") + if not cls.OPENAI_API_KEY: + missing.append("OPENAI_API_KEY") + if not cls.CARTESIA_API_KEY: + missing.append("CARTESIA_API_KEY") + + if missing: + logger.error(f"Missing required environment variables: {', '.join(missing)}") + return False + return True + + +class TranscriptionLogger(FrameProcessor): + """Log transcriptions for test verification.""" + + async def process_frame(self, frame: Frame, direction: FrameDirection): + if isinstance(frame, TranscriptionFrame): + logger.info(f"📝 TRANSCRIPTION: {frame.text}") + logger.info(f" Speaker: {frame.user_id}") + logger.info(f" Finalized: {frame.finalized}") + if hasattr(frame, "result") and frame.result: + if hasattr(frame.result, "speaker"): + logger.info(f" Diarization: {frame.result.speaker}") + + await self.push_frame(frame, direction) + + +async def create_basic_voice_agent( + connection_params: AssemblyAIConnectionParams, + vad_force_turn_endpoint: bool = True, + speaker_format: str = None, +) -> tuple[PipelineTask, LocalAudioTransport]: + """Create a basic voice agent for testing. + + Args: + connection_params: AssemblyAI connection parameters + vad_force_turn_endpoint: Turn detection mode + speaker_format: Optional speaker formatting string + + Returns: + Tuple of (PipelineTask, LocalAudioTransport) + """ + # Create local audio transport (uses your microphone and speakers) + transport = LocalAudioTransport( + params=LocalAudioTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ) + ) + + # Create STT + stt = AssemblyAISTTService( + api_key=TestConfig.ASSEMBLYAI_API_KEY, + connection_params=connection_params, + vad_force_turn_endpoint=vad_force_turn_endpoint, + speaker_format=speaker_format, + ) + + # Create TTS + tts = CartesiaTTSService( + api_key=TestConfig.CARTESIA_API_KEY, + voice_id="a0e99841-438c-4a64-b679-ae501e7d6091", # Conversational English + ) + + # Create LLM context and service + messages = [ + { + "role": "system", + "content": ( + "You are a helpful voice assistant. Keep responses brief and natural. " + "If you see speaker tags like text, acknowledge " + "that you understand multiple speakers are present." + ), + } + ] + + context = LLMContext(messages) + llm = OpenAILLMService(api_key=TestConfig.OPENAI_API_KEY, model="gpt-4") + + # Create aggregators with VAD + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams( + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + # Create transcription logger + transcription_logger = TranscriptionLogger() + + # Create pipeline + pipeline = Pipeline( + [ + transport.input(), + stt, + transcription_logger, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + # Create task + task = PipelineTask(pipeline) + + return task, transport + + +# ============================================================================ +# Test Functions +# ============================================================================ + + +async def test_basic_config(): + """Test 1: Basic default configuration.""" + logger.info("=" * 80) + logger.info("TEST 1: Basic Default Configuration") + logger.info("=" * 80) + + connection_params = AssemblyAIConnectionParams(speech_model="u3-rt-pro") + + task, transport = await create_basic_voice_agent(connection_params) + + logger.info("✅ Service created successfully with default params") + logger.info("Expected: min=max=100ms, u3-rt-pro model") + logger.info("Speak into your microphone to test transcription") + + # Trigger initial bot greeting + await task.queue_frames([LLMRunFrame()]) + + runner = PipelineRunner() + await runner.run(task) + + +async def test_custom_min_silence(): + """Test 2: Custom min_turn_silence.""" + logger.info("=" * 80) + logger.info("TEST 2: Custom min_turn_silence") + logger.info("=" * 80) + + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", min_turn_silence=200 + ) + + task, transport = await create_basic_voice_agent(connection_params) + + logger.info("✅ Service created with min=200ms") + logger.info("Expected: Both min and max set to 200ms") + logger.info("Speak short phrases and observe turn detection timing") + + runner = PipelineRunner() + await runner.run(task) + + +async def test_max_silence_warning(): + """Test 3: Setting max_turn_silence should trigger warning.""" + logger.info("=" * 80) + logger.info("TEST 3: max_turn_silence Warning") + logger.info("=" * 80) + + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + min_turn_silence=100, + max_turn_silence=500, # Should trigger warning + ) + + task, transport = await create_basic_voice_agent(connection_params) + + logger.info("⚠️ Check logs above for warning about max_turn_silence being overridden") + logger.info("Expected: Warning logged, max set to 100ms (same as min)") + + runner = PipelineRunner() + await runner.run(task) + + +async def test_custom_prompt_warning(): + """Test 5: Custom prompt should trigger warning.""" + logger.info("=" * 80) + logger.info("TEST 5: Custom Prompt Warning") + logger.info("=" * 80) + + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + prompt="Transcribe verbatim. Always include punctuation.", + ) + + task, transport = await create_basic_voice_agent(connection_params) + + logger.info("⚠️ Check logs above for warning about testing without prompt first") + logger.info("Expected: Warning logged, service continues with custom prompt") + + runner = PipelineRunner() + await runner.run(task) + + +async def test_prompt_keyterms_conflict(): + """Test 6: Prompt + keyterms_prompt should raise error.""" + logger.info("=" * 80) + logger.info("TEST 6: Prompt + Keyterms Conflict (Error)") + logger.info("=" * 80) + + try: + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + prompt="Custom prompt", + keyterms_prompt=["test", "words"], + ) + + task, transport = await create_basic_voice_agent(connection_params) + logger.error("❌ TEST FAILED: Should have raised ValueError") + except ValueError as e: + logger.info(f"✅ TEST PASSED: ValueError raised as expected") + logger.info(f" Error message: {e}") + + +async def test_keyterms_basic(): + """Test 7: Basic keyterms at initialization.""" + logger.info("=" * 80) + logger.info("TEST 7: Basic Keyterms Prompting") + logger.info("=" * 80) + + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + keyterms_prompt=["Pipecat", "AssemblyAI", "Universal-3", "streaming"], + ) + + task, transport = await create_basic_voice_agent(connection_params) + + logger.info("✅ Service created with keyterms: Pipecat, AssemblyAI, Universal-3, streaming") + logger.info("Expected: Boosted recognition for these terms") + logger.info("Try saying: 'I'm testing Pipecat with AssemblyAI Universal-3 for streaming'") + + runner = PipelineRunner() + await runner.run(task) + + +async def test_diarization_no_format(): + """Test 10: Diarization enabled without formatting.""" + logger.info("=" * 80) + logger.info("TEST 10: Diarization Enabled (No Formatting)") + logger.info("=" * 80) + + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", speaker_labels=True + ) + + task, transport = await create_basic_voice_agent(connection_params) + + logger.info("✅ Service created with speaker_labels=True") + logger.info("Expected: Speaker IDs in user_id field, plain text in transcript") + logger.info("Have multiple people speak to see different speaker labels") + + runner = PipelineRunner() + await runner.run(task) + + +async def test_diarization_xml_format(): + """Test 11: Diarization with XML formatting.""" + logger.info("=" * 80) + logger.info("TEST 11: Diarization with XML Formatting") + logger.info("=" * 80) + + connection_params = AssemblyAIConnectionParams( + speech_model="u3-rt-pro", speaker_labels=True + ) + + task, transport = await create_basic_voice_agent( + connection_params, speaker_format="<{speaker}>{text}" + ) + + logger.info("✅ Service created with XML speaker formatting") + logger.info("Expected: Text like 'Hello'") + logger.info("Have multiple people speak to see formatted speaker tags") + + runner = PipelineRunner() + await runner.run(task) + + +async def test_dynamic_keyterms(): + """Test 13: Dynamic keyterms updates.""" + logger.info("=" * 80) + logger.info("TEST 13: Dynamic Keyterms Updates") + logger.info("=" * 80) + + connection_params = AssemblyAIConnectionParams(speech_model="u3-rt-pro") + + task, transport = await create_basic_voice_agent(connection_params) + + async def update_keyterms_stages(): + """Simulate multi-stage conversation with keyterms updates.""" + await asyncio.sleep(5) # Wait for connection + + # Stage 1: Greeting + logger.info("🔄 STAGE 1: Greeting (general terms)") + update1 = STTUpdateSettingsFrame( + settings={"keyterms_prompt": ["hello", "hi", "good morning", "welcome"]} + ) + await task.queue_frames([update1]) + + await asyncio.sleep(10) + + # Stage 2: Name collection + logger.info("🔄 STAGE 2: Name Collection") + update2 = STTUpdateSettingsFrame( + settings={ + "keyterms_prompt": [ + "first name", + "last name", + "John", + "Jane", + "Smith", + "Johnson", + ] + } + ) + await task.queue_frames([update2]) + + await asyncio.sleep(10) + + # Stage 3: Medical info + logger.info("🔄 STAGE 3: Medical Information") + update3 = STTUpdateSettingsFrame( + settings={ + "keyterms_prompt": [ + "cardiology", + "echocardiogram", + "blood pressure", + "Dr. Smith", + "metoprolol", + ] + } + ) + await task.queue_frames([update3]) + + await asyncio.sleep(10) + + # Stage 4: Clear keyterms + logger.info("🔄 STAGE 4: Clear Keyterms") + update4 = STTUpdateSettingsFrame(settings={"keyterms_prompt": []}) + await task.queue_frames([update4]) + + # Start update task + asyncio.create_task(update_keyterms_stages()) + + logger.info("✅ Service created, will update keyterms every 10 seconds") + logger.info("Expected: Different keyterms at each stage") + logger.info("Watch logs for 'STAGE X' messages and test relevant terms") + + runner = PipelineRunner() + await runner.run(task) + + +async def test_dynamic_silence_params(): + """Test 15: Dynamic silence parameter updates.""" + logger.info("=" * 80) + logger.info("TEST 15: Dynamic Silence Parameters") + logger.info("=" * 80) + + connection_params = AssemblyAIConnectionParams(speech_model="u3-rt-pro") + + task, transport = await create_basic_voice_agent(connection_params) + + async def update_silence_params(): + """Update silence parameters for different scenarios.""" + await asyncio.sleep(5) + + # Normal conversation + logger.info("🔄 PHASE 1: Normal conversation (default timing)") + await asyncio.sleep(10) + + # Reading credit card + logger.info("🔄 PHASE 2: Reading numbers (longer silence tolerance)") + update1 = STTUpdateSettingsFrame( + settings={ + "max_turn_silence": 5000, + "min_turn_silence": 300, + } + ) + await task.queue_frames([update1]) + + await asyncio.sleep(15) + + # Back to normal + logger.info("🔄 PHASE 3: Back to normal conversation") + update2 = STTUpdateSettingsFrame( + settings={ + "max_turn_silence": 1200, + "min_turn_silence": 100, + } + ) + await task.queue_frames([update2]) + + asyncio.create_task(update_silence_params()) + + logger.info("✅ Service will update silence parameters during conversation") + logger.info("Expected: Longer pauses tolerated in Phase 2") + logger.info("Try pausing between words to test") + + runner = PipelineRunner() + await runner.run(task) + + +async def test_multi_param_update(): + """Test 17: Update multiple parameters at once.""" + logger.info("=" * 80) + logger.info("TEST 17: Multiple Parameter Update") + logger.info("=" * 80) + + connection_params = AssemblyAIConnectionParams(speech_model="u3-rt-pro") + + task, transport = await create_basic_voice_agent(connection_params) + + async def multi_update(): + await asyncio.sleep(5) + + logger.info("🔄 Updating multiple parameters together") + update = STTUpdateSettingsFrame( + settings={ + "keyterms_prompt": ["account", "routing", "number"], + "max_turn_silence": 3000, + "min_turn_silence": 200, + } + ) + await task.queue_frames([update]) + + logger.info("✅ Check logs for single UpdateConfiguration message") + + asyncio.create_task(multi_update()) + + logger.info("Expected: All params updated in single WebSocket message") + + runner = PipelineRunner() + await runner.run(task) + + +# ============================================================================ +# Main Test Runner +# ============================================================================ + + +def main(): + """Main test runner.""" + parser = argparse.ArgumentParser(description="Test AssemblyAI u3-rt-pro integration") + parser.add_argument( + "--test", + type=str, + default="basic", + help="Test to run (basic, custom_min, max_warning, prompt_warning, " + "prompt_keyterms_conflict, keyterms, diarization, diarization_xml, " + "dynamic_keyterms, dynamic_silence, multi_param, all)", + ) + parser.add_argument( + "--interactive", action="store_true", help="Run in interactive mode" + ) + + args = parser.parse_args() + + # Validate environment + if not TestConfig.validate(): + logger.error("Please set all required environment variables in .env") + sys.exit(1) + + # Test mapping + tests = { + "basic": test_basic_config, + "custom_min": test_custom_min_silence, + "max_warning": test_max_silence_warning, + "prompt_warning": test_custom_prompt_warning, + "prompt_keyterms_conflict": test_prompt_keyterms_conflict, + "keyterms": test_keyterms_basic, + "diarization": test_diarization_no_format, + "diarization_xml": test_diarization_xml_format, + "dynamic_keyterms": test_dynamic_keyterms, + "dynamic_silence": test_dynamic_silence_params, + "multi_param": test_multi_param_update, + } + + if args.interactive: + logger.info("Interactive mode - select test to run:") + for i, (name, _) in enumerate(tests.items(), 1): + logger.info(f"{i}. {name}") + logger.info(f"{len(tests) + 1}. Run all tests") + + choice = input("\nEnter test number: ") + try: + choice_num = int(choice) + if choice_num == len(tests) + 1: + args.test = "all" + else: + args.test = list(tests.keys())[choice_num - 1] + except (ValueError, IndexError): + logger.error("Invalid choice") + sys.exit(1) + + # Run test(s) + if args.test == "all": + logger.info("Running all tests sequentially...") + for test_name, test_func in tests.items(): + try: + asyncio.run(test_func()) + except KeyboardInterrupt: + logger.info(f"Test '{test_name}' interrupted") + break + except Exception as e: + logger.error(f"Test '{test_name}' failed: {e}") + else: + if args.test not in tests: + logger.error(f"Unknown test: {args.test}") + logger.info(f"Available tests: {', '.join(tests.keys())}") + sys.exit(1) + + try: + asyncio.run(tests[args.test]()) + except KeyboardInterrupt: + logger.info("Test interrupted") + except Exception as e: + logger.error(f"Test failed: {e}") + raise + + +if __name__ == "__main__": + main() From 07ae4b8d385f3f67b32bc06548382fc1bcade3e2 Mon Sep 17 00:00:00 2001 From: zack Date: Sun, 1 Mar 2026 11:27:31 -0500 Subject: [PATCH 09/23] Update AssemblyAI examples to use u3-rt-pro and improve 55d example - Update 13d-assemblyai-transcription.py to explicitly use u3-rt-pro model - Update 55d-update-settings-assemblyai-stt.py to demonstrate keyterms updates instead of language updates - Add helpful logging to show before/after keyterms boosting effect - Use difficult names (Xiomara, Saoirse, Krzystof) to demonstrate boosting effectiveness --- .../13d-assemblyai-transcription.py | 4 +++ .../55d-update-settings-assemblyai-stt.py | 25 ++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/examples/foundational/13d-assemblyai-transcription.py b/examples/foundational/13d-assemblyai-transcription.py index 06ea52cd5..2dcbaf59b 100644 --- a/examples/foundational/13d-assemblyai-transcription.py +++ b/examples/foundational/13d-assemblyai-transcription.py @@ -16,6 +16,7 @@ from pipecat.pipeline.task import PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport +from pipecat.services.assemblyai.models import AssemblyAIConnectionParams from pipecat.services.assemblyai.stt import AssemblyAISTTService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -49,6 +50,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): stt = AssemblyAISTTService( api_key=os.getenv("ASSEMBLYAI_API_KEY"), + connection_params=AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + ), ) tl = TranscriptionLogger() diff --git a/examples/foundational/55d-update-settings-assemblyai-stt.py b/examples/foundational/55d-update-settings-assemblyai-stt.py index d37c3ec7b..b0d676e25 100644 --- a/examples/foundational/55d-update-settings-assemblyai-stt.py +++ b/examples/foundational/55d-update-settings-assemblyai-stt.py @@ -22,10 +22,10 @@ from pipecat.processors.aggregators.llm_response_universal import ( ) from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport +from pipecat.services.assemblyai.models import AssemblyAIConnectionParams from pipecat.services.assemblyai.stt import AssemblyAISTTService, AssemblyAISTTSettings from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transcriptions.language import Language from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -51,7 +51,12 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = AssemblyAISTTService(api_key=os.getenv("ASSEMBLYAI_API_KEY")) + stt = AssemblyAISTTService( + api_key=os.getenv("ASSEMBLYAI_API_KEY"), + connection_params=AssemblyAIConnectionParams( + speech_model="u3-rt-pro", + ), + ) tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), @@ -63,7 +68,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call demonstrating dynamic keyterms updates. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Try saying difficult names like 'Xiomara', 'Saoirse', or 'Krzystof' to test transcription accuracy.", }, ] @@ -97,14 +102,22 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") + logger.info("Phase 1: No keyterms boosting - try saying 'Xiomara', 'Saoirse', or 'Krzystof'") messages.append({"role": "system", "content": "Please introduce yourself to the user."}) await task.queue_frames([LLMRunFrame()]) - await asyncio.sleep(10) - logger.info("Updating AssemblyAI STT settings: language=es") + await asyncio.sleep(15) + logger.info("🔄 Updating keyterms: Adding difficult names for boosting") await task.queue_frame( - STTUpdateSettingsFrame(delta=AssemblyAISTTSettings(language=Language.ES)) + STTUpdateSettingsFrame( + delta=AssemblyAISTTSettings( + connection_params=AssemblyAIConnectionParams( + keyterms_prompt=["Xiomara", "Saoirse", "Krzystof", "Nguyen", "Pipecat"] + ) + ) + ) ) + logger.info("Phase 2: Keyterms active - same names should transcribe better now!") @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): From 66fca7e3822f75e2042e7cd9e2010c94d0242339 Mon Sep 17 00:00:00 2001 From: zack Date: Sun, 1 Mar 2026 11:33:22 -0500 Subject: [PATCH 10/23] Add backward compatibility for min_end_of_turn_silence_when_confident parameter - Keep old parameter name for backward compatibility - Add deprecation warning when old parameter is used - Automatically migrate old parameter value to new min_turn_silence parameter - Exclude deprecated parameter from WebSocket URL to avoid sending it to API - New parameter takes precedence if both are set --- src/pipecat/services/assemblyai/models.py | 20 +++++++++++++++++++- src/pipecat/services/assemblyai/stt.py | 3 +++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/assemblyai/models.py b/src/pipecat/services/assemblyai/models.py index f92b1b8bf..d8df07899 100644 --- a/src/pipecat/services/assemblyai/models.py +++ b/src/pipecat/services/assemblyai/models.py @@ -11,8 +11,9 @@ transcription WebSocket messages and connection configuration. """ from typing import List, Literal, Optional +import warnings -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator class Word(BaseModel): @@ -130,6 +131,7 @@ class AssemblyAIConnectionParams(BaseModel): word_finalization_max_wait_time: Maximum time to wait for word finalization in milliseconds. end_of_turn_confidence_threshold: Confidence threshold for end-of-turn detection. min_turn_silence: Minimum silence duration when confident about end-of-turn. + min_end_of_turn_silence_when_confident: DEPRECATED. Use min_turn_silence instead. max_turn_silence: Maximum silence duration before forcing end-of-turn. keyterms_prompt: List of key terms to guide transcription. Will be JSON serialized before sending. prompt: Optional text prompt to guide the transcription. Only used when speech_model is "u3-rt-pro". @@ -149,6 +151,7 @@ class AssemblyAIConnectionParams(BaseModel): word_finalization_max_wait_time: Optional[int] = None end_of_turn_confidence_threshold: Optional[float] = None min_turn_silence: Optional[int] = None + min_end_of_turn_silence_when_confident: Optional[int] = None # Deprecated max_turn_silence: Optional[int] = None keyterms_prompt: Optional[List[str]] = None prompt: Optional[str] = None @@ -158,3 +161,18 @@ class AssemblyAIConnectionParams(BaseModel): language_detection: Optional[bool] = None format_turns: bool = True speaker_labels: Optional[bool] = None + + @model_validator(mode="after") + def handle_deprecated_param(self): + """Handle deprecated min_end_of_turn_silence_when_confident parameter.""" + if self.min_end_of_turn_silence_when_confident is not None: + warnings.warn( + "The 'min_end_of_turn_silence_when_confident' parameter is deprecated and will be " + "removed in a future version. Please use 'min_turn_silence' instead.", + DeprecationWarning, + stacklevel=2, + ) + # If min_turn_silence is not set, use the deprecated value + if self.min_turn_silence is None: + self.min_turn_silence = self.min_end_of_turn_silence_when_confident + return self diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index d82b38a99..7e2378408 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -462,6 +462,9 @@ class AssemblyAISTTService(WebsocketSTTService): """Build WebSocket URL with query parameters using urllib.parse.urlencode.""" params = {} for k, v in self._settings.connection_params.model_dump().items(): + # Skip deprecated parameter - it's been migrated to min_turn_silence + if k == "min_end_of_turn_silence_when_confident": + continue if v is not None: if k == "keyterms_prompt": params[k] = json.dumps(v) From d1cbc811083cf2ebfffe2c66c498647b0027fe55 Mon Sep 17 00:00:00 2001 From: zack Date: Sun, 1 Mar 2026 11:36:46 -0500 Subject: [PATCH 11/23] Fix 07o example to use new min_turn_silence parameter name in docs and comments --- examples/foundational/07o-interruptible-assemblyai-stt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/foundational/07o-interruptible-assemblyai-stt.py b/examples/foundational/07o-interruptible-assemblyai-stt.py index ee2994c0e..2765f8590 100644 --- a/examples/foundational/07o-interruptible-assemblyai-stt.py +++ b/examples/foundational/07o-interruptible-assemblyai-stt.py @@ -66,8 +66,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - More natural turn detection based on speech patterns and pauses 2. Advanced Turn Detection Tuning (STT Mode) - - `min_turn_silence`: Minimum silence (ms) when confident - about end-of-turn. Lower values = faster responses. Default: 200ms + - `min_turn_silence`: Minimum silence (ms) when confident about end-of-turn. + Lower values = faster responses. Default: 100ms - `max_turn_silence`: Maximum silence (ms) before forcing end-of-turn. Prevents long pauses. Default: 1000ms From 5de495cc989adf7ae2d2624c9e6613978f69b77c Mon Sep 17 00:00:00 2001 From: zack Date: Sun, 1 Mar 2026 11:39:00 -0500 Subject: [PATCH 12/23] Use logger.warning instead of warnings.warn for deprecation message - Makes deprecation warning visible in logs without needing Python warning flags - Users will see the warning during normal operation --- src/pipecat/services/assemblyai/models.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/pipecat/services/assemblyai/models.py b/src/pipecat/services/assemblyai/models.py index d8df07899..3a022dce1 100644 --- a/src/pipecat/services/assemblyai/models.py +++ b/src/pipecat/services/assemblyai/models.py @@ -11,8 +11,8 @@ transcription WebSocket messages and connection configuration. """ from typing import List, Literal, Optional -import warnings +from loguru import logger from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -166,11 +166,9 @@ class AssemblyAIConnectionParams(BaseModel): def handle_deprecated_param(self): """Handle deprecated min_end_of_turn_silence_when_confident parameter.""" if self.min_end_of_turn_silence_when_confident is not None: - warnings.warn( + logger.warning( "The 'min_end_of_turn_silence_when_confident' parameter is deprecated and will be " - "removed in a future version. Please use 'min_turn_silence' instead.", - DeprecationWarning, - stacklevel=2, + "removed in a future version. Please use 'min_turn_silence' instead." ) # If min_turn_silence is not set, use the deprecated value if self.min_turn_silence is None: From 42f91a905613e7b481489cb6437df87821a81e95 Mon Sep 17 00:00:00 2001 From: zack Date: Sun, 1 Mar 2026 11:44:37 -0500 Subject: [PATCH 13/23] Apply ruff formatting fixes --- .../55d-update-settings-assemblyai-stt.py | 4 +- src/pipecat/services/assemblyai/stt.py | 7 +- test_assemblyai_custom.py | 32 ++------ test_assemblyai_interactive.py | 77 +++++++++---------- test_assemblyai_u3pro.py | 17 ++-- 5 files changed, 56 insertions(+), 81 deletions(-) diff --git a/examples/foundational/55d-update-settings-assemblyai-stt.py b/examples/foundational/55d-update-settings-assemblyai-stt.py index b0d676e25..f57865588 100644 --- a/examples/foundational/55d-update-settings-assemblyai-stt.py +++ b/examples/foundational/55d-update-settings-assemblyai-stt.py @@ -102,7 +102,9 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") - logger.info("Phase 1: No keyterms boosting - try saying 'Xiomara', 'Saoirse', or 'Krzystof'") + logger.info( + "Phase 1: No keyterms boosting - try saying 'Xiomara', 'Saoirse', or 'Krzystof'" + ) messages.append({"role": "system", "content": "Please introduce yourself to the user."}) await task.queue_frames([LLMRunFrame()]) diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index 7e2378408..9938afdee 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -355,13 +355,10 @@ class AssemblyAISTTService(WebsocketSTTService): if hasattr(conn_params, "min_turn_silence"): if ( old_conn_params is None - or conn_params.min_turn_silence - != old_conn_params.min_turn_silence + or conn_params.min_turn_silence != old_conn_params.min_turn_silence ): if conn_params.min_turn_silence is not None: - update_config["min_turn_silence"] = ( - conn_params.min_turn_silence - ) + update_config["min_turn_silence"] = conn_params.min_turn_silence logger.info( f"Updating min_turn_silence to: {conn_params.min_turn_silence}ms" ) diff --git a/test_assemblyai_custom.py b/test_assemblyai_custom.py index c406918c0..e8e0a28d2 100755 --- a/test_assemblyai_custom.py +++ b/test_assemblyai_custom.py @@ -48,11 +48,11 @@ async def main(): logger.remove(0) logger.add(sys.stderr, level=LOG_LEVEL) - logger.info("="*80) + logger.info("=" * 80) logger.info("AssemblyAI u3-rt-pro Custom Test") - logger.info("="*80) + logger.info("=" * 80) logger.info("Starting bot... Speak after you hear the greeting!") - logger.info("="*80) + logger.info("=" * 80) # Create local audio transport transport = LocalAudioTransport( @@ -74,78 +74,63 @@ async def main(): speech_model="u3-rt-pro", # speech_model="universal-streaming-english", # speech_model="universal-streaming-multilingual", - # ==================================================================== # Turn Detection Timing # ==================================================================== - # Minimum silence when confident about end of turn (milliseconds) # Default: 100ms | Higher = more patient | Lower = faster responses # Only used in Pipecat mode (vad_force_turn_endpoint=True) min_turn_silence=100000, # min_turn_silence=200, # min_turn_silence=300, - # Maximum turn silence (milliseconds) # WARNING: In Pipecat mode (vad_force_turn_endpoint=True), this is # automatically set equal to min_turn_silence # to avoid double turn detection. Only used as-is in STT mode. max_turn_silence=500, - # End of turn confidence threshold (0.0 to 1.0) # Higher = requires more confidence before ending turn # end_of_turn_confidence_threshold=0.8, - # ==================================================================== # Prompting & Boosting # ==================================================================== - # Custom Prompt (WARNING: test carefully, default is optimized!) # None = Use AssemblyAI's optimized default (recommended for 88% accuracy) prompt=None, # prompt="Transcribe speech with focus on technical terms.", # prompt="Context: Medical conversation. Transcribe accurately.", - # Keyterms Prompting (boosts recognition for specific words) # NOTE: Cannot use both prompt and keyterms_prompt! keyterms_prompt=None, # keyterms_prompt=["Pipecat", "AssemblyAI", "OpenAI", "Cartesia"], # keyterms_prompt=["Python", "JavaScript", "TypeScript", "API"], - # ==================================================================== # Diarization (Speaker Identification) # ==================================================================== - # Enable speaker labels (identifies different speakers) speaker_labels=None, # None or True # speaker_labels=True, - # ==================================================================== # Audio Configuration # ==================================================================== - # Audio sample rate (Hz) # sample_rate=16000, # sample_rate=8000, - # Audio encoding format # encoding="pcm_s16le", # Default: 16-bit PCM # encoding="pcm_mulaw", # μ-law encoding (telephony) - # ==================================================================== # Other Options # ==================================================================== - # Format transcript turns (applies formatting rules) # format_turns=True, # Default # format_turns=False, - # Language detection (only for universal-streaming-multilingual) # language_detection=True, ) # Log connection parameters for debugging - logger.info("="*80) + logger.info("=" * 80) logger.info("CONNECTION PARAMETERS:") logger.info(f" speech_model: {connection_params.speech_model}") logger.info(f" min_turn_silence: {connection_params.min_turn_silence}") @@ -156,27 +141,26 @@ async def main(): logger.info(f" keyterms_prompt: {connection_params.keyterms_prompt}") logger.info(f" speaker_labels: {connection_params.speaker_labels}") logger.info(f" format_turns: {connection_params.format_turns}") - logger.info(f" end_of_turn_confidence_threshold: {connection_params.end_of_turn_confidence_threshold}") + logger.info( + f" end_of_turn_confidence_threshold: {connection_params.end_of_turn_confidence_threshold}" + ) logger.info(f" language_detection: {connection_params.language_detection}") - logger.info("="*80) + logger.info("=" * 80) # AssemblyAI Speech-to-Text Service stt = AssemblyAISTTService( api_key=os.getenv("ASSEMBLYAI_API_KEY"), connection_params=connection_params, - # Turn Detection Mode # True = Pipecat mode (VAD + Smart Turn controls turns) # False = STT mode (u3-rt-pro model controls turns) vad_force_turn_endpoint=True, - # Speaker Formatting (only used if speaker_labels=True) # None = Just log speaker IDs, don't modify transcript speaker_format=None, # speaker_format="{text}", # speaker_format="{speaker}: {text}", # speaker_format="[{speaker}] {text}", - # Additional available parameters (uncomment to use): # should_interrupt=True, # Only for STT mode ) diff --git a/test_assemblyai_interactive.py b/test_assemblyai_interactive.py index ce468ab3d..c5ec0b429 100755 --- a/test_assemblyai_interactive.py +++ b/test_assemblyai_interactive.py @@ -52,11 +52,11 @@ async def run_bot( test_dynamic_updates: Optional[callable] = None, ): """Run the voice bot with specified configuration.""" - logger.info("="*80) + logger.info("=" * 80) logger.info(f"TEST: {test_name}") - logger.info("="*80) + logger.info("=" * 80) logger.info("Starting bot... Speak into your microphone after you hear the greeting!") - logger.info("="*80) + logger.info("=" * 80) # Create local audio transport transport = LocalAudioTransport( @@ -150,6 +150,7 @@ async def run_bot( # === BASIC CONFIGURATION (1-3) === + async def test_01_basic_100ms(): """Test 1: Basic default configuration (100ms).""" connection_params = AssemblyAIConnectionParams( @@ -179,6 +180,7 @@ async def test_03_custom_500ms(): # === PROMPTING & WARNINGS (4-7) === + async def test_04_max_warning(): """Test 4: max_turn_silence warning (should be overridden).""" logger.warning("⚠️ EXPECT WARNING: max_turn_silence will be overridden") @@ -230,6 +232,7 @@ async def test_07_keyterms_difficult(): # === DIARIZATION (8-9) === + async def test_08_diarization_basic(): """Test 8: Basic diarization (speaker IDs logged).""" connection_params = AssemblyAIConnectionParams( @@ -258,6 +261,7 @@ async def test_09_diarization_xml(): # === DYNAMIC UPDATES - SINGLE PARAMETER (10-13) === + async def test_10_dynamic_keyterms(): """Test 10: Dynamic keyterms update with difficult names.""" connection_params = AssemblyAIConnectionParams( @@ -265,16 +269,16 @@ async def test_10_dynamic_keyterms(): ) async def dynamic_update(task): - logger.info("\n" + "="*80) + logger.info("\n" + "=" * 80) logger.info("PHASE 1: No keyterms boosting") logger.info(" Try saying: Xiomara, Saoirse, Krzystof") logger.info(" (May not transcribe correctly)") - logger.info("="*80) + logger.info("=" * 80) await asyncio.sleep(15) - logger.info("\n" + "="*80) + logger.info("\n" + "=" * 80) logger.info("🔄 UPDATING: Adding keyterms boost") - logger.info("="*80) + logger.info("=" * 80) await task.queue_frame( STTUpdateSettingsFrame( delta=AssemblyAISTTSettings( @@ -284,11 +288,11 @@ async def test_10_dynamic_keyterms(): ) ) ) - logger.info("\n" + "="*80) + logger.info("\n" + "=" * 80) logger.info("PHASE 2: Keyterms NOW boosted") logger.info(" Say the same names again: Xiomara, Saoirse, Krzystof") logger.info(" (Should transcribe better now!)") - logger.info("="*80) + logger.info("=" * 80) logger.info("🔄 This test has 2 phases:") logger.info(" Phase 1 (15s): No boosting - names may be wrong") @@ -308,29 +312,27 @@ async def test_11_dynamic_silence(): ) async def dynamic_update(task): - logger.info("\n" + "="*80) + logger.info("\n" + "=" * 80) logger.info("PHASE 1: Quick responses (100ms silence threshold)") logger.info(" Speak normally - bot responds quickly") - logger.info("="*80) + logger.info("=" * 80) await asyncio.sleep(10) - logger.info("\n" + "="*80) + logger.info("\n" + "=" * 80) logger.info("🔄 UPDATING: Changing silence from 100ms → 3000ms (3 seconds!)") - logger.info("="*80) + logger.info("=" * 80) await task.queue_frame( STTUpdateSettingsFrame( delta=AssemblyAISTTSettings( - connection_params=AssemblyAIConnectionParams( - min_turn_silence=3000 - ) + connection_params=AssemblyAIConnectionParams(min_turn_silence=3000) ) ) ) - logger.info("\n" + "="*80) + logger.info("\n" + "=" * 80) logger.info("PHASE 2: Patient responses (3 second silence threshold)") logger.info(" Bot will wait 3 full seconds before responding") logger.info(" Try pausing mid-sentence - bot should NOT interrupt") - logger.info("="*80) + logger.info("=" * 80) logger.info("🔄 Dramatic change: 100ms → 3000ms after 10 seconds") await run_bot( @@ -347,16 +349,16 @@ async def test_12_dynamic_prompt(): ) async def dynamic_update(task): - logger.info("\n" + "="*80) + logger.info("\n" + "=" * 80) logger.info("PHASE 1: Default prompt (no keyterms)") logger.info(" Try saying: Xiomara, Saoirse, Krzystof") logger.info(" (May not transcribe correctly)") - logger.info("="*80) + logger.info("=" * 80) await asyncio.sleep(15) - logger.info("\n" + "="*80) + logger.info("\n" + "=" * 80) logger.info("🔄 UPDATING: Adding custom prompt with keyterms") - logger.info("="*80) + logger.info("=" * 80) custom_prompt = """Transcribe verbatim. Rules: 1) Always include punctuation in output. 2) Use period/question mark ONLY for complete sentences. @@ -368,17 +370,15 @@ Pay special attention to these names and transcribe them exactly: Xiomara, Saoir await task.queue_frame( STTUpdateSettingsFrame( delta=AssemblyAISTTSettings( - connection_params=AssemblyAIConnectionParams( - prompt=custom_prompt - ) + connection_params=AssemblyAIConnectionParams(prompt=custom_prompt) ) ) ) - logger.info("\n" + "="*80) + logger.info("\n" + "=" * 80) logger.info("PHASE 2: Prompt with keyterms NOW active") logger.info(" Say the same names again: Xiomara, Saoirse, Krzystof") logger.info(" (Should transcribe better now!)") - logger.info("="*80) + logger.info("=" * 80) logger.info("🔄 This test has 2 phases:") logger.info(" Phase 1 (15s): Default prompt - names may be wrong") @@ -403,9 +403,7 @@ async def test_13_dynamic_clear_keyterms(): await task.queue_frame( STTUpdateSettingsFrame( delta=AssemblyAISTTSettings( - connection_params=AssemblyAIConnectionParams( - keyterms_prompt=[] - ) + connection_params=AssemblyAIConnectionParams(keyterms_prompt=[]) ) ) ) @@ -421,6 +419,7 @@ async def test_13_dynamic_clear_keyterms(): # === DYNAMIC UPDATES - MULTIPLE PARAMETERS (14-15) === + async def test_14_multi_param_update(): """Test 14: Update multiple parameters at once.""" connection_params = AssemblyAIConnectionParams( @@ -464,9 +463,7 @@ async def test_15_complex_sequence(): await task.queue_frame( STTUpdateSettingsFrame( delta=AssemblyAISTTSettings( - connection_params=AssemblyAIConnectionParams( - keyterms_prompt=["Pipecat"] - ) + connection_params=AssemblyAIConnectionParams(keyterms_prompt=["Pipecat"]) ) ) ) @@ -476,9 +473,7 @@ async def test_15_complex_sequence(): await task.queue_frame( STTUpdateSettingsFrame( delta=AssemblyAISTTSettings( - connection_params=AssemblyAIConnectionParams( - min_turn_silence=200 - ) + connection_params=AssemblyAIConnectionParams(min_turn_silence=200) ) ) ) @@ -506,6 +501,7 @@ async def test_15_complex_sequence(): # === MODE COMPARISON (16-17) === + async def test_16_pipecat_mode(): """Test 16: Pipecat mode (VAD + Smart Turn controls turns).""" connection_params = AssemblyAIConnectionParams( @@ -538,6 +534,7 @@ async def test_17_stt_mode(): # === STT MODE TIMING EXPERIMENTS (18-20) === + async def test_18_stt_long_max_short_min(): """Test 18: STT mode - Long max_turn_silence + Short min (5000ms + 100ms).""" connection_params = AssemblyAIConnectionParams( @@ -596,6 +593,7 @@ async def test_20_stt_both_short(): # === EDGE CASES (21-23) === + async def test_21_very_long_silence(): """Test 21: Very long silence threshold (STT mode only).""" connection_params = AssemblyAIConnectionParams( @@ -645,9 +643,9 @@ async def test_23_keyterms_plus_diarization(): def show_menu(): """Display the comprehensive test menu.""" - print("\n" + "="*80) + print("\n" + "=" * 80) print("AssemblyAI u3-rt-pro Comprehensive Test Suite") - print("="*80) + print("=" * 80) print("\n📋 BASIC CONFIGURATION (1-3)") print(" 1. Basic Default (100ms)") print(" 2. Custom Silence (200ms)") @@ -688,7 +686,7 @@ def show_menu(): print(" 23. Keyterms + Diarization Combined") print("\n 0. Exit") - print("\n" + "="*80) + print("\n" + "=" * 80) async def main(): @@ -735,6 +733,7 @@ async def main(): except Exception as e: logger.error(f"Test failed with error: {e}") import traceback + traceback.print_exc() input("\n\nPress Enter to return to menu...") diff --git a/test_assemblyai_u3pro.py b/test_assemblyai_u3pro.py index ace700753..236ab9b50 100644 --- a/test_assemblyai_u3pro.py +++ b/test_assemblyai_u3pro.py @@ -50,6 +50,7 @@ from pipecat.transports.local.audio import LocalAudioTransport, LocalAudioTransp load_dotenv() + # Test configuration class TestConfig: """Centralized test configuration.""" @@ -205,9 +206,7 @@ async def test_custom_min_silence(): logger.info("TEST 2: Custom min_turn_silence") logger.info("=" * 80) - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", min_turn_silence=200 - ) + connection_params = AssemblyAIConnectionParams(speech_model="u3-rt-pro", min_turn_silence=200) task, transport = await create_basic_voice_agent(connection_params) @@ -307,9 +306,7 @@ async def test_diarization_no_format(): logger.info("TEST 10: Diarization Enabled (No Formatting)") logger.info("=" * 80) - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", speaker_labels=True - ) + connection_params = AssemblyAIConnectionParams(speech_model="u3-rt-pro", speaker_labels=True) task, transport = await create_basic_voice_agent(connection_params) @@ -327,9 +324,7 @@ async def test_diarization_xml_format(): logger.info("TEST 11: Diarization with XML Formatting") logger.info("=" * 80) - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", speaker_labels=True - ) + connection_params = AssemblyAIConnectionParams(speech_model="u3-rt-pro", speaker_labels=True) task, transport = await create_basic_voice_agent( connection_params, speaker_format="<{speaker}>{text}" @@ -516,9 +511,7 @@ def main(): "prompt_keyterms_conflict, keyterms, diarization, diarization_xml, " "dynamic_keyterms, dynamic_silence, multi_param, all)", ) - parser.add_argument( - "--interactive", action="store_true", help="Run in interactive mode" - ) + parser.add_argument("--interactive", action="store_true", help="Run in interactive mode") args = parser.parse_args() From 6968d83ccb1a3e9961df693aadaad1cd0e3636ed Mon Sep 17 00:00:00 2001 From: zack Date: Sun, 1 Mar 2026 11:44:51 -0500 Subject: [PATCH 14/23] Add changelog entries for PR #3856 --- changelog/3856.added.md | 1 + changelog/3856.changed.md | 1 + changelog/3856.fixed.md | 1 + 3 files changed, 3 insertions(+) create mode 100644 changelog/3856.added.md create mode 100644 changelog/3856.changed.md create mode 100644 changelog/3856.fixed.md diff --git a/changelog/3856.added.md b/changelog/3856.added.md new file mode 100644 index 000000000..0d620a925 --- /dev/null +++ b/changelog/3856.added.md @@ -0,0 +1 @@ +Add AssemblyAI u3-rt-pro model support with STT-controlled turn detection mode diff --git a/changelog/3856.changed.md b/changelog/3856.changed.md new file mode 100644 index 000000000..cb714aeba --- /dev/null +++ b/changelog/3856.changed.md @@ -0,0 +1 @@ +Rename AssemblyAI min_end_of_turn_silence_when_confident parameter to min_turn_silence (old name still supported with deprecation warning) diff --git a/changelog/3856.fixed.md b/changelog/3856.fixed.md new file mode 100644 index 000000000..d9a63a692 --- /dev/null +++ b/changelog/3856.fixed.md @@ -0,0 +1 @@ +Add beta feature warning when using custom prompts with AssemblyAI From 36b9c057309955d241751a8efcdb30897caddd73 Mon Sep 17 00:00:00 2001 From: zack Date: Sun, 1 Mar 2026 11:45:24 -0500 Subject: [PATCH 15/23] Fix changelog entries to use proper markdown bullet format --- changelog/3856.added.md | 2 +- changelog/3856.changed.md | 2 +- changelog/3856.fixed.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog/3856.added.md b/changelog/3856.added.md index 0d620a925..95b656058 100644 --- a/changelog/3856.added.md +++ b/changelog/3856.added.md @@ -1 +1 @@ -Add AssemblyAI u3-rt-pro model support with STT-controlled turn detection mode +- Add AssemblyAI u3-rt-pro model support with STT-controlled turn detection mode diff --git a/changelog/3856.changed.md b/changelog/3856.changed.md index cb714aeba..72331c068 100644 --- a/changelog/3856.changed.md +++ b/changelog/3856.changed.md @@ -1 +1 @@ -Rename AssemblyAI min_end_of_turn_silence_when_confident parameter to min_turn_silence (old name still supported with deprecation warning) +- Rename AssemblyAI min_end_of_turn_silence_when_confident parameter to min_turn_silence (old name still supported with deprecation warning) diff --git a/changelog/3856.fixed.md b/changelog/3856.fixed.md index d9a63a692..c31fe8ddf 100644 --- a/changelog/3856.fixed.md +++ b/changelog/3856.fixed.md @@ -1 +1 @@ -Add beta feature warning when using custom prompts with AssemblyAI +- Add beta feature warning when using custom prompts with AssemblyAI From cb7e6127387ee802fcbed9cbaa4a9eabc6bd875f Mon Sep 17 00:00:00 2001 From: zack Date: Sun, 1 Mar 2026 11:51:51 -0500 Subject: [PATCH 16/23] Remove test files and testing documentation from PR --- TESTING_CHECKLIST.md | 273 ------------ TESTING_SETUP.md | 310 -------------- test_assemblyai_custom.py | 240 ----------- test_assemblyai_interactive.py | 749 --------------------------------- test_assemblyai_u3pro.py | 582 ------------------------- 5 files changed, 2154 deletions(-) delete mode 100644 TESTING_CHECKLIST.md delete mode 100644 TESTING_SETUP.md delete mode 100755 test_assemblyai_custom.py delete mode 100755 test_assemblyai_interactive.py delete mode 100644 test_assemblyai_u3pro.py diff --git a/TESTING_CHECKLIST.md b/TESTING_CHECKLIST.md deleted file mode 100644 index 8d2d147d4..000000000 --- a/TESTING_CHECKLIST.md +++ /dev/null @@ -1,273 +0,0 @@ -# AssemblyAI u3-rt-pro Testing Checklist - -## Test Environment Setup -- [ ] Install dependencies: `uv sync --group dev --all-extras` -- [ ] Set up `.env` file with API keys -- [ ] Verify LiveKit connection -- [ ] Run basic voice agent test - ---- - -## Feature Testing Checklist - -### ✅ Basic Configuration Tests - -#### Test 1: Default u3-rt-pro Configuration -- [ ] **Setup:** Create service with default params -- [ ] **Expected:** No errors, uses u3-rt-pro model with 100ms min/max -- [ ] **Verify:** Check logs for connection confirmation - -#### Test 2: Custom min_turn_silence -- [ ] **Setup:** Set `min_turn_silence=200` -- [ ] **Expected:** Both min and max set to 200ms -- [ ] **Verify:** Speak short phrases, observe turn detection timing - -#### Test 3: User sets max_turn_silence (Warning Test) -- [ ] **Setup:** Set `max_turn_silence=500` in connection params -- [ ] **Expected:** Warning logged, value overridden to match min -- [ ] **Verify:** Check logs for warning message - ---- - -### ✅ Prompting Tests - -#### Test 4: No Prompt (Default - Recommended) -- [ ] **Setup:** Don't set prompt parameter -- [ ] **Expected:** Uses default prompt, 88% accuracy, no warnings -- [ ] **Verify:** Transcription quality is good - -#### Test 5: Custom Prompt (Warning Test) -- [ ] **Setup:** Set custom prompt in connection params -- [ ] **Expected:** Warning logged about testing without prompt first -- [ ] **Verify:** Check logs for prompt warning - -#### Test 6: Prompt + Keyterms Conflict (Error Test) -- [ ] **Setup:** Set both `prompt` and `keyterms_prompt` at init -- [ ] **Expected:** ValueError raised with helpful error message -- [ ] **Verify:** Service fails to initialize with clear error - ---- - -### ✅ Keyterms Prompting Tests - -#### Test 7: Basic Keyterms at Init -- [ ] **Setup:** Set `keyterms_prompt=["Pipecat", "AssemblyAI", "Universal-3"]` -- [ ] **Expected:** Terms are boosted in recognition -- [ ] **Verify:** Say the boosted terms, check accuracy - -#### Test 8: Empty Keyterms (No Boosting) -- [ ] **Setup:** Set `keyterms_prompt=[]` -- [ ] **Expected:** No boosting, default behavior -- [ ] **Verify:** Normal transcription - ---- - -### ✅ Diarization Tests - -#### Test 9: Diarization Disabled (Default) -- [ ] **Setup:** Don't set `speaker_labels` parameter -- [ ] **Expected:** No speaker info in transcripts -- [ ] **Verify:** TranscriptionFrame.user_id is default user_id - -#### Test 10: Diarization Enabled (No Formatting) -- [ ] **Setup:** Set `speaker_labels=True` -- [ ] **Expected:** Speaker ID in user_id field, plain text -- [ ] **Verify:** Multiple speakers show different IDs (Speaker A, Speaker B) - -#### Test 11: Diarization with XML Formatting -- [ ] **Setup:** Set `speaker_labels=True`, `speaker_format="<{speaker}>{text}"` -- [ ] **Expected:** Text includes speaker tags: `Hello` -- [ ] **Verify:** Formatted text in transcript, speaker ID in user_id - -#### Test 12: Diarization with Colon Prefix -- [ ] **Setup:** Set `speaker_labels=True`, `speaker_format="{speaker}: {text}"` -- [ ] **Expected:** Text includes prefix: `Speaker A: Hello` -- [ ] **Verify:** Formatted text, multiple speakers distinguishable - ---- - -### ✅ Dynamic Updates Tests - -#### Test 13: Dynamic Keyterms Update (Stage 1 → Stage 2) -- [ ] **Setup:** Start with empty keyterms, update mid-conversation -- [ ] **Expected:** New keyterms take effect immediately -- [ ] **Test Steps:** - 1. Start conversation with no keyterms - 2. Send update frame with `keyterms_prompt=["cardiology", "Dr. Smith"]` - 3. Say the new terms -- [ ] **Verify:** Improved recognition after update - -#### Test 14: Clear Keyterms (Reset Context) -- [ ] **Setup:** Start with keyterms, clear them mid-stream -- [ ] **Expected:** Context biasing removed -- [ ] **Test Steps:** - 1. Start with `keyterms_prompt=["test", "words"]` - 2. Send update frame with `keyterms_prompt=[]` -- [ ] **Verify:** No more boosting after clear - -#### Test 15: Dynamic Silence Parameters -- [ ] **Setup:** Update `max_turn_silence` mid-stream -- [ ] **Expected:** Turn detection timing changes -- [ ] **Test Steps:** - 1. Start with default (1200ms) - 2. Update to `max_turn_silence=5000` (for reading numbers) - 3. Pause longer between words - 4. Update back to `max_turn_silence=1200` -- [ ] **Verify:** Longer pauses tolerated when increased - -#### Test 16: Dynamic Prompt Update -- [ ] **Setup:** Update prompt mid-stream -- [ ] **Expected:** New instructions take effect -- [ ] **Test Steps:** - 1. Start with default prompt - 2. Send update with custom prompt -- [ ] **Verify:** Behavior changes according to new prompt - -#### Test 17: Multiple Parameters at Once -- [ ] **Setup:** Update keyterms, max_turn_silence, and min_end_of_turn together -- [ ] **Expected:** All parameters updated in single WebSocket message -- [ ] **Verify:** Check logs for single UpdateConfiguration message - -#### Test 18: Dynamic Update - Prompt + Keyterms Conflict (Error) -- [ ] **Setup:** Try to update both prompt and keyterms_prompt in same update -- [ ] **Expected:** ValueError raised -- [ ] **Verify:** Update fails with clear error message - ---- - -### ✅ Turn Detection Mode Tests - -#### Test 19: Pipecat Mode (vad_force_turn_endpoint=True) - Default -- [ ] **Setup:** Use default settings (Pipecat mode) -- [ ] **Expected:** - - ForceEndpoint sent on VAD stop - - Smart Turn Analyzer makes decisions - - min=max=100ms for u3-rt-pro -- [ ] **Verify:** Fast finals, Smart Turn handles completeness - -#### Test 20: STT Mode (vad_force_turn_endpoint=False) - u3-rt-pro only -- [ ] **Setup:** Set `vad_force_turn_endpoint=False` with u3-rt-pro -- [ ] **Expected:** - - AssemblyAI controls turn endings - - SpeechStarted message triggers interruptions - - UserStarted/StoppedSpeakingFrame emitted -- [ ] **Verify:** Turn detection from AssemblyAI model - -#### Test 21: STT Mode with universal-streaming (Error Test) -- [ ] **Setup:** Set `vad_force_turn_endpoint=False` with universal-streaming -- [ ] **Expected:** ValueError raised (requires u3-rt-pro) -- [ ] **Verify:** Service fails with clear error - ---- - -### ✅ Language Detection Tests (If Multilingual Model) - -#### Test 22: Language Detection Enabled -- [ ] **Setup:** Use `universal-streaming-multilingual` with `language_detection=True` -- [ ] **Expected:** Language codes in transcripts -- [ ] **Verify:** Speak different languages, check language_code field - -#### Test 23: Language Confidence Threshold -- [ ] **Setup:** Enable language detection -- [ ] **Expected:** High confidence (≥0.7) → detected language, Low → fallback to English -- [ ] **Verify:** Check logs for confidence warnings - ---- - -### ✅ Edge Cases & Error Handling - -#### Test 24: WebSocket Disconnect During Update -- [ ] **Setup:** Simulate disconnect, try update -- [ ] **Expected:** Error logged, update queued for reconnection -- [ ] **Verify:** Graceful handling, no crash - -#### Test 25: Invalid Parameter Types -- [ ] **Setup:** Send update with wrong type (e.g., keyterms_prompt as string) -- [ ] **Expected:** Warning logged, parameter skipped -- [ ] **Verify:** Service continues, invalid param ignored - -#### Test 26: Unknown Parameter in Update -- [ ] **Setup:** Send update with unsupported parameter (e.g., `language`) -- [ ] **Expected:** Warning logged about parameter -- [ ] **Verify:** Other valid params still updated - ---- - -### ✅ Integration Tests - -#### Test 27: Full Voice Agent Flow (Multi-Stage) -- [ ] **Setup:** Complete voice agent with stage transitions -- [ ] **Test Steps:** - 1. Greeting stage (general keyterms) - 2. Name collection stage (name keyterms) - 3. Account number stage (number keyterms, longer silence) - 4. Medical info stage (medical keyterms) - 5. Closing stage (goodbye keyterms) -- [ ] **Verify:** Each stage has appropriate keyterms and timing - -#### Test 28: Diarization + Dynamic Updates -- [ ] **Setup:** Enable diarization, update keyterms mid-stream -- [ ] **Expected:** Both features work together -- [ ] **Verify:** Speaker IDs persist, keyterms update correctly - -#### Test 29: Interruption Handling -- [ ] **Setup:** Bot speaking, user interrupts -- [ ] **Expected:** - - Pipecat mode: VAD + Smart Turn handles - - STT mode: SpeechStarted triggers interrupt -- [ ] **Verify:** Bot stops, user speech processed - ---- - -## Testing Results Template - -``` -| Test # | Feature | Status | Notes | -|--------|---------|--------|-------| -| 1 | Default Config | ✅ PASS | | -| 2 | Custom min_silence | ✅ PASS | | -| 3 | max_silence Warning | ✅ PASS | | -| ... | ... | ... | ... | -``` - ---- - -## Expected Outcomes Summary - -### ✅ Should Work (No Errors) -- Default configuration -- Custom min_turn_silence -- Keyterms prompting -- Diarization with/without formatting -- Dynamic updates (one parameter or multiple) -- Pipecat mode turn detection - -### ⚠️ Should Warn (Logs Warning, Continues) -- Custom prompt set at init -- max_turn_silence set (overridden) -- Invalid parameter types in updates -- Language update attempted -- Prompt used with universal-streaming - -### ❌ Should Error (Raises Exception, Stops) -- prompt + keyterms_prompt at init -- prompt + keyterms_prompt in same update -- vad_force_turn_endpoint=False with universal-streaming - ---- - -## Quick Test Commands - -```bash -# Run basic test -python test_assemblyai_u3pro.py --test basic - -# Run specific test -python test_assemblyai_u3pro.py --test diarization - -# Run all tests -python test_assemblyai_u3pro.py --test all - -# Interactive mode -python test_assemblyai_u3pro.py --interactive -``` diff --git a/TESTING_SETUP.md b/TESTING_SETUP.md deleted file mode 100644 index fa1dca462..000000000 --- a/TESTING_SETUP.md +++ /dev/null @@ -1,310 +0,0 @@ -# AssemblyAI u3-rt-pro Testing Setup Guide - -## Quick Start - -### 1. Setup Environment - -```bash -# Copy API keys -cp .env.testing .env - -# Install dependencies -uv sync --group dev --all-extras --no-extra gstreamer --no-extra krisp - -# Make test script executable -chmod +x test_assemblyai_u3pro.py -``` - -### 2. Ensure Audio Devices - -Make sure you have: -- **Microphone** enabled and working -- **Speakers/headphones** connected -- Audio permissions granted (macOS will prompt on first run) - -### 3. Run Tests - -```bash -# Run a specific test -python test_assemblyai_u3pro.py --test basic - -# Interactive mode (choose from menu) -python test_assemblyai_u3pro.py --interactive - -# Run all tests sequentially -python test_assemblyai_u3pro.py --test all -``` - ---- - -## Available Tests - -### Basic Configuration Tests -```bash -# Test 1: Default configuration (min=max=100ms) -python test_assemblyai_u3pro.py --test basic - -# Test 2: Custom min_turn_silence -python test_assemblyai_u3pro.py --test custom_min - -# Test 3: max_turn_silence warning (should be overridden) -python test_assemblyai_u3pro.py --test max_warning -``` - -### Prompting Tests -```bash -# Test 5: Custom prompt warning -python test_assemblyai_u3pro.py --test prompt_warning - -# Test 6: Prompt + keyterms conflict (should error) -python test_assemblyai_u3pro.py --test prompt_keyterms_conflict - -# Test 7: Basic keyterms prompting -python test_assemblyai_u3pro.py --test keyterms -``` - -### Diarization Tests -```bash -# Test 10: Diarization without formatting -python test_assemblyai_u3pro.py --test diarization - -# Test 11: Diarization with XML formatting -python test_assemblyai_u3pro.py --test diarization_xml -``` - -### Dynamic Updates Tests -```bash -# Test 13: Dynamic keyterms (multi-stage) -python test_assemblyai_u3pro.py --test dynamic_keyterms - -# Test 15: Dynamic silence parameters -python test_assemblyai_u3pro.py --test dynamic_silence - -# Test 17: Multiple parameters at once -python test_assemblyai_u3pro.py --test multi_param -``` - ---- - -## Test Execution Flow - -### For Each Test: - -1. **Start the test script** - ```bash - python test_assemblyai_u3pro.py --test - ``` - -2. **Wait for "started" message** indicating the bot is ready - -3. **Speak into your microphone** to test - the bot will: - - Transcribe your speech (you'll see `📝 TRANSCRIPTION:` logs) - - Process through the LLM - - Respond with voice through your speakers - -4. **Observe logs** for: - - ✅ Success indicators - - ⚠️ Warning messages - - ❌ Error messages - - 📝 Transcription output - -5. **Verify expected behavior** against checklist - -6. **Stop test** with Ctrl+C - ---- - -## Expected Test Outcomes - -### Should Pass (✅) -- Basic configuration creates service -- Custom parameters are applied -- Keyterms boost recognition -- Diarization shows speaker IDs -- Dynamic updates work without errors - -### Should Warn (⚠️) -Check logs for warnings: -- "We recommend testing at first with no prompt" -- "max_turn_silence is not used in Pipecat mode" -- "Unknown setting for AssemblyAI STT service" - -### Should Error (❌) -Should raise ValueError and fail to start: -- Both prompt and keyterms_prompt set at init -- Both prompt and keyterms_prompt in same update -- vad_force_turn_endpoint=False with universal-streaming - ---- - -## Debugging Tips - -### Check Logs -```bash -# Run with verbose logging -LOGURU_LEVEL=DEBUG python test_assemblyai_u3pro.py --test -``` - -### Common Issues - -**Issue: "WebSocket connection failed"** -- Check ASSEMBLYAI_API_KEY is correct -- Verify network connection -- Check firewall settings - -**Issue: "No audio input/output"** -- Verify microphone permissions (System Preferences → Security & Privacy → Microphone) -- Check default audio devices in System Preferences → Sound -- Test microphone with another app first -- Make sure no other app is using the microphone - -**Issue: "No transcriptions appearing"** -- Verify microphone permissions -- Check audio levels (speak louder or move closer to mic) -- Speak clearly and wait for VAD to detect -- Check if microphone is muted - -**Issue: "Can't hear bot responses"** -- Check speaker/headphone volume -- Verify correct output device is selected -- Check terminal for TTS errors - -**Issue: "Service fails to start"** -- Check all API keys in .env -- Run `uv sync` to ensure dependencies installed -- Check Python version (3.10+) - ---- - -## Manual Testing Checklist - -After running automated tests, manually verify: - -### ✅ Audio Quality -- [ ] Transcriptions are accurate -- [ ] No distortion or dropouts -- [ ] Latency is acceptable - -### ✅ Turn Detection -- [ ] Bot waits for user to finish speaking -- [ ] No premature cutoffs -- [ ] Handles natural pauses correctly - -### ✅ Interruptions -- [ ] Can interrupt bot mid-sentence -- [ ] Interruption is smooth -- [ ] Bot stops speaking immediately - -### ✅ Diarization (if enabled) -- [ ] Multiple speakers detected correctly -- [ ] Speaker IDs consistent -- [ ] Speaker formatting works - -### ✅ Dynamic Updates -- [ ] Keyterms update without disconnection -- [ ] Turn detection timing changes work -- [ ] Updates logged correctly - ---- - -## Test Results Recording - -### Use this template: - -```markdown -## Test Run: YYYY-MM-DD - -| Test # | Test Name | Status | Notes | -|--------|-----------|--------|-------| -| 1 | basic | ✅ PASS | Transcriptions working | -| 2 | custom_min | ✅ PASS | Turn timing changed | -| 3 | max_warning | ✅ PASS | Warning logged | -| 5 | prompt_warning | ✅ PASS | Warning shown | -| 6 | prompt_keyterms_conflict | ✅ PASS | ValueError raised | -| 7 | keyterms | ✅ PASS | Terms boosted | -| 10 | diarization | ✅ PASS | Speaker IDs correct | -| 11 | diarization_xml | ✅ PASS | XML tags shown | -| 13 | dynamic_keyterms | ✅ PASS | Updates worked | -| 15 | dynamic_silence | ✅ PASS | Timing adjusted | -| 17 | multi_param | ✅ PASS | All params updated | - -### Issues Found: -- None - -### Notes: -- All tests passed successfully -- Latency is excellent (sub-300ms) -- Diarization accuracy is good -``` - ---- - -## Advanced Testing - -### Custom Test Scenarios - -Create custom tests by modifying `test_assemblyai_u3pro.py`: - -```python -async def test_my_custom_scenario(): - """My custom test scenario.""" - logger.info("Testing my specific use case") - - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - # Your custom params here - ) - - task, transport = await create_basic_voice_agent(connection_params) - - # Your test logic here - - runner = PipelineRunner() - await runner.run(task) -``` - -### Stress Testing - -Test with: -- Multiple simultaneous speakers -- Long conversations (30+ minutes) -- Rapid speech -- Heavy accents -- Background noise -- Poor network conditions - ---- - -## Reporting Issues - -When reporting issues, include: - -1. **Test name and number** -2. **Full error message and stack trace** -3. **Relevant log output** (use LOGURU_LEVEL=DEBUG) -4. **Configuration used** (connection_params) -5. **Expected vs actual behavior** -6. **Steps to reproduce** - ---- - -## Next Steps - -After testing: - -1. ✅ Mark completed tests in `TESTING_CHECKLIST.md` -2. 📝 Document any issues found -3. 🐛 Create GitHub issues for bugs -4. ✨ Suggest improvements -5. 📊 Share results with team - ---- - -## Contact - -Questions? Issues? -- Check `TESTING_CHECKLIST.md` for detailed test descriptions -- Review logs with `LOGURU_LEVEL=DEBUG` -- Reach out to the team with your findings - -Happy testing! 🎯 diff --git a/test_assemblyai_custom.py b/test_assemblyai_custom.py deleted file mode 100755 index e8e0a28d2..000000000 --- a/test_assemblyai_custom.py +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env python3 -"""Custom AssemblyAI u3-rt-pro Test Script -Easy parameter tweaking for experimentation - -Edit the CONFIGURATION section below to test different settings! -""" - -import asyncio -import os -import sys - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMRunFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) -from pipecat.services.assemblyai.models import AssemblyAIConnectionParams -from pipecat.services.assemblyai.stt import AssemblyAISTTService -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.local.audio import LocalAudioTransport, LocalAudioTransportParams - -load_dotenv(override=True) - -# ============================================================================ -# CONFIGURATION -# ============================================================================ - -# Log Level: "DEBUG" for detailed logs, "INFO" for normal operation -LOG_LEVEL = "INFO" - -# ============================================================================ -# BOT IMPLEMENTATION -# ============================================================================ - - -async def main(): - """Run the custom test bot with your configured parameters.""" - # Setup logging - logger.remove(0) - logger.add(sys.stderr, level=LOG_LEVEL) - - logger.info("=" * 80) - logger.info("AssemblyAI u3-rt-pro Custom Test") - logger.info("=" * 80) - logger.info("Starting bot... Speak after you hear the greeting!") - logger.info("=" * 80) - - # Create local audio transport - transport = LocalAudioTransport( - LocalAudioTransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - ) - ) - - # ======================================================================== - # EDIT PARAMETERS HERE - # ======================================================================== - - # Build connection params - connection_params = AssemblyAIConnectionParams( - # ==================================================================== - # Model Selection - # ==================================================================== - speech_model="u3-rt-pro", - # speech_model="universal-streaming-english", - # speech_model="universal-streaming-multilingual", - # ==================================================================== - # Turn Detection Timing - # ==================================================================== - # Minimum silence when confident about end of turn (milliseconds) - # Default: 100ms | Higher = more patient | Lower = faster responses - # Only used in Pipecat mode (vad_force_turn_endpoint=True) - min_turn_silence=100000, - # min_turn_silence=200, - # min_turn_silence=300, - # Maximum turn silence (milliseconds) - # WARNING: In Pipecat mode (vad_force_turn_endpoint=True), this is - # automatically set equal to min_turn_silence - # to avoid double turn detection. Only used as-is in STT mode. - max_turn_silence=500, - # End of turn confidence threshold (0.0 to 1.0) - # Higher = requires more confidence before ending turn - # end_of_turn_confidence_threshold=0.8, - # ==================================================================== - # Prompting & Boosting - # ==================================================================== - # Custom Prompt (WARNING: test carefully, default is optimized!) - # None = Use AssemblyAI's optimized default (recommended for 88% accuracy) - prompt=None, - # prompt="Transcribe speech with focus on technical terms.", - # prompt="Context: Medical conversation. Transcribe accurately.", - # Keyterms Prompting (boosts recognition for specific words) - # NOTE: Cannot use both prompt and keyterms_prompt! - keyterms_prompt=None, - # keyterms_prompt=["Pipecat", "AssemblyAI", "OpenAI", "Cartesia"], - # keyterms_prompt=["Python", "JavaScript", "TypeScript", "API"], - # ==================================================================== - # Diarization (Speaker Identification) - # ==================================================================== - # Enable speaker labels (identifies different speakers) - speaker_labels=None, # None or True - # speaker_labels=True, - # ==================================================================== - # Audio Configuration - # ==================================================================== - # Audio sample rate (Hz) - # sample_rate=16000, - # sample_rate=8000, - # Audio encoding format - # encoding="pcm_s16le", # Default: 16-bit PCM - # encoding="pcm_mulaw", # μ-law encoding (telephony) - # ==================================================================== - # Other Options - # ==================================================================== - # Format transcript turns (applies formatting rules) - # format_turns=True, # Default - # format_turns=False, - # Language detection (only for universal-streaming-multilingual) - # language_detection=True, - ) - - # Log connection parameters for debugging - logger.info("=" * 80) - logger.info("CONNECTION PARAMETERS:") - logger.info(f" speech_model: {connection_params.speech_model}") - logger.info(f" min_turn_silence: {connection_params.min_turn_silence}") - logger.info(f" max_turn_silence: {connection_params.max_turn_silence}") - logger.info(f" sample_rate: {connection_params.sample_rate}") - logger.info(f" encoding: {connection_params.encoding}") - logger.info(f" prompt: {connection_params.prompt}") - logger.info(f" keyterms_prompt: {connection_params.keyterms_prompt}") - logger.info(f" speaker_labels: {connection_params.speaker_labels}") - logger.info(f" format_turns: {connection_params.format_turns}") - logger.info( - f" end_of_turn_confidence_threshold: {connection_params.end_of_turn_confidence_threshold}" - ) - logger.info(f" language_detection: {connection_params.language_detection}") - logger.info("=" * 80) - - # AssemblyAI Speech-to-Text Service - stt = AssemblyAISTTService( - api_key=os.getenv("ASSEMBLYAI_API_KEY"), - connection_params=connection_params, - # Turn Detection Mode - # True = Pipecat mode (VAD + Smart Turn controls turns) - # False = STT mode (u3-rt-pro model controls turns) - vad_force_turn_endpoint=True, - # Speaker Formatting (only used if speaker_labels=True) - # None = Just log speaker IDs, don't modify transcript - speaker_format=None, - # speaker_format="{text}", - # speaker_format="{speaker}: {text}", - # speaker_format="[{speaker}] {text}", - # Additional available parameters (uncomment to use): - # should_interrupt=True, # Only for STT mode - ) - - # ======================================================================== - - # Text-to-Speech - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="a0e99841-438c-4a64-b679-ae501e7d6091", # Conversational English - ) - - # LLM - llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), - model="gpt-4", - ) - - # Conversation context - messages = [ - { - "role": "system", - "content": ( - "You are a helpful voice assistant testing the AssemblyAI u3-rt-pro model. " - "Keep responses very brief (1-2 sentences). " - "Start by introducing yourself briefly and asking the user to speak." - ), - }, - ] - - context = LLMContext(messages) - - # Configure aggregator based on mode - # In STT mode, don't use VAD (model handles turn detection) - # In Pipecat mode, use VAD + Smart Turn - vad_force_turn_endpoint = True # Must match the value in stt configuration above - user_params = None - if vad_force_turn_endpoint: - user_params = LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()) - - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=user_params, - ) - - # Pipeline - pipeline = Pipeline( - [ - transport.input(), - stt, - user_aggregator, - llm, - tts, - transport.output(), - assistant_aggregator, - ] - ) - - # Task - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - ) - - # Start the conversation - await task.queue_frames([LLMRunFrame()]) - - # Run - runner = PipelineRunner() - await runner.run(task) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/test_assemblyai_interactive.py b/test_assemblyai_interactive.py deleted file mode 100755 index c5ec0b429..000000000 --- a/test_assemblyai_interactive.py +++ /dev/null @@ -1,749 +0,0 @@ -#!/usr/bin/env python3 -"""Interactive AssemblyAI u3-rt-pro Comprehensive Test Suite - -Tests all features with detailed scenarios: -- Basic configuration variations -- Prompting and keyterms with difficult names -- Diarization -- Dynamic parameter updates (single and multiple) -- Mode comparisons -- STT mode timing experiments (testing silence parameters) -- Edge cases - -Usage: - python test_assemblyai_interactive.py -""" - -import asyncio -import os -import sys -from typing import Optional - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMRunFrame, STTUpdateSettingsFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) -from pipecat.services.assemblyai.models import AssemblyAIConnectionParams -from pipecat.services.assemblyai.stt import AssemblyAISTTService, AssemblyAISTTSettings -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.local.audio import LocalAudioTransport, LocalAudioTransportParams - -load_dotenv(override=True) - -logger.remove(0) -logger.add(sys.stderr, level="INFO") - - -async def run_bot( - connection_params: AssemblyAIConnectionParams, - test_name: str, - vad_force_turn_endpoint: bool = True, - speaker_format: Optional[str] = None, - test_dynamic_updates: Optional[callable] = None, -): - """Run the voice bot with specified configuration.""" - logger.info("=" * 80) - logger.info(f"TEST: {test_name}") - logger.info("=" * 80) - logger.info("Starting bot... Speak into your microphone after you hear the greeting!") - logger.info("=" * 80) - - # Create local audio transport - transport = LocalAudioTransport( - LocalAudioTransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - ) - ) - - # AssemblyAI Speech-to-Text - stt = AssemblyAISTTService( - api_key=os.getenv("ASSEMBLYAI_API_KEY"), - connection_params=connection_params, - vad_force_turn_endpoint=vad_force_turn_endpoint, - speaker_format=speaker_format, - ) - - # Text-to-Speech - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="a0e99841-438c-4a64-b679-ae501e7d6091", - ) - - # LLM - llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), - model="gpt-4", - ) - - # Conversation context - messages = [ - { - "role": "system", - "content": ( - "You are a helpful voice assistant testing the AssemblyAI u3-rt-pro model. " - "Keep responses very brief (1-2 sentences). " - "Start by introducing yourself briefly and asking the user to speak." - ), - }, - ] - - context = LLMContext(messages) - - # Configure aggregator based on mode - user_params = None - if vad_force_turn_endpoint: - user_params = LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()) - - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=user_params, - ) - - # Pipeline - pipeline = Pipeline( - [ - transport.input(), - stt, - user_aggregator, - llm, - tts, - transport.output(), - assistant_aggregator, - ] - ) - - # Task - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - ) - - # Handle dynamic updates if provided - if test_dynamic_updates: - asyncio.create_task(test_dynamic_updates(task)) - - # Start the conversation - await task.queue_frames([LLMRunFrame()]) - - # Run - runner = PipelineRunner() - await runner.run(task) - - -# ============================================================================ -# Test Configurations -# ============================================================================ - -# === BASIC CONFIGURATION (1-3) === - - -async def test_01_basic_100ms(): - """Test 1: Basic default configuration (100ms).""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - min_turn_silence=100, - ) - await run_bot(connection_params, "Basic Default Configuration (100ms)") - - -async def test_02_custom_200ms(): - """Test 2: Custom min_end_of_turn_silence (200ms).""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - min_turn_silence=200, - ) - await run_bot(connection_params, "Custom Turn Silence (200ms)") - - -async def test_03_custom_500ms(): - """Test 3: Longer silence threshold (500ms).""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - min_turn_silence=500, - ) - await run_bot(connection_params, "Longer Turn Silence (500ms)") - - -# === PROMPTING & WARNINGS (4-7) === - - -async def test_04_max_warning(): - """Test 4: max_turn_silence warning (should be overridden).""" - logger.warning("⚠️ EXPECT WARNING: max_turn_silence will be overridden") - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - max_turn_silence=500, - ) - await run_bot(connection_params, "max_turn_silence Override Warning") - - -async def test_05_prompt_warning(): - """Test 5: Custom prompt warning.""" - logger.warning("⚠️ EXPECT WARNING: Custom prompts should be tested carefully") - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - prompt="Transcribe speech accurately with proper punctuation.", - ) - await run_bot(connection_params, "Custom Prompt Warning Test") - - -async def test_06_prompt_keyterms_conflict(): - """Test 6: Prompt + keyterms conflict (should error).""" - logger.error("❌ EXPECT ERROR: Cannot use both prompt and keyterms_prompt") - try: - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - prompt="Custom prompt", - keyterms_prompt=["test"], - ) - await run_bot(connection_params, "Prompt + Keyterms Conflict (ERROR)") - except ValueError as e: - logger.error(f"✅ EXPECTED ERROR: {e}") - input("\nPress Enter to continue...") - return - - -async def test_07_keyterms_difficult(): - """Test 7: Keyterms with difficult/unusual names.""" - # Use names that STT wouldn't normally get right - keyterms = ["Xiomara", "Saoirse", "Krzystof", "Nguyen", "Pipecat", "AssemblyAI"] - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - keyterms_prompt=keyterms, - ) - logger.info("🎯 Boosted terms: Xiomara, Saoirse, Krzystof, Nguyen, Pipecat, AssemblyAI") - logger.info(" Try saying these difficult names to test boosting!") - await run_bot(connection_params, "Keyterms with Difficult Names") - - -# === DIARIZATION (8-9) === - - -async def test_08_diarization_basic(): - """Test 8: Basic diarization (speaker IDs logged).""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - speaker_labels=True, - ) - logger.info("🎤 Diarization enabled - speaker IDs will be logged") - logger.info(" Try having multiple people speak!") - await run_bot(connection_params, "Diarization - Basic") - - -async def test_09_diarization_xml(): - """Test 9: Diarization with XML formatting.""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - speaker_labels=True, - ) - logger.info("🎤 Diarization with XML tags") - logger.info(" Transcripts will include text") - await run_bot( - connection_params, - "Diarization - XML Formatting", - speaker_format="{text}", - ) - - -# === DYNAMIC UPDATES - SINGLE PARAMETER (10-13) === - - -async def test_10_dynamic_keyterms(): - """Test 10: Dynamic keyterms update with difficult names.""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - ) - - async def dynamic_update(task): - logger.info("\n" + "=" * 80) - logger.info("PHASE 1: No keyterms boosting") - logger.info(" Try saying: Xiomara, Saoirse, Krzystof") - logger.info(" (May not transcribe correctly)") - logger.info("=" * 80) - await asyncio.sleep(15) - - logger.info("\n" + "=" * 80) - logger.info("🔄 UPDATING: Adding keyterms boost") - logger.info("=" * 80) - await task.queue_frame( - STTUpdateSettingsFrame( - delta=AssemblyAISTTSettings( - connection_params=AssemblyAIConnectionParams( - keyterms_prompt=["Xiomara", "Saoirse", "Krzystof", "Nguyen"] - ) - ) - ) - ) - logger.info("\n" + "=" * 80) - logger.info("PHASE 2: Keyterms NOW boosted") - logger.info(" Say the same names again: Xiomara, Saoirse, Krzystof") - logger.info(" (Should transcribe better now!)") - logger.info("=" * 80) - - logger.info("🔄 This test has 2 phases:") - logger.info(" Phase 1 (15s): No boosting - names may be wrong") - logger.info(" Phase 2: Keyterms added - names should improve") - await run_bot( - connection_params, - "Dynamic Keyterms Update (Before/After)", - test_dynamic_updates=dynamic_update, - ) - - -async def test_11_dynamic_silence(): - """Test 11: Dynamic silence parameter update (dramatic change).""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - min_turn_silence=100, - ) - - async def dynamic_update(task): - logger.info("\n" + "=" * 80) - logger.info("PHASE 1: Quick responses (100ms silence threshold)") - logger.info(" Speak normally - bot responds quickly") - logger.info("=" * 80) - await asyncio.sleep(10) - - logger.info("\n" + "=" * 80) - logger.info("🔄 UPDATING: Changing silence from 100ms → 3000ms (3 seconds!)") - logger.info("=" * 80) - await task.queue_frame( - STTUpdateSettingsFrame( - delta=AssemblyAISTTSettings( - connection_params=AssemblyAIConnectionParams(min_turn_silence=3000) - ) - ) - ) - logger.info("\n" + "=" * 80) - logger.info("PHASE 2: Patient responses (3 second silence threshold)") - logger.info(" Bot will wait 3 full seconds before responding") - logger.info(" Try pausing mid-sentence - bot should NOT interrupt") - logger.info("=" * 80) - - logger.info("🔄 Dramatic change: 100ms → 3000ms after 10 seconds") - await run_bot( - connection_params, - "Dynamic Silence Update (100ms → 3s)", - test_dynamic_updates=dynamic_update, - ) - - -async def test_12_dynamic_prompt(): - """Test 12: Dynamic prompt update with keyterms in prompt.""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - ) - - async def dynamic_update(task): - logger.info("\n" + "=" * 80) - logger.info("PHASE 1: Default prompt (no keyterms)") - logger.info(" Try saying: Xiomara, Saoirse, Krzystof") - logger.info(" (May not transcribe correctly)") - logger.info("=" * 80) - await asyncio.sleep(15) - - logger.info("\n" + "=" * 80) - logger.info("🔄 UPDATING: Adding custom prompt with keyterms") - logger.info("=" * 80) - custom_prompt = """Transcribe verbatim. Rules: -1) Always include punctuation in output. -2) Use period/question mark ONLY for complete sentences. -3) Use comma for mid-sentence pauses. -4) Use no punctuation for incomplete trailing speech. -5) Filler words (um, uh, so, like) indicate speaker will continue. - -Pay special attention to these names and transcribe them exactly: Xiomara, Saoirse, Krzystof, Nguyen.""" - await task.queue_frame( - STTUpdateSettingsFrame( - delta=AssemblyAISTTSettings( - connection_params=AssemblyAIConnectionParams(prompt=custom_prompt) - ) - ) - ) - logger.info("\n" + "=" * 80) - logger.info("PHASE 2: Prompt with keyterms NOW active") - logger.info(" Say the same names again: Xiomara, Saoirse, Krzystof") - logger.info(" (Should transcribe better now!)") - logger.info("=" * 80) - - logger.info("🔄 This test has 2 phases:") - logger.info(" Phase 1 (15s): Default prompt - names may be wrong") - logger.info(" Phase 2: Custom prompt with keyterms - names should improve") - await run_bot( - connection_params, - "Dynamic Prompt Update (with keyterms)", - test_dynamic_updates=dynamic_update, - ) - - -async def test_13_dynamic_clear_keyterms(): - """Test 13: Clear keyterms dynamically.""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - keyterms_prompt=["Pipecat", "AssemblyAI"], - ) - - async def dynamic_update(task): - await asyncio.sleep(10) - logger.info("🔄 UPDATING: Clearing keyterms (empty array)") - await task.queue_frame( - STTUpdateSettingsFrame( - delta=AssemblyAISTTSettings( - connection_params=AssemblyAIConnectionParams(keyterms_prompt=[]) - ) - ) - ) - - logger.info("🎯 Initial: Pipecat, AssemblyAI boosted") - logger.info("🔄 After 10s: Keyterms will be cleared") - await run_bot( - connection_params, - "Dynamic Clear Keyterms", - test_dynamic_updates=dynamic_update, - ) - - -# === DYNAMIC UPDATES - MULTIPLE PARAMETERS (14-15) === - - -async def test_14_multi_param_update(): - """Test 14: Update multiple parameters at once.""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - min_turn_silence=100, - ) - - async def dynamic_update(task): - await asyncio.sleep(10) - logger.info("🔄 UPDATING MULTIPLE: keyterms + silence") - await task.queue_frame( - STTUpdateSettingsFrame( - delta=AssemblyAISTTSettings( - connection_params=AssemblyAIConnectionParams( - keyterms_prompt=["Xiomara", "Pipecat"], - min_turn_silence=250, - ) - ) - ) - ) - - logger.info("🔄 After 10s: Will update BOTH keyterms AND silence threshold") - await run_bot( - connection_params, - "Multiple Parameter Update", - test_dynamic_updates=dynamic_update, - ) - - -async def test_15_complex_sequence(): - """Test 15: Complex multi-stage update sequence.""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - ) - - async def dynamic_update(task): - logger.info("Stage 1: Initial (10s)") - await asyncio.sleep(10) - - logger.info("🔄 Stage 2: Add keyterms") - await task.queue_frame( - STTUpdateSettingsFrame( - delta=AssemblyAISTTSettings( - connection_params=AssemblyAIConnectionParams(keyterms_prompt=["Pipecat"]) - ) - ) - ) - await asyncio.sleep(10) - - logger.info("🔄 Stage 3: Change silence") - await task.queue_frame( - STTUpdateSettingsFrame( - delta=AssemblyAISTTSettings( - connection_params=AssemblyAIConnectionParams(min_turn_silence=200) - ) - ) - ) - await asyncio.sleep(10) - - logger.info("🔄 Stage 4: Update both") - await task.queue_frame( - STTUpdateSettingsFrame( - delta=AssemblyAISTTSettings( - connection_params=AssemblyAIConnectionParams( - keyterms_prompt=["AssemblyAI", "OpenAI"], - min_turn_silence=150, - ) - ) - ) - ) - - logger.info("🔄 Multi-stage: 4 configuration changes over 30 seconds") - await run_bot( - connection_params, - "Complex Update Sequence (4 stages)", - test_dynamic_updates=dynamic_update, - ) - - -# === MODE COMPARISON (16-17) === - - -async def test_16_pipecat_mode(): - """Test 16: Pipecat mode (VAD + Smart Turn controls turns).""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - min_turn_silence=100, - ) - logger.info("🎯 Pipecat Mode: VAD + Smart Turn control turn detection") - logger.info(" Your min_end_of_turn_silence is sent but ForceEndpoint overrides it") - await run_bot( - connection_params, - "Pipecat Mode (VAD + Smart Turn)", - vad_force_turn_endpoint=True, - ) - - -async def test_17_stt_mode(): - """Test 17: STT mode (model controls turns).""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - min_turn_silence=100, - ) - logger.info("🎯 STT Mode: u3-rt-pro model controls turn detection") - logger.info(" No ForceEndpoint - parameters are respected") - await run_bot( - connection_params, - "STT Mode (Model Turn Detection)", - vad_force_turn_endpoint=False, - ) - - -# === STT MODE TIMING EXPERIMENTS (18-20) === - - -async def test_18_stt_long_max_short_min(): - """Test 18: STT mode - Long max_turn_silence + Short min (5000ms + 100ms).""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - min_turn_silence=100, # Short - quick confident turns - max_turn_silence=5000, # Long - allows pauses up to 5 seconds - ) - logger.info("🎯 STT Mode: Testing max/min parameter interaction") - logger.info(" min_turn_silence: 100ms (quick when confident)") - logger.info(" max_turn_silence: 5000ms (allows up to 5 second pauses)") - logger.info(" Try: Quick sentences (should respond fast) + Long pauses mid-thought") - await run_bot( - connection_params, - "STT: Long Max (5s) + Short Min (100ms)", - vad_force_turn_endpoint=False, - ) - - -async def test_19_stt_long_min(): - """Test 19: STT mode - Long min_turn_silence (3000ms).""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - min_turn_silence=3000, # 3 seconds - max_turn_silence=5000, # 5 seconds - ) - logger.info("🎯 STT Mode: Testing long minimum silence requirement") - logger.info(" min_turn_silence: 3000ms") - logger.info(" max_turn_silence: 5000ms") - logger.info(" Bot will wait 3 full seconds of silence before responding!") - logger.info(" Try: Speaking with short pauses - bot should NOT interrupt") - await run_bot( - connection_params, - "STT: Long Min (3s)", - vad_force_turn_endpoint=False, - ) - - -async def test_20_stt_both_short(): - """Test 20: STT mode - Both short (max=300ms, min=100ms).""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - min_turn_silence=100, # 100ms - max_turn_silence=300, # 300ms - ) - logger.info("🎯 STT Mode: Testing aggressive/quick response timing") - logger.info(" min_turn_silence: 100ms") - logger.info(" max_turn_silence: 300ms") - logger.info(" Bot will respond VERY quickly to any pause!") - logger.info(" Try: Speaking with natural pauses - expect quick responses") - await run_bot( - connection_params, - "STT: Both Short (300ms/100ms)", - vad_force_turn_endpoint=False, - ) - - -# === EDGE CASES (21-23) === - - -async def test_21_very_long_silence(): - """Test 21: Very long silence threshold (STT mode only).""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - min_turn_silence=10000, # 10 seconds - ) - logger.warning("⚠️ STT Mode with 10 second silence threshold") - logger.info(" Bot will wait 10 seconds of silence before responding!") - await run_bot( - connection_params, - "Very Long Silence (10s) - STT Mode", - vad_force_turn_endpoint=False, - ) - - -async def test_22_very_short_silence(): - """Test 22: Very short silence threshold (50ms).""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - min_turn_silence=50, - ) - logger.info("⚡ Very short silence threshold (50ms)") - logger.info(" Bot will respond very quickly!") - await run_bot(connection_params, "Very Short Silence (50ms)") - - -async def test_23_keyterms_plus_diarization(): - """Test 23: Keyterms + Diarization combined.""" - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - keyterms_prompt=["Xiomara", "Saoirse", "Pipecat"], - speaker_labels=True, - ) - logger.info("🎯 Keyterms + 🎤 Diarization both enabled") - logger.info(" Try multiple speakers saying difficult names!") - await run_bot( - connection_params, - "Keyterms + Diarization Combined", - speaker_format="[{speaker}] {text}", - ) - - -# ============================================================================ -# Interactive Menu -# ============================================================================ - - -def show_menu(): - """Display the comprehensive test menu.""" - print("\n" + "=" * 80) - print("AssemblyAI u3-rt-pro Comprehensive Test Suite") - print("=" * 80) - print("\n📋 BASIC CONFIGURATION (1-3)") - print(" 1. Basic Default (100ms)") - print(" 2. Custom Silence (200ms)") - print(" 3. Longer Silence (500ms)") - - print("\n⚠️ PROMPTING & WARNINGS (4-7)") - print(" 4. max_turn_silence Warning") - print(" 5. Custom Prompt Warning") - print(" 6. Prompt + Keyterms Conflict (ERROR)") - print(" 7. Keyterms with Difficult Names") - - print("\n🎤 DIARIZATION (8-9)") - print(" 8. Diarization - Basic") - print(" 9. Diarization - XML Formatting") - - print("\n🔄 DYNAMIC UPDATES - SINGLE (10-13)") - print(" 10. Dynamic Keyterms (Before/After with difficult names)") - print(" 11. Dynamic Silence (100ms → 3s DRAMATIC)") - print(" 12. Dynamic Prompt with Keyterms (Before/After)") - print(" 13. Dynamic Clear Keyterms") - - print("\n🔄 DYNAMIC UPDATES - MULTIPLE (14-15)") - print(" 14. Multiple Parameters at Once") - print(" 15. Complex Update Sequence (4 stages)") - - print("\n⚖️ MODE COMPARISON (16-17)") - print(" 16. Pipecat Mode (VAD + Smart Turn)") - print(" 17. STT Mode (Model Turn Detection)") - - print("\n⏱️ STT MODE TIMING EXPERIMENTS (18-20)") - print(" 18. STT: Long Max (5s) + Short Min (100ms)") - print(" 19. STT: Long Min (3s)") - print(" 20. STT: Both Short (300ms/100ms)") - - print("\n🎯 EDGE CASES (21-23)") - print(" 21. Very Long Silence (10s - STT Mode)") - print(" 22. Very Short Silence (50ms)") - print(" 23. Keyterms + Diarization Combined") - - print("\n 0. Exit") - print("\n" + "=" * 80) - - -async def main(): - """Main interactive menu.""" - tests = { - "1": test_01_basic_100ms, - "2": test_02_custom_200ms, - "3": test_03_custom_500ms, - "4": test_04_max_warning, - "5": test_05_prompt_warning, - "6": test_06_prompt_keyterms_conflict, - "7": test_07_keyterms_difficult, - "8": test_08_diarization_basic, - "9": test_09_diarization_xml, - "10": test_10_dynamic_keyterms, - "11": test_11_dynamic_silence, - "12": test_12_dynamic_prompt, - "13": test_13_dynamic_clear_keyterms, - "14": test_14_multi_param_update, - "15": test_15_complex_sequence, - "16": test_16_pipecat_mode, - "17": test_17_stt_mode, - "18": test_18_stt_long_max_short_min, - "19": test_19_stt_long_min, - "20": test_20_stt_both_short, - "21": test_21_very_long_silence, - "22": test_22_very_short_silence, - "23": test_23_keyterms_plus_diarization, - } - - while True: - show_menu() - choice = input("Enter test number (or 0 to exit): ").strip() - - if choice == "0": - print("\n👋 Goodbye!") - break - - if choice in tests: - try: - await tests[choice]() - except KeyboardInterrupt: - print("\n\n⚠️ Test interrupted by user") - except Exception as e: - logger.error(f"Test failed with error: {e}") - import traceback - - traceback.print_exc() - - input("\n\nPress Enter to return to menu...") - else: - print(f"\n❌ Invalid choice: {choice}") - input("Press Enter to continue...") - - -if __name__ == "__main__": - try: - asyncio.run(main()) - except KeyboardInterrupt: - print("\n\n👋 Goodbye!") diff --git a/test_assemblyai_u3pro.py b/test_assemblyai_u3pro.py deleted file mode 100644 index 236ab9b50..000000000 --- a/test_assemblyai_u3pro.py +++ /dev/null @@ -1,582 +0,0 @@ -#!/usr/bin/env python3 -"""AssemblyAI u3-rt-pro Comprehensive Test Script - -Tests all features: -- Basic configuration -- Prompting and keyterms -- Diarization -- Dynamic updates -- Turn detection modes - -Usage: - python test_assemblyai_u3pro.py --test - python test_assemblyai_u3pro.py --interactive -""" - -import argparse -import asyncio -import os -import sys -from typing import List - -from dotenv import load_dotenv -from loguru import logger - -# Add src to path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src")) - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import ( - EndFrame, - Frame, - LLMRunFrame, - STTUpdateSettingsFrame, - TranscriptionFrame, -) -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import ( - LLMContextAggregatorPair, - LLMUserAggregatorParams, -) -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.services.assemblyai.models import AssemblyAIConnectionParams -from pipecat.services.assemblyai.stt import AssemblyAISTTService -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.local.audio import LocalAudioTransport, LocalAudioTransportParams - -load_dotenv() - - -# Test configuration -class TestConfig: - """Centralized test configuration.""" - - ASSEMBLYAI_API_KEY = os.getenv("ASSEMBLYAI_API_KEY") - OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") - CARTESIA_API_KEY = os.getenv("CARTESIA_API_KEY") - - @classmethod - def validate(cls): - """Validate all required API keys are set.""" - missing = [] - if not cls.ASSEMBLYAI_API_KEY: - missing.append("ASSEMBLYAI_API_KEY") - if not cls.OPENAI_API_KEY: - missing.append("OPENAI_API_KEY") - if not cls.CARTESIA_API_KEY: - missing.append("CARTESIA_API_KEY") - - if missing: - logger.error(f"Missing required environment variables: {', '.join(missing)}") - return False - return True - - -class TranscriptionLogger(FrameProcessor): - """Log transcriptions for test verification.""" - - async def process_frame(self, frame: Frame, direction: FrameDirection): - if isinstance(frame, TranscriptionFrame): - logger.info(f"📝 TRANSCRIPTION: {frame.text}") - logger.info(f" Speaker: {frame.user_id}") - logger.info(f" Finalized: {frame.finalized}") - if hasattr(frame, "result") and frame.result: - if hasattr(frame.result, "speaker"): - logger.info(f" Diarization: {frame.result.speaker}") - - await self.push_frame(frame, direction) - - -async def create_basic_voice_agent( - connection_params: AssemblyAIConnectionParams, - vad_force_turn_endpoint: bool = True, - speaker_format: str = None, -) -> tuple[PipelineTask, LocalAudioTransport]: - """Create a basic voice agent for testing. - - Args: - connection_params: AssemblyAI connection parameters - vad_force_turn_endpoint: Turn detection mode - speaker_format: Optional speaker formatting string - - Returns: - Tuple of (PipelineTask, LocalAudioTransport) - """ - # Create local audio transport (uses your microphone and speakers) - transport = LocalAudioTransport( - params=LocalAudioTransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - ) - ) - - # Create STT - stt = AssemblyAISTTService( - api_key=TestConfig.ASSEMBLYAI_API_KEY, - connection_params=connection_params, - vad_force_turn_endpoint=vad_force_turn_endpoint, - speaker_format=speaker_format, - ) - - # Create TTS - tts = CartesiaTTSService( - api_key=TestConfig.CARTESIA_API_KEY, - voice_id="a0e99841-438c-4a64-b679-ae501e7d6091", # Conversational English - ) - - # Create LLM context and service - messages = [ - { - "role": "system", - "content": ( - "You are a helpful voice assistant. Keep responses brief and natural. " - "If you see speaker tags like text, acknowledge " - "that you understand multiple speakers are present." - ), - } - ] - - context = LLMContext(messages) - llm = OpenAILLMService(api_key=TestConfig.OPENAI_API_KEY, model="gpt-4") - - # Create aggregators with VAD - user_aggregator, assistant_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams( - vad_analyzer=SileroVADAnalyzer(), - ), - ) - - # Create transcription logger - transcription_logger = TranscriptionLogger() - - # Create pipeline - pipeline = Pipeline( - [ - transport.input(), - stt, - transcription_logger, - user_aggregator, - llm, - tts, - transport.output(), - assistant_aggregator, - ] - ) - - # Create task - task = PipelineTask(pipeline) - - return task, transport - - -# ============================================================================ -# Test Functions -# ============================================================================ - - -async def test_basic_config(): - """Test 1: Basic default configuration.""" - logger.info("=" * 80) - logger.info("TEST 1: Basic Default Configuration") - logger.info("=" * 80) - - connection_params = AssemblyAIConnectionParams(speech_model="u3-rt-pro") - - task, transport = await create_basic_voice_agent(connection_params) - - logger.info("✅ Service created successfully with default params") - logger.info("Expected: min=max=100ms, u3-rt-pro model") - logger.info("Speak into your microphone to test transcription") - - # Trigger initial bot greeting - await task.queue_frames([LLMRunFrame()]) - - runner = PipelineRunner() - await runner.run(task) - - -async def test_custom_min_silence(): - """Test 2: Custom min_turn_silence.""" - logger.info("=" * 80) - logger.info("TEST 2: Custom min_turn_silence") - logger.info("=" * 80) - - connection_params = AssemblyAIConnectionParams(speech_model="u3-rt-pro", min_turn_silence=200) - - task, transport = await create_basic_voice_agent(connection_params) - - logger.info("✅ Service created with min=200ms") - logger.info("Expected: Both min and max set to 200ms") - logger.info("Speak short phrases and observe turn detection timing") - - runner = PipelineRunner() - await runner.run(task) - - -async def test_max_silence_warning(): - """Test 3: Setting max_turn_silence should trigger warning.""" - logger.info("=" * 80) - logger.info("TEST 3: max_turn_silence Warning") - logger.info("=" * 80) - - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - min_turn_silence=100, - max_turn_silence=500, # Should trigger warning - ) - - task, transport = await create_basic_voice_agent(connection_params) - - logger.info("⚠️ Check logs above for warning about max_turn_silence being overridden") - logger.info("Expected: Warning logged, max set to 100ms (same as min)") - - runner = PipelineRunner() - await runner.run(task) - - -async def test_custom_prompt_warning(): - """Test 5: Custom prompt should trigger warning.""" - logger.info("=" * 80) - logger.info("TEST 5: Custom Prompt Warning") - logger.info("=" * 80) - - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - prompt="Transcribe verbatim. Always include punctuation.", - ) - - task, transport = await create_basic_voice_agent(connection_params) - - logger.info("⚠️ Check logs above for warning about testing without prompt first") - logger.info("Expected: Warning logged, service continues with custom prompt") - - runner = PipelineRunner() - await runner.run(task) - - -async def test_prompt_keyterms_conflict(): - """Test 6: Prompt + keyterms_prompt should raise error.""" - logger.info("=" * 80) - logger.info("TEST 6: Prompt + Keyterms Conflict (Error)") - logger.info("=" * 80) - - try: - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - prompt="Custom prompt", - keyterms_prompt=["test", "words"], - ) - - task, transport = await create_basic_voice_agent(connection_params) - logger.error("❌ TEST FAILED: Should have raised ValueError") - except ValueError as e: - logger.info(f"✅ TEST PASSED: ValueError raised as expected") - logger.info(f" Error message: {e}") - - -async def test_keyterms_basic(): - """Test 7: Basic keyterms at initialization.""" - logger.info("=" * 80) - logger.info("TEST 7: Basic Keyterms Prompting") - logger.info("=" * 80) - - connection_params = AssemblyAIConnectionParams( - speech_model="u3-rt-pro", - keyterms_prompt=["Pipecat", "AssemblyAI", "Universal-3", "streaming"], - ) - - task, transport = await create_basic_voice_agent(connection_params) - - logger.info("✅ Service created with keyterms: Pipecat, AssemblyAI, Universal-3, streaming") - logger.info("Expected: Boosted recognition for these terms") - logger.info("Try saying: 'I'm testing Pipecat with AssemblyAI Universal-3 for streaming'") - - runner = PipelineRunner() - await runner.run(task) - - -async def test_diarization_no_format(): - """Test 10: Diarization enabled without formatting.""" - logger.info("=" * 80) - logger.info("TEST 10: Diarization Enabled (No Formatting)") - logger.info("=" * 80) - - connection_params = AssemblyAIConnectionParams(speech_model="u3-rt-pro", speaker_labels=True) - - task, transport = await create_basic_voice_agent(connection_params) - - logger.info("✅ Service created with speaker_labels=True") - logger.info("Expected: Speaker IDs in user_id field, plain text in transcript") - logger.info("Have multiple people speak to see different speaker labels") - - runner = PipelineRunner() - await runner.run(task) - - -async def test_diarization_xml_format(): - """Test 11: Diarization with XML formatting.""" - logger.info("=" * 80) - logger.info("TEST 11: Diarization with XML Formatting") - logger.info("=" * 80) - - connection_params = AssemblyAIConnectionParams(speech_model="u3-rt-pro", speaker_labels=True) - - task, transport = await create_basic_voice_agent( - connection_params, speaker_format="<{speaker}>{text}" - ) - - logger.info("✅ Service created with XML speaker formatting") - logger.info("Expected: Text like 'Hello'") - logger.info("Have multiple people speak to see formatted speaker tags") - - runner = PipelineRunner() - await runner.run(task) - - -async def test_dynamic_keyterms(): - """Test 13: Dynamic keyterms updates.""" - logger.info("=" * 80) - logger.info("TEST 13: Dynamic Keyterms Updates") - logger.info("=" * 80) - - connection_params = AssemblyAIConnectionParams(speech_model="u3-rt-pro") - - task, transport = await create_basic_voice_agent(connection_params) - - async def update_keyterms_stages(): - """Simulate multi-stage conversation with keyterms updates.""" - await asyncio.sleep(5) # Wait for connection - - # Stage 1: Greeting - logger.info("🔄 STAGE 1: Greeting (general terms)") - update1 = STTUpdateSettingsFrame( - settings={"keyterms_prompt": ["hello", "hi", "good morning", "welcome"]} - ) - await task.queue_frames([update1]) - - await asyncio.sleep(10) - - # Stage 2: Name collection - logger.info("🔄 STAGE 2: Name Collection") - update2 = STTUpdateSettingsFrame( - settings={ - "keyterms_prompt": [ - "first name", - "last name", - "John", - "Jane", - "Smith", - "Johnson", - ] - } - ) - await task.queue_frames([update2]) - - await asyncio.sleep(10) - - # Stage 3: Medical info - logger.info("🔄 STAGE 3: Medical Information") - update3 = STTUpdateSettingsFrame( - settings={ - "keyterms_prompt": [ - "cardiology", - "echocardiogram", - "blood pressure", - "Dr. Smith", - "metoprolol", - ] - } - ) - await task.queue_frames([update3]) - - await asyncio.sleep(10) - - # Stage 4: Clear keyterms - logger.info("🔄 STAGE 4: Clear Keyterms") - update4 = STTUpdateSettingsFrame(settings={"keyterms_prompt": []}) - await task.queue_frames([update4]) - - # Start update task - asyncio.create_task(update_keyterms_stages()) - - logger.info("✅ Service created, will update keyterms every 10 seconds") - logger.info("Expected: Different keyterms at each stage") - logger.info("Watch logs for 'STAGE X' messages and test relevant terms") - - runner = PipelineRunner() - await runner.run(task) - - -async def test_dynamic_silence_params(): - """Test 15: Dynamic silence parameter updates.""" - logger.info("=" * 80) - logger.info("TEST 15: Dynamic Silence Parameters") - logger.info("=" * 80) - - connection_params = AssemblyAIConnectionParams(speech_model="u3-rt-pro") - - task, transport = await create_basic_voice_agent(connection_params) - - async def update_silence_params(): - """Update silence parameters for different scenarios.""" - await asyncio.sleep(5) - - # Normal conversation - logger.info("🔄 PHASE 1: Normal conversation (default timing)") - await asyncio.sleep(10) - - # Reading credit card - logger.info("🔄 PHASE 2: Reading numbers (longer silence tolerance)") - update1 = STTUpdateSettingsFrame( - settings={ - "max_turn_silence": 5000, - "min_turn_silence": 300, - } - ) - await task.queue_frames([update1]) - - await asyncio.sleep(15) - - # Back to normal - logger.info("🔄 PHASE 3: Back to normal conversation") - update2 = STTUpdateSettingsFrame( - settings={ - "max_turn_silence": 1200, - "min_turn_silence": 100, - } - ) - await task.queue_frames([update2]) - - asyncio.create_task(update_silence_params()) - - logger.info("✅ Service will update silence parameters during conversation") - logger.info("Expected: Longer pauses tolerated in Phase 2") - logger.info("Try pausing between words to test") - - runner = PipelineRunner() - await runner.run(task) - - -async def test_multi_param_update(): - """Test 17: Update multiple parameters at once.""" - logger.info("=" * 80) - logger.info("TEST 17: Multiple Parameter Update") - logger.info("=" * 80) - - connection_params = AssemblyAIConnectionParams(speech_model="u3-rt-pro") - - task, transport = await create_basic_voice_agent(connection_params) - - async def multi_update(): - await asyncio.sleep(5) - - logger.info("🔄 Updating multiple parameters together") - update = STTUpdateSettingsFrame( - settings={ - "keyterms_prompt": ["account", "routing", "number"], - "max_turn_silence": 3000, - "min_turn_silence": 200, - } - ) - await task.queue_frames([update]) - - logger.info("✅ Check logs for single UpdateConfiguration message") - - asyncio.create_task(multi_update()) - - logger.info("Expected: All params updated in single WebSocket message") - - runner = PipelineRunner() - await runner.run(task) - - -# ============================================================================ -# Main Test Runner -# ============================================================================ - - -def main(): - """Main test runner.""" - parser = argparse.ArgumentParser(description="Test AssemblyAI u3-rt-pro integration") - parser.add_argument( - "--test", - type=str, - default="basic", - help="Test to run (basic, custom_min, max_warning, prompt_warning, " - "prompt_keyterms_conflict, keyterms, diarization, diarization_xml, " - "dynamic_keyterms, dynamic_silence, multi_param, all)", - ) - parser.add_argument("--interactive", action="store_true", help="Run in interactive mode") - - args = parser.parse_args() - - # Validate environment - if not TestConfig.validate(): - logger.error("Please set all required environment variables in .env") - sys.exit(1) - - # Test mapping - tests = { - "basic": test_basic_config, - "custom_min": test_custom_min_silence, - "max_warning": test_max_silence_warning, - "prompt_warning": test_custom_prompt_warning, - "prompt_keyterms_conflict": test_prompt_keyterms_conflict, - "keyterms": test_keyterms_basic, - "diarization": test_diarization_no_format, - "diarization_xml": test_diarization_xml_format, - "dynamic_keyterms": test_dynamic_keyterms, - "dynamic_silence": test_dynamic_silence_params, - "multi_param": test_multi_param_update, - } - - if args.interactive: - logger.info("Interactive mode - select test to run:") - for i, (name, _) in enumerate(tests.items(), 1): - logger.info(f"{i}. {name}") - logger.info(f"{len(tests) + 1}. Run all tests") - - choice = input("\nEnter test number: ") - try: - choice_num = int(choice) - if choice_num == len(tests) + 1: - args.test = "all" - else: - args.test = list(tests.keys())[choice_num - 1] - except (ValueError, IndexError): - logger.error("Invalid choice") - sys.exit(1) - - # Run test(s) - if args.test == "all": - logger.info("Running all tests sequentially...") - for test_name, test_func in tests.items(): - try: - asyncio.run(test_func()) - except KeyboardInterrupt: - logger.info(f"Test '{test_name}' interrupted") - break - except Exception as e: - logger.error(f"Test '{test_name}' failed: {e}") - else: - if args.test not in tests: - logger.error(f"Unknown test: {args.test}") - logger.info(f"Available tests: {', '.join(tests.keys())}") - sys.exit(1) - - try: - asyncio.run(tests[args.test]()) - except KeyboardInterrupt: - logger.info("Test interrupted") - except Exception as e: - logger.error(f"Test failed: {e}") - raise - - -if __name__ == "__main__": - main() From 7648b62e6e50639d4f6607da75fe69e1d4132252 Mon Sep 17 00:00:00 2001 From: zkleb-aai <146127913+zkleb-aai@users.noreply.github.com> Date: Mon, 2 Mar 2026 17:04:17 -0500 Subject: [PATCH 17/23] Update src/pipecat/services/assemblyai/stt.py Co-authored-by: Mark Backman --- src/pipecat/services/assemblyai/stt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index 9938afdee..8ef6b5a07 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -590,7 +590,7 @@ class AssemblyAISTTService(WebsocketSTTService): data = json.loads(message) # Log raw JSON for Turn messages to debug speaker_label if data.get("type") == "Turn": - logger.debug(f"{self} RAW JSON from AssemblyAI: {json.dumps(data, indent=2)}") + logger.trace(f"{self} RAW JSON from AssemblyAI: {json.dumps(data, indent=2)}") await self._handle_message(data) except json.JSONDecodeError: logger.warning(f"Received non-JSON message: {message}") From 6729f4366a68bb56f4b8a11f22b15b1c4ce4a333 Mon Sep 17 00:00:00 2001 From: zkleb-aai <146127913+zkleb-aai@users.noreply.github.com> Date: Mon, 2 Mar 2026 17:04:42 -0500 Subject: [PATCH 18/23] Update src/pipecat/services/assemblyai/stt.py Co-authored-by: Mark Backman --- src/pipecat/services/assemblyai/stt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index 8ef6b5a07..bbbdad3f8 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -717,7 +717,7 @@ class AssemblyAISTTService(WebsocketSTTService): finalize_confirmed = bool(message.turn_is_formatted) if finalize_confirmed: self.confirm_finalize() - logger.debug(f'{self} Final transcript: "{transcript_text}"') + logger.debug(f'{self} Transcript: "{transcript_text}"') await self.push_frame( TranscriptionFrame( transcript_text, From 5c2ca0ce64d41f096bcfb7569dbda6480da31c13 Mon Sep 17 00:00:00 2001 From: zkleb-aai <146127913+zkleb-aai@users.noreply.github.com> Date: Mon, 2 Mar 2026 17:04:54 -0500 Subject: [PATCH 19/23] Update changelog/3856.changed.md Co-authored-by: Mark Backman --- changelog/3856.changed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/3856.changed.md b/changelog/3856.changed.md index 72331c068..1e7f4c916 100644 --- a/changelog/3856.changed.md +++ b/changelog/3856.changed.md @@ -1 +1 @@ -- Rename AssemblyAI min_end_of_turn_silence_when_confident parameter to min_turn_silence (old name still supported with deprecation warning) +- Rename `AssemblyAISTTService` parameter `min_end_of_turn_silence_when_confident` parameter to `min_turn_silence` (old name still supported with deprecation warning) From b449515410eac0b577af86d82abf71225bb58460 Mon Sep 17 00:00:00 2001 From: zack Date: Mon, 2 Mar 2026 17:54:31 -0500 Subject: [PATCH 20/23] Address PR review feedback: remove debug logs, fix hasattr logic, add VADAnalyzer --- .../07o-interruptible-assemblyai-stt.py | 6 +- src/pipecat/services/assemblyai/stt.py | 86 ++++++++----------- 2 files changed, 40 insertions(+), 52 deletions(-) diff --git a/examples/foundational/07o-interruptible-assemblyai-stt.py b/examples/foundational/07o-interruptible-assemblyai-stt.py index 2765f8590..01d53914d 100644 --- a/examples/foundational/07o-interruptible-assemblyai-stt.py +++ b/examples/foundational/07o-interruptible-assemblyai-stt.py @@ -10,6 +10,7 @@ import os from dotenv import load_dotenv from loguru import logger +from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -122,7 +123,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): context = LLMContext(messages) user_aggregator, assistant_aggregator = LLMContextAggregatorPair( context, - user_params=LLMUserAggregatorParams(user_turn_strategies=ExternalUserTurnStrategies()), + user_params=LLMUserAggregatorParams( + user_turn_strategies=ExternalUserTurnStrategies(), + vad_analyzer=SileroVADAnalyzer(), + ), ) pipeline = Pipeline( diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index bbbdad3f8..2e14b1a66 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -212,7 +212,6 @@ class AssemblyAISTTService(WebsocketSTTService): self._chunk_size_bytes = 0 self._user_speaking = False - self._vad_speaking = False def _configure_pipecat_turn_mode( self, connection_params: AssemblyAIConnectionParams, is_u3_pro: bool @@ -320,48 +319,44 @@ class AssemblyAISTTService(WebsocketSTTService): old_conn_params = changed.get("connection_params") # Check each potentially changed parameter - if hasattr(conn_params, "keyterms_prompt"): - if ( - old_conn_params is None - or conn_params.keyterms_prompt != old_conn_params.keyterms_prompt - ): - if conn_params.keyterms_prompt is not None: - update_config["keyterms_prompt"] = conn_params.keyterms_prompt - logger.info(f"Updating keyterms_prompt to: {conn_params.keyterms_prompt}") + if ( + old_conn_params is None + or conn_params.keyterms_prompt != old_conn_params.keyterms_prompt + ): + if conn_params.keyterms_prompt is not None: + update_config["keyterms_prompt"] = conn_params.keyterms_prompt + logger.info(f"Updating keyterms_prompt to: {conn_params.keyterms_prompt}") - if hasattr(conn_params, "prompt"): - if old_conn_params is None or conn_params.prompt != old_conn_params.prompt: - if conn_params.prompt is not None: - if conn_params.speech_model != "u3-rt-pro": - logger.warning( - f"prompt parameter is only supported with u3-rt-pro model, " - f"current model is {conn_params.speech_model}" - ) - else: - update_config["prompt"] = conn_params.prompt - logger.info(f"Updating prompt") - - if hasattr(conn_params, "max_turn_silence"): - if ( - old_conn_params is None - or conn_params.max_turn_silence != old_conn_params.max_turn_silence - ): - if conn_params.max_turn_silence is not None: - update_config["max_turn_silence"] = conn_params.max_turn_silence - logger.info( - f"Updating max_turn_silence to: {conn_params.max_turn_silence}ms" + if old_conn_params is None or conn_params.prompt != old_conn_params.prompt: + if conn_params.prompt is not None: + if conn_params.speech_model != "u3-rt-pro": + logger.warning( + f"prompt parameter is only supported with u3-rt-pro model, " + f"current model is {conn_params.speech_model}" ) + else: + update_config["prompt"] = conn_params.prompt + logger.info(f"Updating prompt") - if hasattr(conn_params, "min_turn_silence"): - if ( - old_conn_params is None - or conn_params.min_turn_silence != old_conn_params.min_turn_silence - ): - if conn_params.min_turn_silence is not None: - update_config["min_turn_silence"] = conn_params.min_turn_silence - logger.info( - f"Updating min_turn_silence to: {conn_params.min_turn_silence}ms" - ) + if ( + old_conn_params is None + or conn_params.max_turn_silence != old_conn_params.max_turn_silence + ): + if conn_params.max_turn_silence is not None: + update_config["max_turn_silence"] = conn_params.max_turn_silence + logger.info( + f"Updating max_turn_silence to: {conn_params.max_turn_silence}ms" + ) + + if ( + old_conn_params is None + or conn_params.min_turn_silence != old_conn_params.min_turn_silence + ): + if conn_params.min_turn_silence is not None: + update_config["min_turn_silence"] = conn_params.min_turn_silence + logger.info( + f"Updating min_turn_silence to: {conn_params.min_turn_silence}ms" + ) # Send update if we have parameters to update if len(update_config) > 1: # More than just "type" @@ -639,20 +634,14 @@ class AssemblyAISTTService(WebsocketSTTService): Only applies to Mode 2 (STT turn detection). In Mode 1, VAD + smart turn analyzer handle interruptions via the aggregator. """ - logger.debug( - f"{self} SpeechStarted received (vad_force_turn_endpoint={self._vad_force_turn_endpoint})" - ) if self._vad_force_turn_endpoint: - logger.debug(f"{self} SpeechStarted ignored in Pipecat mode") return # Mode 1: handled by aggregator - logger.debug(f"{self} Processing SpeechStarted in STT mode") await self.start_processing_metrics() await self.broadcast_frame(UserStartedSpeakingFrame) if self._should_interrupt: await self.push_interruption_task_frame_and_wait() self._user_speaking = True - logger.debug(f"{self} _user_speaking set to True") async def _handle_termination(self, message: TerminationMessage): """Handle termination message.""" @@ -730,7 +719,6 @@ class AssemblyAISTTService(WebsocketSTTService): await self._trace_transcription(transcript_text, True, language) await self.stop_processing_metrics() else: - logger.debug(f'{self} Interim transcript: "{transcript_text}"') await self.push_frame( InterimTranscriptionFrame( transcript_text, @@ -744,10 +732,6 @@ class AssemblyAISTTService(WebsocketSTTService): # --- Mode 2: STT turn detection --- # SpeechStarted always arrives before transcripts with u3-rt-pro, # so UserStartedSpeakingFrame is guaranteed to be broadcast first. - logger.debug( - f"{self} Transcript received in STT mode (_user_speaking={self._user_speaking})" - ) - if is_final_turn: # STT mode: AssemblyAI controls finalization, just mark as finalized await self.push_frame( From 32773b42d694cddfd7cbdc1d86743781b47d8d4c Mon Sep 17 00:00:00 2001 From: zack Date: Mon, 2 Mar 2026 18:08:46 -0500 Subject: [PATCH 21/23] Improve terminology: rename file and replace 'STT mode' with 'AssemblyAI turn detection' - Rename 07o-interruptible-assemblyai-stt.py -> 07o-interruptible-assemblyai-turn-detection.py - Replace 'STT mode' with 'AssemblyAI turn detection mode' throughout codebase - Replace 'Mode 1'/'Mode 2' with descriptive 'Pipecat turn detection'/'AssemblyAI turn detection' - Update changelog to use 'built-in turn detection' terminology - Addresses PR feedback about confusing terminology --- changelog/3856.added.md | 2 +- ...nterruptible-assemblyai-turn-detection.py} | 14 ++++---- src/pipecat/services/assemblyai/stt.py | 33 ++++++++++--------- 3 files changed, 25 insertions(+), 24 deletions(-) rename examples/foundational/{07o-interruptible-assemblyai-stt.py => 07o-interruptible-assemblyai-turn-detection.py} (93%) diff --git a/changelog/3856.added.md b/changelog/3856.added.md index 95b656058..8074a5281 100644 --- a/changelog/3856.added.md +++ b/changelog/3856.added.md @@ -1 +1 @@ -- Add AssemblyAI u3-rt-pro model support with STT-controlled turn detection mode +- Add AssemblyAI u3-rt-pro model support with built-in turn detection mode diff --git a/examples/foundational/07o-interruptible-assemblyai-stt.py b/examples/foundational/07o-interruptible-assemblyai-turn-detection.py similarity index 93% rename from examples/foundational/07o-interruptible-assemblyai-stt.py rename to examples/foundational/07o-interruptible-assemblyai-turn-detection.py index 01d53914d..cf052ada4 100644 --- a/examples/foundational/07o-interruptible-assemblyai-stt.py +++ b/examples/foundational/07o-interruptible-assemblyai-turn-detection.py @@ -53,20 +53,20 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - """AssemblyAI u3-rt-pro STT Example with STT-Controlled Turn Detection + """AssemblyAI u3-rt-pro with Built-in Turn Detection This example demonstrates using AssemblyAI's u3-rt-pro Speech-to-Text model - with STT-controlled turn detection for more natural conversation flow. + with AssemblyAI's built-in turn detection for more natural conversation flow. Key features: - 1. STT-Controlled Turn Detection - - Set `vad_force_turn_endpoint=False` to enable STT mode + 1. AssemblyAI Turn Detection + - Set `vad_force_turn_endpoint=False` to use AssemblyAI's built-in turn detection - AssemblyAI's model determines when user starts/stops speaking - - Uses `ExternalUserTurnStrategies` instead of Pipecat's VAD + - Uses `ExternalUserTurnStrategies` to delegate turn control to AssemblyAI - More natural turn detection based on speech patterns and pauses - 2. Advanced Turn Detection Tuning (STT Mode) + 2. Advanced Turn Detection Tuning - `min_turn_silence`: Minimum silence (ms) when confident about end-of-turn. Lower values = faster responses. Default: 100ms - `max_turn_silence`: Maximum silence (ms) before forcing end-of-turn. @@ -93,7 +93,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): stt = AssemblyAISTTService( api_key=os.getenv("ASSEMBLYAI_API_KEY"), - vad_force_turn_endpoint=False, # Enable STT-controlled turn detection + vad_force_turn_endpoint=False, # Use AssemblyAI's built-in turn detection connection_params=AssemblyAIConnectionParams( speech_model="u3-rt-pro", # Optional: Tune turn detection timing (defaults shown below) diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index 2e14b1a66..cccd3d8b0 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -131,14 +131,15 @@ class AssemblyAISTTService(WebsocketSTTService): - max_turn_silence is ALWAYS set equal to min_turn_silence - VAD stop sends ForceEndpoint as ceiling - No UserStarted/StoppedSpeakingFrame emitted from STT - When False (STT mode, u3-rt-pro only): AssemblyAI's model controls turn endings. + When False (AssemblyAI turn detection mode, u3-rt-pro only): AssemblyAI's model + controls turn endings using built-in turn detection. - Uses AssemblyAI API defaults for all parameters (unless user explicitly sets them) - Respects all user-provided connection_params as-is - Emits UserStarted/StoppedSpeakingFrame from STT - No ForceEndpoint on VAD stop should_interrupt: Whether to interrupt the bot when the user starts speaking - in STT mode (vad_force_turn_endpoint=False). Only applies to STT mode. - Defaults to True. + in AssemblyAI turn detection mode (vad_force_turn_endpoint=False). Only applies + when using AssemblyAI's built-in turn detection. Defaults to True. speaker_format: Optional format string for speaker labels when diarization is enabled. Use {speaker} for speaker label and {text} for transcript text. Example: "<{speaker}>{text}" or "{speaker}: {text}" @@ -147,13 +148,13 @@ class AssemblyAISTTService(WebsocketSTTService): Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark **kwargs: Additional arguments passed to parent STTService class. """ - # STT turn detection (vad_force_turn_endpoint=False) requires the + # AssemblyAI turn detection mode (vad_force_turn_endpoint=False) requires the # SpeechStarted event for reliable barge-in. Only u3-rt-pro supports # this. Other models must use Pipecat turn detection. is_u3_pro = connection_params.speech_model == "u3-rt-pro" if not vad_force_turn_endpoint and not is_u3_pro: raise ValueError( - f"STT turn detection (vad_force_turn_endpoint=False) requires " + f"AssemblyAI turn detection mode (vad_force_turn_endpoint=False) requires " f"u3-rt-pro for SpeechStarted support. Either set " f"vad_force_turn_endpoint=True for {connection_params.speech_model}, " f"or use speech_model='u3-rt-pro'." @@ -256,7 +257,7 @@ class AssemblyAISTTService(WebsocketSTTService): f"OVERRIDDEN in Pipecat mode (vad_force_turn_endpoint=True). It will be set to " f"{min_silence}ms (matching min_turn_silence) and SENT to " f"AssemblyAI to avoid double turn detection. To use your max_turn_silence as-is, " - f"switch to STT mode (vad_force_turn_endpoint=False)." + f"switch to AssemblyAI turn detection mode (vad_force_turn_endpoint=False)." ) updates = { @@ -624,18 +625,18 @@ class AssemblyAISTTService(WebsocketSTTService): await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) async def _handle_speech_started(self, message: SpeechStartedMessage): - """Handle SpeechStarted event — fast barge-in for Mode 2. + """Handle SpeechStarted event — fast barge-in for AssemblyAI turn detection. Broadcasts UserStartedSpeakingFrame to signal the start of user speech, then pushes an interruption to cancel any bot audio. SpeechStarted fires before any transcript arrives, so the turn is cleanly started before any transcription frames are pushed. - Only applies to Mode 2 (STT turn detection). In Mode 1, VAD + - smart turn analyzer handle interruptions via the aggregator. + Only applies when using AssemblyAI's built-in turn detection. When using + Pipecat turn detection, VAD + smart turn analyzer handle interruptions. """ if self._vad_force_turn_endpoint: - return # Mode 1: handled by aggregator + return # Pipecat mode: handled by aggregator await self.start_processing_metrics() await self.broadcast_frame(UserStartedSpeakingFrame) @@ -655,15 +656,15 @@ class AssemblyAISTTService(WebsocketSTTService): await self.push_frame(EndFrame()) async def _handle_transcription(self, message: TurnMessage): - """Handle transcription results with two-mode turn detection. + """Handle transcription results with two turn detection modes. - Mode 1 (vad_force_turn_endpoint=True, Pipecat turn detection): + Pipecat turn detection (vad_force_turn_endpoint=True): - No UserStarted/StoppedSpeakingFrame from STT - end_of_turn → TranscriptionFrame (finalized set by base class if this is a ForceEndpoint response) - else → InterimTranscriptionFrame - Mode 2 (vad_force_turn_endpoint=False, STT turn detection): + AssemblyAI turn detection (vad_force_turn_endpoint=False): - UserStartedSpeakingFrame on first transcript - end_of_turn → TranscriptionFrame + UserStoppedSpeakingFrame - else → InterimTranscriptionFrame @@ -700,7 +701,7 @@ class AssemblyAISTTService(WebsocketSTTService): ) if self._vad_force_turn_endpoint: - # --- Mode 1: Pipecat turn detection --- + # --- Pipecat turn detection mode --- # No UserStarted/StoppedSpeakingFrame — VAD + smart turn analyzer handle this if is_final_turn: finalize_confirmed = bool(message.turn_is_formatted) @@ -729,11 +730,11 @@ class AssemblyAISTTService(WebsocketSTTService): ) ) else: - # --- Mode 2: STT turn detection --- + # --- AssemblyAI turn detection mode --- # SpeechStarted always arrives before transcripts with u3-rt-pro, # so UserStartedSpeakingFrame is guaranteed to be broadcast first. if is_final_turn: - # STT mode: AssemblyAI controls finalization, just mark as finalized + # AssemblyAI controls finalization, just mark as finalized await self.push_frame( TranscriptionFrame( transcript_text, From c6c2c5ba05f66d04c0999037603dc75d07fed476 Mon Sep 17 00:00:00 2001 From: zack Date: Mon, 2 Mar 2026 18:25:25 -0500 Subject: [PATCH 22/23] Fix end_of_turn_confidence_threshold: set to 1.0 (not 0.0) for universal-streaming - u3-rt-pro: Does not set parameter (not used) - universal-streaming models: Set to 1.0 to maintain fast response - This ensures fast response time matches previous implementation --- src/pipecat/services/assemblyai/stt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index cccd3d8b0..659fa3b4d 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -267,7 +267,7 @@ class AssemblyAISTTService(WebsocketSTTService): else: # universal-streaming: Different configuration (works differently) updates = { - "end_of_turn_confidence_threshold": 0.0, + "end_of_turn_confidence_threshold": 1.0, "min_turn_silence": 160, } From 038f6a77d164c3e9cbe37fa89008d7a3bb2e1e58 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 2 Mar 2026 20:24:30 -0500 Subject: [PATCH 23/23] Linting --- src/pipecat/services/assemblyai/stt.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index 659fa3b4d..c62ae959b 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -345,9 +345,7 @@ class AssemblyAISTTService(WebsocketSTTService): ): if conn_params.max_turn_silence is not None: update_config["max_turn_silence"] = conn_params.max_turn_silence - logger.info( - f"Updating max_turn_silence to: {conn_params.max_turn_silence}ms" - ) + logger.info(f"Updating max_turn_silence to: {conn_params.max_turn_silence}ms") if ( old_conn_params is None @@ -355,9 +353,7 @@ class AssemblyAISTTService(WebsocketSTTService): ): if conn_params.min_turn_silence is not None: update_config["min_turn_silence"] = conn_params.min_turn_silence - logger.info( - f"Updating min_turn_silence to: {conn_params.min_turn_silence}ms" - ) + logger.info(f"Updating min_turn_silence to: {conn_params.min_turn_silence}ms") # Send update if we have parameters to update if len(update_config) > 1: # More than just "type"