From 3fe7c1d73062b1fbb70a45b606d681fbcf24585b Mon Sep 17 00:00:00 2001 From: Michael Louis Date: Tue, 4 Mar 2025 13:59:03 -0500 Subject: [PATCH 001/132] Added ultravox service --- src/pipecat/services/ultravox.py | 393 +++++++++++++++++++++++++++++++ 1 file changed, 393 insertions(+) create mode 100644 src/pipecat/services/ultravox.py diff --git a/src/pipecat/services/ultravox.py b/src/pipecat/services/ultravox.py new file mode 100644 index 000000000..f39dc348f --- /dev/null +++ b/src/pipecat/services/ultravox.py @@ -0,0 +1,393 @@ +"""This module implements Ultravox speech-to-text with a locally-loaded model.""" + +import json +import time +import os +import numpy as np +from enum import Enum +from typing import AsyncGenerator, Optional, List +from loguru import logger +from pydantic import BaseModel +from huggingface_hub import login + +from pipecat.frames.frames import ( + Frame, + AudioRawFrame, + TranscriptionFrame, + TextFrame, + StartFrame, + EndFrame, + CancelFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, + ErrorFrame +) +from pipecat.services.ai_services import AIService +from pipecat.processors.frame_processor import FrameDirection +from pipecat.utils.time import time_now_iso8601 + +try: + from vllm import SamplingParams, AsyncLLMEngine + from vllm.engine.arg_utils import AsyncEngineArgs + from transformers import AutoTokenizer +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use Ultravox, you need to `pip install pipecat-ai[ultravox]`.") + raise Exception(f"Missing module: {e}") + +class AudioBuffer: + """Buffer to collect audio frames before processing. + + Attributes: + frames: List of AudioRawFrames to process + started_at: Timestamp when speech started + is_processing: Flag to prevent concurrent processing + """ + def __init__(self): + self.frames: List[AudioRawFrame] = [] + self.started_at: Optional[float] = None + self.is_processing: bool = False + +class UltravoxModel: + """Model wrapper for the Ultravox multimodal model. + + This class handles loading and running the Ultravox model for speech-to-text. + + Args: + model_name: The name or path of the Ultravox model to load + + Attributes: + model_name: The name of the loaded model + engine: The vLLM engine for model inference + tokenizer: The tokenizer for the model + stop_token_ids: Optional token IDs to stop generation + """ + def __init__(self, model_name: str = "fixie-ai/ultravox-v0_4_1-llama-3_1-8b"): + self.model_name = model_name + self._initialize_engine() + self._initialize_tokenizer() + self.stop_token_ids = None + + def _initialize_engine(self): + """Initialize the vLLM engine for inference.""" + engine_args = AsyncEngineArgs( + model=self.model_name, + gpu_memory_utilization=0.9, + max_model_len=8192, + trust_remote_code=True + ) + self.engine = AsyncLLMEngine.from_engine_args(engine_args) + + def _initialize_tokenizer(self): + """Initialize the tokenizer for the model.""" + self.tokenizer = AutoTokenizer.from_pretrained(self.model_name) + + def format_prompt(self, messages: list): + """Format chat messages into a prompt for the model. + + Args: + messages: List of message dictionaries with 'role' and 'content' + + Returns: + str: Formatted prompt string + """ + return self.tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True + ) + + async def generate(self, messages: list, temperature: float = 0.7, max_tokens: int = 100, audio: np.ndarray = None): + """Generate text from audio input using the model. + + Args: + messages: List of message dictionaries + temperature: Sampling temperature + max_tokens: Maximum tokens to generate + audio: Audio data as numpy array + + Yields: + str: JSON chunks of the generated response + """ + sampling_params = SamplingParams( + temperature=temperature, + max_tokens=max_tokens, + stop_token_ids=self.stop_token_ids + ) + + mm_data = { + "audio": audio + } + inputs = {"prompt": self.format_prompt(messages), "multi_modal_data": mm_data} + results_generator = self.engine.generate(inputs, sampling_params, str(time.time())) + + previous_text = "" + first_chunk = True + + async for output in results_generator: + prompt_output = output.outputs + new_text = prompt_output[0].text[len(previous_text):] + previous_text = prompt_output[0].text + + # Construct OpenAI-compatible chunk + chunk = { + "id": str(int(time.time() * 1000)), + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": self.model_name, + "choices": [ + { + "index": 0, + "delta": {}, + "finish_reason": None, + } + ], + } + + # Include the role in the first chunk + if first_chunk: + chunk["choices"][0]["delta"]["role"] = "assistant" + first_chunk = False + + # Add new text to the delta if any + if new_text: + chunk["choices"][0]["delta"]["content"] = new_text + + # Capture a finish reason if it's provided + finish_reason = prompt_output[0].finish_reason or None + if finish_reason and finish_reason != "none": + chunk["choices"][0]["finish_reason"] = finish_reason + + yield json.dumps(chunk) + +class UltravoxSTTService(AIService): + """Service to transcribe audio using the Ultravox multimodal model. + + This service collects audio frames and processes them with Ultravox + to generate text transcriptions. + + Args: + model_size: The Ultravox model to use (ModelSize enum or string) + hf_token: Hugging Face token for model access + temperature: Sampling temperature for generation + max_tokens: Maximum tokens to generate + **kwargs: Additional arguments passed to AIService + + Attributes: + model: The UltravoxModel instance + buffer: Buffer to collect audio frames + temperature: Temperature for text generation + max_tokens: Maximum tokens to generate + _connection_active: Flag indicating if service is active + """ + def __init__( + self, + *, + model_size: str = "fixie-ai/ultravox-v0_4_1-llama-3_1-8b", + hf_token: Optional[str] = None, + temperature: float = 0.7, + max_tokens: int = 100, + **kwargs, + ): + super().__init__(**kwargs) + + # Authenticate with Hugging Face if token provided + if hf_token: + login(token=hf_token) + elif os.environ.get("HF_TOKEN"): + login(token=os.environ.get("HF_TOKEN")) + else: + logger.warning("No Hugging Face token provided. Model may not load correctly.") + + # Initialize model + model_name = model_size if isinstance(model_size, str) else model_size.value + self.model = UltravoxModel(model_name=model_name) + + # Initialize service state + self.buffer = AudioBuffer() + self.temperature = temperature + self.max_tokens = max_tokens + self._connection_active = False + + logger.info(f"Initialized UltravoxSTTService with model: {model_name}") + + def can_generate_metrics(self) -> bool: + """Indicates whether this service can generate metrics. + + Returns: + bool: True, as this service supports metric generation. + """ + return True + + async def start(self, frame: StartFrame): + """Handle service start. + + Args: + frame: StartFrame that triggered this method + """ + await super().start(frame) + self._connection_active = True + logger.info("UltravoxSTTService started") + + async def stop(self, frame: EndFrame): + """Handle service stop. + + Args: + frame: EndFrame that triggered this method + """ + await super().stop(frame) + self._connection_active = False + logger.info("UltravoxSTTService stopped") + + async def cancel(self, frame: CancelFrame): + """Handle service cancellation. + + Args: + frame: CancelFrame that triggered this method + """ + await super().cancel(frame) + self._connection_active = False + self.buffer = AudioBuffer() + logger.info("UltravoxSTTService cancelled") + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames. + + This method collects audio frames and processes them when speech ends. + + Args: + frame: The frame to process + direction: Direction of the frame (input/output) + """ + await super().process_frame(frame, direction) + + if isinstance(frame, UserStartedSpeakingFrame): + logger.info("Speech started") + self.buffer = AudioBuffer() + self.buffer.started_at = time.time() + + elif isinstance(frame, AudioRawFrame) and self.buffer.started_at is not None: + self.buffer.frames.append(frame) + + elif isinstance(frame, UserStoppedSpeakingFrame): + if self.buffer.frames and not self.buffer.is_processing: + logger.info("Speech ended, processing buffer...") + await self.process_generator(self._process_audio_buffer()) + return # Return early to avoid pushing None frame + + # Only push the original frame if we haven't processed audio + if frame is not None: + await self.push_frame(frame, direction) + + async def _process_audio_buffer(self) -> AsyncGenerator[Frame, None]: + """Process collected audio frames with Ultravox. + + This method concatenates audio frames, processes them with the model, + and yields the resulting text frames. + + Yields: + Frame: TextFrame containing the transcribed text + """ + try: + self.buffer.is_processing = True + + # Check if we have valid frames before processing + if not self.buffer.frames: + logger.warning("No audio frames to process") + yield ErrorFrame("No audio frames to process") + return + + # Process audio frames + audio_arrays = [] + for f in self.buffer.frames: + if hasattr(f, 'audio') and f.audio: + # Handle bytes data - these are int16 PCM samples + if isinstance(f.audio, bytes): + try: + # Convert bytes to int16 array + arr = np.frombuffer(f.audio, dtype=np.int16) + if arr.size > 0: # Check if array is not empty + audio_arrays.append(arr) + except Exception as e: + logger.error(f"Error processing bytes audio frame: {e}") + # Handle numpy array data + elif isinstance(f.audio, np.ndarray): + if f.audio.size > 0: # Check if array is not empty + # Ensure it's int16 data + if f.audio.dtype != np.int16: + logger.info(f"Converting array from {f.audio.dtype} to int16") + audio_arrays.append(f.audio.astype(np.int16)) + else: + audio_arrays.append(f.audio) + + # Only proceed if we have valid audio arrays + if not audio_arrays: + logger.warning("No valid audio data found in frames") + yield ErrorFrame("No valid audio data found in frames") + return + + # Concatenate audio frames - all should be int16 now + audio_data = np.concatenate(audio_arrays) + + # Generate text using the model + if self.model: + try: + logger.info("Generating text from audio using model...") + full_response = "" + + # Start metrics tracking + await self.start_ttfb_metrics() + await self.start_processing_metrics() + + async for response in self.model.generate( + messages=[{ + 'role': 'user', + 'content': "<|audio|>\n" + }], + temperature=self.temperature, + max_tokens=self.max_tokens, + audio=audio_data + ): + # Stop TTFB metrics after first response + await self.stop_ttfb_metrics() + + chunk = json.loads(response) + if "choices" in chunk and len(chunk["choices"]) > 0: + delta = chunk["choices"][0]["delta"] + if "content" in delta: + new_text = delta["content"] + full_response += new_text + + # Stop processing metrics after completion + await self.stop_processing_metrics() + + logger.info(f"Generated text: {full_response}") + + # Create a transcription frame with the generated text + transcription = full_response.strip() + if transcription: + yield TranscriptionFrame( + text=transcription, + interim_text="", + timestamp=time_now_iso8601() + ) + else: + logger.warning("Empty transcription result") + yield ErrorFrame("Empty transcription result") + + except Exception as e: + logger.error(f"Error generating text from model: {e}") + yield ErrorFrame(f"Error generating text: {str(e)}") + else: + logger.warning("No model available for text generation") + yield ErrorFrame("No model available for text generation") + + except Exception as e: + logger.error(f"Error processing audio buffer: {e}") + import traceback + logger.error(traceback.format_exc()) + yield ErrorFrame(f"Error processing audio: {str(e)}") + finally: + self.buffer.is_processing = False + self.buffer.frames = [] + self.buffer.started_at = None From a62741df94a54f00e364cb3e0a8d0b7aec3bb998 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 11 Mar 2025 07:56:27 -0400 Subject: [PATCH 002/132] Add support for Chirp voices in GoogleTTSService --- CHANGELOG.md | 2 ++ src/pipecat/services/google/google.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dafdfa26..9c4ee5eaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added support for Chirp voices in `GoogleTTSService`. + - Added `on_user_turn_audio_data` and `on_bot_turn_audio_data` to `AudioBufferProcessor`. This gives the ability to grab the audio of only that turn for both the user and the bot. diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 1d914a9bb..009fd772f 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -1448,10 +1448,13 @@ class GoogleTTSService(TTSService): try: await self.start_ttfb_metrics() + # Check if the voice is a Chirp voice (including Chirp 3) or Journey voice + is_chirp_voice = "chirp" in self._voice_id.lower() is_journey_voice = "journey" in self._voice_id.lower() # Create synthesis input based on voice_id - if is_journey_voice: + if is_chirp_voice or is_journey_voice: + # Chirp and Journey voices don't support SSML, use plain text synthesis_input = texttospeech_v1.SynthesisInput(text=text) else: ssml = self._construct_ssml(text) From 740ba4e759387a11384a02728e1cbe2a6fefd50d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 11 Mar 2025 14:29:03 -0700 Subject: [PATCH 003/132] ai_services: fix abstractmethod issues --- src/pipecat/services/ai_services.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 65c9b5d92..8533a23fc 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -270,10 +270,6 @@ class TTSService(AIService): def set_voice(self, voice: str): self._voice_id = voice - @abstractmethod - async def flush_audio(self): - pass - # Converts the text to audio. @abstractmethod async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: @@ -539,6 +535,9 @@ class WebsocketTTSService(TTSService, WebsocketService): TTSService.__init__(self, **kwargs) WebsocketService.__init__(self) + async def flush_audio(self): + pass + class InterruptibleTTSService(WebsocketTTSService): """This is a base class for websocket-based TTS services that don't support @@ -762,11 +761,9 @@ class STTService(AIService): def sample_rate(self) -> int: return self._sample_rate - @abstractmethod async def set_model(self, model: str): self.set_model_name(model) - @abstractmethod async def set_language(self, language: Language): pass From ecc44111283940bad5216547c60e2bf2b4e76e36 Mon Sep 17 00:00:00 2001 From: Lucas Rothman Date: Tue, 11 Mar 2025 16:02:33 -0700 Subject: [PATCH 004/132] Tavus support for custom output rate --- src/pipecat/services/tavus.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/tavus.py b/src/pipecat/services/tavus.py index b0ca699cb..abb8b921d 100644 --- a/src/pipecat/services/tavus.py +++ b/src/pipecat/services/tavus.py @@ -37,6 +37,7 @@ class TavusVideoService(AIService): replica_id: str, persona_id: str = "pipecat0", # Use `pipecat0` so that your TTS voice is used in place of the Tavus persona session: aiohttp.ClientSession, + output_sample_rate: int = 16000, **kwargs, ) -> None: super().__init__(**kwargs) @@ -44,6 +45,7 @@ class TavusVideoService(AIService): self._replica_id = replica_id self._persona_id = persona_id self._session = session + self._output_sample_rate = output_sample_rate self._conversation_id: str @@ -94,7 +96,7 @@ class TavusVideoService(AIService): async def _encode_audio_and_send(self, audio: bytes, in_rate: int, done: bool) -> None: """Encodes audio to base64 and sends it to Tavus""" if not done: - audio = await self._resampler.resample(audio, in_rate, 16000) + audio = await self._resampler.resample(audio, in_rate, self._output_sample_rate) audio_base64 = base64.b64encode(audio).decode("utf-8") logger.trace(f"{self}: sending {len(audio)} bytes") await self._send_audio_message(audio_base64, done=done) @@ -108,7 +110,7 @@ class TavusVideoService(AIService): elif isinstance(frame, TTSAudioRawFrame): await self._encode_audio_and_send(frame.audio, frame.sample_rate, done=False) elif isinstance(frame, TTSStoppedFrame): - await self._encode_audio_and_send(b"\x00", 16000, done=True) + await self._encode_audio_and_send(b"\x00", self._output_sample_rate, done=True) await self.stop_ttfb_metrics() await self.stop_processing_metrics() elif isinstance(frame, StartInterruptionFrame): @@ -137,6 +139,7 @@ class TavusVideoService(AIService): "inference_id": self._current_idx_str, "audio": audio_base64, "done": done, + "sample_rate": self._output_sample_rate, }, } ) From e6f269a903626e60bd5126efa4a71834a2ee7291 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 7 Mar 2025 15:50:48 -0500 Subject: [PATCH 005/132] Add flush_audio to FishTTSService --- CHANGELOG.md | 2 ++ src/pipecat/services/fish.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c4ee5eaf..b1de5e50a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added support for Chirp voices in `GoogleTTSService`. +- Added a `flush_audio()` method to `FishTTSService`. + - Added `on_user_turn_audio_data` and `on_bot_turn_audio_data` to `AudioBufferProcessor`. This gives the ability to grab the audio of only that turn for both the user and the bot. diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py index 0b2729958..96968d6e9 100644 --- a/src/pipecat/services/fish.py +++ b/src/pipecat/services/fish.py @@ -148,6 +148,14 @@ class FishAudioTTSService(InterruptibleTTSService): except Exception as e: logger.error(f"Error closing websocket: {e}") + async def flush_audio(self): + """Flush any buffered audio by sending a flush event to Fish Audio.""" + logger.trace(f"{self}: Flushing audio buffers") + if not self._websocket: + return + flush_message = {"event": "flush"} + await self._get_websocket().send(ormsgpack.packb(flush_message)) + def _get_websocket(self): if self._websocket: return self._websocket From cfca7269f40e417dc7d1f611ecba7d0282c838b9 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 11 Mar 2025 21:53:03 -0400 Subject: [PATCH 006/132] Update the Cartesia voice in all demos with one built for sonic-2 --- examples/bot-ready-signalling/server/signalling_bot.py | 2 +- examples/deployment/modal-example/bot.py | 2 +- examples/foundational/01-say-one-thing.py | 2 +- examples/foundational/01a-local-audio.py | 2 +- examples/foundational/01b-livekit-audio.py | 2 +- examples/foundational/02-llm-say-one-thing.py | 2 +- examples/foundational/05-sync-speech-and-image.py | 2 +- examples/foundational/05a-local-sync-speech-and-image.py | 2 +- examples/foundational/06-listen-and-respond.py | 2 +- examples/foundational/06a-image-sync.py | 2 +- examples/foundational/07-interruptible-vad.py | 2 +- examples/foundational/07-interruptible.py | 2 +- examples/foundational/07a-interruptible-anthropic.py | 2 +- examples/foundational/07b-interruptible-langchain.py | 2 +- examples/foundational/07h-interruptible-openpipe.py | 2 +- examples/foundational/07j-interruptible-gladia.py | 2 +- examples/foundational/07l-interruptible-together.py | 2 +- examples/foundational/07o-interruptible-assemblyai.py | 2 +- examples/foundational/07s-interruptible-google-audio-in.py | 2 +- examples/foundational/10-wake-phrase.py | 2 +- examples/foundational/11-sound-effects.py | 2 +- examples/foundational/12-describe-video.py | 2 +- examples/foundational/12a-describe-video-gemini-flash.py | 2 +- examples/foundational/12b-describe-video-gpt-4o.py | 2 +- examples/foundational/12c-describe-video-anthropic.py | 2 +- examples/foundational/14-function-calling.py | 2 +- examples/foundational/14a-function-calling-anthropic.py | 2 +- examples/foundational/14b-function-calling-anthropic-video.py | 2 +- examples/foundational/14c-function-calling-together.py | 2 +- examples/foundational/14d-function-calling-video.py | 2 +- examples/foundational/14e-function-calling-gemini.py | 2 +- examples/foundational/14f-function-calling-groq.py | 2 +- examples/foundational/14g-function-calling-grok.py | 2 +- examples/foundational/14h-function-calling-azure.py | 2 +- examples/foundational/14i-function-calling-fireworks.py | 2 +- examples/foundational/14j-function-calling-nim.py | 2 +- examples/foundational/14k-function-calling-cerebras.py | 2 +- examples/foundational/14l-function-calling-deepseek.py | 2 +- examples/foundational/14n-function-calling-perplexity.py | 2 +- examples/foundational/15-switch-voices.py | 2 +- examples/foundational/15a-switch-languages.py | 2 +- examples/foundational/17-detect-user-idle.py | 2 +- examples/foundational/20a-persistent-context-openai.py | 2 +- examples/foundational/20c-persistent-context-anthropic.py | 2 +- examples/foundational/20d-persistent-context-gemini.py | 2 +- examples/foundational/22-natural-conversation.py | 2 +- examples/foundational/22b-natural-conversation-proposal.py | 2 +- examples/foundational/22c-natural-conversation-mixed-llms.py | 2 +- examples/foundational/22d-natural-conversation-gemini-audio.py | 2 +- examples/foundational/23-bot-background-sound.py | 2 +- examples/foundational/25-google-audio-in.py | 2 +- examples/foundational/26d-gemini-multimodal-live-text.py | 2 +- examples/foundational/28a-transcription-processor-openai.py | 2 +- examples/foundational/28b-transcript-processor-anthropic.py | 2 +- examples/foundational/28c-transcription-processor-gemini.py | 2 +- examples/foundational/29-livekit-audio-chat.py | 2 +- examples/foundational/30-observer.py | 2 +- examples/foundational/32-gemini-grounding-metadata.py | 2 +- examples/foundational/34-audio-recording.py | 2 +- examples/moondream-chatbot/bot.py | 2 +- examples/news-chatbot/server/news_bot.py | 2 +- examples/patient-intake/bot.py | 2 +- examples/telnyx-chatbot/bot.py | 2 +- examples/twilio-chatbot/bot.py | 2 +- examples/websocket-server/bot.py | 2 +- 65 files changed, 65 insertions(+), 65 deletions(-) diff --git a/examples/bot-ready-signalling/server/signalling_bot.py b/examples/bot-ready-signalling/server/signalling_bot.py index c6e47b812..31c940181 100644 --- a/examples/bot-ready-signalling/server/signalling_bot.py +++ b/examples/bot-ready-signalling/server/signalling_bot.py @@ -64,7 +64,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) runner = PipelineRunner() diff --git a/examples/deployment/modal-example/bot.py b/examples/deployment/modal-example/bot.py index 02bcff379..97ec11905 100644 --- a/examples/deployment/modal-example/bot.py +++ b/examples/deployment/modal-example/bot.py @@ -34,7 +34,7 @@ async def main(room_url: str, token: str): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY", ""), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22" + api_key=os.getenv("CARTESIA_API_KEY", ""), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121" ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/01-say-one-thing.py b/examples/foundational/01-say-one-thing.py index dbbb94cb4..05db53c66 100644 --- a/examples/foundational/01-say-one-thing.py +++ b/examples/foundational/01-say-one-thing.py @@ -36,7 +36,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) runner = PipelineRunner() diff --git a/examples/foundational/01a-local-audio.py b/examples/foundational/01a-local-audio.py index 20aaedfae..bb5cc22bc 100644 --- a/examples/foundational/01a-local-audio.py +++ b/examples/foundational/01a-local-audio.py @@ -29,7 +29,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) pipeline = Pipeline([tts, transport.output()]) diff --git a/examples/foundational/01b-livekit-audio.py b/examples/foundational/01b-livekit-audio.py index ed610e027..46c7f8b4c 100644 --- a/examples/foundational/01b-livekit-audio.py +++ b/examples/foundational/01b-livekit-audio.py @@ -83,7 +83,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) runner = PipelineRunner() diff --git a/examples/foundational/02-llm-say-one-thing.py b/examples/foundational/02-llm-say-one-thing.py index d19496fae..b11e1d302 100644 --- a/examples/foundational/02-llm-say-one-thing.py +++ b/examples/foundational/02-llm-say-one-thing.py @@ -37,7 +37,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/05-sync-speech-and-image.py b/examples/foundational/05-sync-speech-and-image.py index 4b343d2af..8b0021c6c 100644 --- a/examples/foundational/05-sync-speech-and-image.py +++ b/examples/foundational/05-sync-speech-and-image.py @@ -87,7 +87,7 @@ async def main(): tts = CartesiaHttpTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) imagegen = FalImageGenService( diff --git a/examples/foundational/05a-local-sync-speech-and-image.py b/examples/foundational/05a-local-sync-speech-and-image.py index 7bd689fe6..6b24629ce 100644 --- a/examples/foundational/05a-local-sync-speech-and-image.py +++ b/examples/foundational/05a-local-sync-speech-and-image.py @@ -97,7 +97,7 @@ async def main(): tts = CartesiaHttpTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) imagegen = FalImageGenService( diff --git a/examples/foundational/06-listen-and-respond.py b/examples/foundational/06-listen-and-respond.py index ee9ae056a..1a4f4d5a6 100644 --- a/examples/foundational/06-listen-and-respond.py +++ b/examples/foundational/06-listen-and-respond.py @@ -74,7 +74,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index 077f9b7dd..0230051a7 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -93,7 +93,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07-interruptible-vad.py b/examples/foundational/07-interruptible-vad.py index d6d3aca7c..ee1b647ab 100644 --- a/examples/foundational/07-interruptible-vad.py +++ b/examples/foundational/07-interruptible-vad.py @@ -47,7 +47,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index 13e728bfc..e0f7fcf95 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -46,7 +46,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07a-interruptible-anthropic.py b/examples/foundational/07a-interruptible-anthropic.py index f855f81e6..84c3838f3 100644 --- a/examples/foundational/07a-interruptible-anthropic.py +++ b/examples/foundational/07a-interruptible-anthropic.py @@ -46,7 +46,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/07b-interruptible-langchain.py b/examples/foundational/07b-interruptible-langchain.py index efa631b34..6f61eafc5 100644 --- a/examples/foundational/07b-interruptible-langchain.py +++ b/examples/foundational/07b-interruptible-langchain.py @@ -64,7 +64,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) prompt = ChatPromptTemplate.from_messages( diff --git a/examples/foundational/07h-interruptible-openpipe.py b/examples/foundational/07h-interruptible-openpipe.py index 95d786297..831919412 100644 --- a/examples/foundational/07h-interruptible-openpipe.py +++ b/examples/foundational/07h-interruptible-openpipe.py @@ -47,7 +47,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) timestamp = int(time.time()) diff --git a/examples/foundational/07j-interruptible-gladia.py b/examples/foundational/07j-interruptible-gladia.py index b1e92e787..08fc3a1c9 100644 --- a/examples/foundational/07j-interruptible-gladia.py +++ b/examples/foundational/07j-interruptible-gladia.py @@ -51,7 +51,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07l-interruptible-together.py b/examples/foundational/07l-interruptible-together.py index 54ebd4c2c..f02748c15 100644 --- a/examples/foundational/07l-interruptible-together.py +++ b/examples/foundational/07l-interruptible-together.py @@ -46,7 +46,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = TogetherLLMService( diff --git a/examples/foundational/07o-interruptible-assemblyai.py b/examples/foundational/07o-interruptible-assemblyai.py index bffa35322..3d6b7d6f9 100644 --- a/examples/foundational/07o-interruptible-assemblyai.py +++ b/examples/foundational/07o-interruptible-assemblyai.py @@ -51,7 +51,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index e1232a155..de6b85b25 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -213,7 +213,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") diff --git a/examples/foundational/10-wake-phrase.py b/examples/foundational/10-wake-phrase.py index 1bdeba15f..b6cfb05a0 100644 --- a/examples/foundational/10-wake-phrase.py +++ b/examples/foundational/10-wake-phrase.py @@ -47,7 +47,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py index 2f62d5d71..d50d9eaa2 100644 --- a/examples/foundational/11-sound-effects.py +++ b/examples/foundational/11-sound-effects.py @@ -100,7 +100,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) messages = [ diff --git a/examples/foundational/12-describe-video.py b/examples/foundational/12-describe-video.py index 2ae3ed1ad..e99599ddc 100644 --- a/examples/foundational/12-describe-video.py +++ b/examples/foundational/12-describe-video.py @@ -77,7 +77,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/12a-describe-video-gemini-flash.py b/examples/foundational/12a-describe-video-gemini-flash.py index 876aa7f07..c830a791c 100644 --- a/examples/foundational/12a-describe-video-gemini-flash.py +++ b/examples/foundational/12a-describe-video-gemini-flash.py @@ -77,7 +77,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/12b-describe-video-gpt-4o.py b/examples/foundational/12b-describe-video-gpt-4o.py index f2944b140..ab3e32837 100644 --- a/examples/foundational/12b-describe-video-gpt-4o.py +++ b/examples/foundational/12b-describe-video-gpt-4o.py @@ -76,7 +76,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/12c-describe-video-anthropic.py b/examples/foundational/12c-describe-video-anthropic.py index a8f3f0455..f5be8e434 100644 --- a/examples/foundational/12c-describe-video-anthropic.py +++ b/examples/foundational/12c-describe-video-anthropic.py @@ -76,7 +76,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index 043f278e2..c592006fe 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -58,7 +58,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/14a-function-calling-anthropic.py b/examples/foundational/14a-function-calling-anthropic.py index 14e788f99..1605b1720 100644 --- a/examples/foundational/14a-function-calling-anthropic.py +++ b/examples/foundational/14a-function-calling-anthropic.py @@ -53,7 +53,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index b31e59442..cd7f0fb97 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -62,7 +62,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index 7f47eb28e..6294617cd 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = TogetherLLMService( diff --git a/examples/foundational/14d-function-calling-video.py b/examples/foundational/14d-function-calling-video.py index 15344b7ac..714f54c5d 100644 --- a/examples/foundational/14d-function-calling-video.py +++ b/examples/foundational/14d-function-calling-video.py @@ -60,7 +60,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index 146ed77d7..99d90aa76 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -68,7 +68,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index 5bbddcc4d..4d915f5d6 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -61,7 +61,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile") diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index 423919772..979c5c252 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = GrokLLMService(api_key=os.getenv("GROK_API_KEY")) diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index 669055a55..ad8b64d34 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = AzureLLMService( diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index 88168383f..719624f08 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = FireworksLLMService( diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index 5d4b123d0..612972846 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady # text_filter=MarkdownTextFilter(), ) diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py index b9f1c6b18..252d29289 100644 --- a/examples/foundational/14k-function-calling-cerebras.py +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = CerebrasLLMService(api_key=os.getenv("CEREBRAS_API_KEY"), model="llama-3.3-70b") diff --git a/examples/foundational/14l-function-calling-deepseek.py b/examples/foundational/14l-function-calling-deepseek.py index 6add663d5..e468f1ef0 100644 --- a/examples/foundational/14l-function-calling-deepseek.py +++ b/examples/foundational/14l-function-calling-deepseek.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = DeepSeekLLMService(api_key=os.getenv("DEEPSEEK_API_KEY"), model="deepseek-chat") diff --git a/examples/foundational/14n-function-calling-perplexity.py b/examples/foundational/14n-function-calling-perplexity.py index aee185f75..6a823b0ba 100644 --- a/examples/foundational/14n-function-calling-perplexity.py +++ b/examples/foundational/14n-function-calling-perplexity.py @@ -55,7 +55,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = PerplexityLLMService(api_key=os.getenv("PERPLEXITY_API_KEY"), model="sonar") diff --git a/examples/foundational/15-switch-voices.py b/examples/foundational/15-switch-voices.py index 49280449f..466e54545 100644 --- a/examples/foundational/15-switch-voices.py +++ b/examples/foundational/15-switch-voices.py @@ -78,7 +78,7 @@ async def main(): british_lady = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) barbershop_man = CartesiaTTSService( diff --git a/examples/foundational/15a-switch-languages.py b/examples/foundational/15a-switch-languages.py index 217502ea2..beb9929e0 100644 --- a/examples/foundational/15a-switch-languages.py +++ b/examples/foundational/15a-switch-languages.py @@ -71,7 +71,7 @@ async def main(): english_tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) spanish_tts = CartesiaTTSService( diff --git a/examples/foundational/17-detect-user-idle.py b/examples/foundational/17-detect-user-idle.py index 00c969df4..0d60acd72 100644 --- a/examples/foundational/17-detect-user-idle.py +++ b/examples/foundational/17-detect-user-idle.py @@ -48,7 +48,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/20a-persistent-context-openai.py b/examples/foundational/20a-persistent-context-openai.py index 8da683a47..67f703e06 100644 --- a/examples/foundational/20a-persistent-context-openai.py +++ b/examples/foundational/20a-persistent-context-openai.py @@ -184,7 +184,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index 64829450d..29eb85bc7 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -179,7 +179,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index 74c4e69be..f5746716c 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -234,7 +234,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = GoogleLLMService(model="gemini-2.0-flash-001", api_key=os.getenv("GOOGLE_API_KEY")) diff --git a/examples/foundational/22-natural-conversation.py b/examples/foundational/22-natural-conversation.py index 9282f4e4b..d4580c712 100644 --- a/examples/foundational/22-natural-conversation.py +++ b/examples/foundational/22-natural-conversation.py @@ -56,7 +56,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) # This is the LLM that will be used to detect if the user has finished a diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 1eb2dd956..17c396bdb 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -229,7 +229,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) # This is the LLM that will be used to detect if the user has finished a diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index 362c7554d..8485be2f3 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -433,7 +433,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) # This is the LLM that will be used to detect if the user has finished a diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 0041ff530..503b0f5c9 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -644,7 +644,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) # This is the LLM that will transcribe user speech. diff --git a/examples/foundational/23-bot-background-sound.py b/examples/foundational/23-bot-background-sound.py index 1b443190d..f1653b655 100644 --- a/examples/foundational/23-bot-background-sound.py +++ b/examples/foundational/23-bot-background-sound.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/25-google-audio-in.py b/examples/foundational/25-google-audio-in.py index e3ff21d95..fb73b70f2 100644 --- a/examples/foundational/25-google-audio-in.py +++ b/examples/foundational/25-google-audio-in.py @@ -294,7 +294,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) conversation_llm = GoogleLLMService( diff --git a/examples/foundational/26d-gemini-multimodal-live-text.py b/examples/foundational/26d-gemini-multimodal-live-text.py index 2e0e27a13..147b882eb 100644 --- a/examples/foundational/26d-gemini-multimodal-live-text.py +++ b/examples/foundational/26d-gemini-multimodal-live-text.py @@ -78,7 +78,7 @@ async def main(): # ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22" + api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121" ) messages = [ diff --git a/examples/foundational/28a-transcription-processor-openai.py b/examples/foundational/28a-transcription-processor-openai.py index f341cc0dd..884197ff9 100644 --- a/examples/foundational/28a-transcription-processor-openai.py +++ b/examples/foundational/28a-transcription-processor-openai.py @@ -113,7 +113,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService( diff --git a/examples/foundational/28b-transcript-processor-anthropic.py b/examples/foundational/28b-transcript-processor-anthropic.py index 05956fdf7..acad368d3 100644 --- a/examples/foundational/28b-transcript-processor-anthropic.py +++ b/examples/foundational/28b-transcript-processor-anthropic.py @@ -113,7 +113,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/28c-transcription-processor-gemini.py b/examples/foundational/28c-transcription-processor-gemini.py index 3760a227f..62426fa85 100644 --- a/examples/foundational/28c-transcription-processor-gemini.py +++ b/examples/foundational/28c-transcription-processor-gemini.py @@ -134,7 +134,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = GoogleLLMService( diff --git a/examples/foundational/29-livekit-audio-chat.py b/examples/foundational/29-livekit-audio-chat.py index 2ad02c296..1965477e2 100644 --- a/examples/foundational/29-livekit-audio-chat.py +++ b/examples/foundational/29-livekit-audio-chat.py @@ -131,7 +131,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) messages = [ diff --git a/examples/foundational/30-observer.py b/examples/foundational/30-observer.py index 49885d82c..f44d47e9f 100644 --- a/examples/foundational/30-observer.py +++ b/examples/foundational/30-observer.py @@ -89,7 +89,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/32-gemini-grounding-metadata.py b/examples/foundational/32-gemini-grounding-metadata.py index 516549694..0432bb830 100644 --- a/examples/foundational/32-gemini-grounding-metadata.py +++ b/examples/foundational/32-gemini-grounding-metadata.py @@ -81,7 +81,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) # Initialize the Gemini Multimodal Live model diff --git a/examples/foundational/34-audio-recording.py b/examples/foundational/34-audio-recording.py index 1d0431b83..94877c722 100644 --- a/examples/foundational/34-audio-recording.py +++ b/examples/foundational/34-audio-recording.py @@ -107,7 +107,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4") diff --git a/examples/moondream-chatbot/bot.py b/examples/moondream-chatbot/bot.py index 5fb86ac79..d44b3a34e 100644 --- a/examples/moondream-chatbot/bot.py +++ b/examples/moondream-chatbot/bot.py @@ -154,7 +154,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/news-chatbot/server/news_bot.py b/examples/news-chatbot/server/news_bot.py index d1cde2f22..f6787e933 100644 --- a/examples/news-chatbot/server/news_bot.py +++ b/examples/news-chatbot/server/news_bot.py @@ -96,7 +96,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady text_filter=MarkdownTextFilter(), ) diff --git a/examples/patient-intake/bot.py b/examples/patient-intake/bot.py index 75279fce3..5ac2ebe93 100644 --- a/examples/patient-intake/bot.py +++ b/examples/patient-intake/bot.py @@ -303,7 +303,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) # tts = CartesiaTTSService( diff --git a/examples/telnyx-chatbot/bot.py b/examples/telnyx-chatbot/bot.py index 8243e05a4..a31e4ac91 100644 --- a/examples/telnyx-chatbot/bot.py +++ b/examples/telnyx-chatbot/bot.py @@ -54,7 +54,7 @@ async def run_bot( tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) messages = [ diff --git a/examples/twilio-chatbot/bot.py b/examples/twilio-chatbot/bot.py index afddc3fe9..be3a0c40d 100644 --- a/examples/twilio-chatbot/bot.py +++ b/examples/twilio-chatbot/bot.py @@ -74,7 +74,7 @@ async def run_bot(websocket_client: WebSocket, stream_sid: str, testing: bool): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady push_silence_after_stop=testing, ) diff --git a/examples/websocket-server/bot.py b/examples/websocket-server/bot.py index d44c77df8..8f2e98e7d 100644 --- a/examples/websocket-server/bot.py +++ b/examples/websocket-server/bot.py @@ -97,7 +97,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) messages = [ From 3522bbb533d9ec49e510b329215f185b0a5a0494 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 11 Mar 2025 21:55:18 -0400 Subject: [PATCH 007/132] tmp --- examples/bot-ready-signalling/server/signalling_bot.py | 2 +- examples/foundational/01-say-one-thing.py | 2 +- examples/foundational/01a-local-audio.py | 2 +- examples/foundational/01b-livekit-audio.py | 2 +- examples/foundational/02-llm-say-one-thing.py | 2 +- examples/foundational/05-sync-speech-and-image.py | 2 +- examples/foundational/05a-local-sync-speech-and-image.py | 2 +- examples/foundational/06-listen-and-respond.py | 2 +- examples/foundational/06a-image-sync.py | 2 +- examples/foundational/07-interruptible-vad.py | 2 +- examples/foundational/07-interruptible.py | 2 +- examples/foundational/07a-interruptible-anthropic.py | 2 +- examples/foundational/07b-interruptible-langchain.py | 2 +- examples/foundational/07h-interruptible-openpipe.py | 2 +- examples/foundational/07j-interruptible-gladia.py | 2 +- examples/foundational/07l-interruptible-together.py | 2 +- examples/foundational/07o-interruptible-assemblyai.py | 2 +- examples/foundational/07s-interruptible-google-audio-in.py | 2 +- examples/foundational/10-wake-phrase.py | 2 +- examples/foundational/11-sound-effects.py | 2 +- examples/foundational/12-describe-video.py | 2 +- examples/foundational/12a-describe-video-gemini-flash.py | 2 +- examples/foundational/12b-describe-video-gpt-4o.py | 2 +- examples/foundational/12c-describe-video-anthropic.py | 2 +- examples/foundational/14-function-calling.py | 2 +- examples/foundational/14a-function-calling-anthropic.py | 2 +- .../foundational/14b-function-calling-anthropic-video.py | 2 +- examples/foundational/14c-function-calling-together.py | 2 +- examples/foundational/14d-function-calling-video.py | 2 +- examples/foundational/14e-function-calling-gemini.py | 2 +- examples/foundational/14f-function-calling-groq.py | 2 +- examples/foundational/14g-function-calling-grok.py | 2 +- examples/foundational/14h-function-calling-azure.py | 2 +- examples/foundational/14i-function-calling-fireworks.py | 2 +- examples/foundational/14j-function-calling-nim.py | 2 +- examples/foundational/14k-function-calling-cerebras.py | 2 +- examples/foundational/14l-function-calling-deepseek.py | 2 +- examples/foundational/14n-function-calling-perplexity.py | 2 +- examples/foundational/15-switch-voices.py | 7 +++++-- examples/foundational/15a-switch-languages.py | 2 +- examples/foundational/17-detect-user-idle.py | 2 +- examples/foundational/20a-persistent-context-openai.py | 2 +- examples/foundational/20c-persistent-context-anthropic.py | 2 +- examples/foundational/20d-persistent-context-gemini.py | 2 +- examples/foundational/22-natural-conversation.py | 2 +- examples/foundational/22b-natural-conversation-proposal.py | 2 +- .../foundational/22c-natural-conversation-mixed-llms.py | 2 +- .../foundational/22d-natural-conversation-gemini-audio.py | 2 +- examples/foundational/23-bot-background-sound.py | 2 +- examples/foundational/25-google-audio-in.py | 2 +- .../foundational/28a-transcription-processor-openai.py | 2 +- .../foundational/28b-transcript-processor-anthropic.py | 2 +- .../foundational/28c-transcription-processor-gemini.py | 2 +- examples/foundational/29-livekit-audio-chat.py | 2 +- examples/foundational/30-observer.py | 2 +- examples/foundational/32-gemini-grounding-metadata.py | 2 +- examples/moondream-chatbot/bot.py | 2 +- examples/news-chatbot/server/news_bot.py | 2 +- examples/patient-intake/bot.py | 2 +- examples/telnyx-chatbot/bot.py | 2 +- examples/twilio-chatbot/bot.py | 2 +- examples/websocket-server/bot.py | 2 +- 62 files changed, 66 insertions(+), 63 deletions(-) diff --git a/examples/bot-ready-signalling/server/signalling_bot.py b/examples/bot-ready-signalling/server/signalling_bot.py index 31c940181..3b98700aa 100644 --- a/examples/bot-ready-signalling/server/signalling_bot.py +++ b/examples/bot-ready-signalling/server/signalling_bot.py @@ -64,7 +64,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) runner = PipelineRunner() diff --git a/examples/foundational/01-say-one-thing.py b/examples/foundational/01-say-one-thing.py index 05db53c66..945afefda 100644 --- a/examples/foundational/01-say-one-thing.py +++ b/examples/foundational/01-say-one-thing.py @@ -36,7 +36,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) runner = PipelineRunner() diff --git a/examples/foundational/01a-local-audio.py b/examples/foundational/01a-local-audio.py index bb5cc22bc..c5ed3610e 100644 --- a/examples/foundational/01a-local-audio.py +++ b/examples/foundational/01a-local-audio.py @@ -29,7 +29,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) pipeline = Pipeline([tts, transport.output()]) diff --git a/examples/foundational/01b-livekit-audio.py b/examples/foundational/01b-livekit-audio.py index 46c7f8b4c..5529d4184 100644 --- a/examples/foundational/01b-livekit-audio.py +++ b/examples/foundational/01b-livekit-audio.py @@ -83,7 +83,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) runner = PipelineRunner() diff --git a/examples/foundational/02-llm-say-one-thing.py b/examples/foundational/02-llm-say-one-thing.py index b11e1d302..f6f8030f1 100644 --- a/examples/foundational/02-llm-say-one-thing.py +++ b/examples/foundational/02-llm-say-one-thing.py @@ -37,7 +37,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/05-sync-speech-and-image.py b/examples/foundational/05-sync-speech-and-image.py index 8b0021c6c..f6d415e64 100644 --- a/examples/foundational/05-sync-speech-and-image.py +++ b/examples/foundational/05-sync-speech-and-image.py @@ -87,7 +87,7 @@ async def main(): tts = CartesiaHttpTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) imagegen = FalImageGenService( diff --git a/examples/foundational/05a-local-sync-speech-and-image.py b/examples/foundational/05a-local-sync-speech-and-image.py index 6b24629ce..c09631061 100644 --- a/examples/foundational/05a-local-sync-speech-and-image.py +++ b/examples/foundational/05a-local-sync-speech-and-image.py @@ -97,7 +97,7 @@ async def main(): tts = CartesiaHttpTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) imagegen = FalImageGenService( diff --git a/examples/foundational/06-listen-and-respond.py b/examples/foundational/06-listen-and-respond.py index 1a4f4d5a6..d59d143f8 100644 --- a/examples/foundational/06-listen-and-respond.py +++ b/examples/foundational/06-listen-and-respond.py @@ -74,7 +74,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index 0230051a7..ab1c9399d 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -93,7 +93,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07-interruptible-vad.py b/examples/foundational/07-interruptible-vad.py index ee1b647ab..2afc36702 100644 --- a/examples/foundational/07-interruptible-vad.py +++ b/examples/foundational/07-interruptible-vad.py @@ -47,7 +47,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index e0f7fcf95..5eab40454 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -46,7 +46,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07a-interruptible-anthropic.py b/examples/foundational/07a-interruptible-anthropic.py index 84c3838f3..85f5ac68e 100644 --- a/examples/foundational/07a-interruptible-anthropic.py +++ b/examples/foundational/07a-interruptible-anthropic.py @@ -46,7 +46,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/07b-interruptible-langchain.py b/examples/foundational/07b-interruptible-langchain.py index 6f61eafc5..dd7b0b69c 100644 --- a/examples/foundational/07b-interruptible-langchain.py +++ b/examples/foundational/07b-interruptible-langchain.py @@ -64,7 +64,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) prompt = ChatPromptTemplate.from_messages( diff --git a/examples/foundational/07h-interruptible-openpipe.py b/examples/foundational/07h-interruptible-openpipe.py index 831919412..bded77139 100644 --- a/examples/foundational/07h-interruptible-openpipe.py +++ b/examples/foundational/07h-interruptible-openpipe.py @@ -47,7 +47,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) timestamp = int(time.time()) diff --git a/examples/foundational/07j-interruptible-gladia.py b/examples/foundational/07j-interruptible-gladia.py index 08fc3a1c9..7dcd44a7a 100644 --- a/examples/foundational/07j-interruptible-gladia.py +++ b/examples/foundational/07j-interruptible-gladia.py @@ -51,7 +51,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07l-interruptible-together.py b/examples/foundational/07l-interruptible-together.py index f02748c15..d17a4fe7f 100644 --- a/examples/foundational/07l-interruptible-together.py +++ b/examples/foundational/07l-interruptible-together.py @@ -46,7 +46,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = TogetherLLMService( diff --git a/examples/foundational/07o-interruptible-assemblyai.py b/examples/foundational/07o-interruptible-assemblyai.py index 3d6b7d6f9..3c30cabc9 100644 --- a/examples/foundational/07o-interruptible-assemblyai.py +++ b/examples/foundational/07o-interruptible-assemblyai.py @@ -51,7 +51,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index de6b85b25..76711d730 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -213,7 +213,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") diff --git a/examples/foundational/10-wake-phrase.py b/examples/foundational/10-wake-phrase.py index b6cfb05a0..0fbd47f4a 100644 --- a/examples/foundational/10-wake-phrase.py +++ b/examples/foundational/10-wake-phrase.py @@ -47,7 +47,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py index d50d9eaa2..ed189551a 100644 --- a/examples/foundational/11-sound-effects.py +++ b/examples/foundational/11-sound-effects.py @@ -100,7 +100,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) messages = [ diff --git a/examples/foundational/12-describe-video.py b/examples/foundational/12-describe-video.py index e99599ddc..5eeb69718 100644 --- a/examples/foundational/12-describe-video.py +++ b/examples/foundational/12-describe-video.py @@ -77,7 +77,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/12a-describe-video-gemini-flash.py b/examples/foundational/12a-describe-video-gemini-flash.py index c830a791c..59a9f40db 100644 --- a/examples/foundational/12a-describe-video-gemini-flash.py +++ b/examples/foundational/12a-describe-video-gemini-flash.py @@ -77,7 +77,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/12b-describe-video-gpt-4o.py b/examples/foundational/12b-describe-video-gpt-4o.py index ab3e32837..fb3aa039e 100644 --- a/examples/foundational/12b-describe-video-gpt-4o.py +++ b/examples/foundational/12b-describe-video-gpt-4o.py @@ -76,7 +76,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/12c-describe-video-anthropic.py b/examples/foundational/12c-describe-video-anthropic.py index f5be8e434..bd364a636 100644 --- a/examples/foundational/12c-describe-video-anthropic.py +++ b/examples/foundational/12c-describe-video-anthropic.py @@ -76,7 +76,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index c592006fe..1d4e37344 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -58,7 +58,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/14a-function-calling-anthropic.py b/examples/foundational/14a-function-calling-anthropic.py index 1605b1720..13505550b 100644 --- a/examples/foundational/14a-function-calling-anthropic.py +++ b/examples/foundational/14a-function-calling-anthropic.py @@ -53,7 +53,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index cd7f0fb97..8c49900fa 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -62,7 +62,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index 6294617cd..dd3eb714f 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = TogetherLLMService( diff --git a/examples/foundational/14d-function-calling-video.py b/examples/foundational/14d-function-calling-video.py index 714f54c5d..6e290d55f 100644 --- a/examples/foundational/14d-function-calling-video.py +++ b/examples/foundational/14d-function-calling-video.py @@ -60,7 +60,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index 99d90aa76..bf022a65a 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -68,7 +68,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index 4d915f5d6..c402ac91a 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -61,7 +61,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile") diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index 979c5c252..e6b822e39 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = GrokLLMService(api_key=os.getenv("GROK_API_KEY")) diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index ad8b64d34..aefa1a474 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = AzureLLMService( diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index 719624f08..50291615b 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = FireworksLLMService( diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index 612972846..d703d637a 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady # text_filter=MarkdownTextFilter(), ) diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py index 252d29289..9b3641307 100644 --- a/examples/foundational/14k-function-calling-cerebras.py +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = CerebrasLLMService(api_key=os.getenv("CEREBRAS_API_KEY"), model="llama-3.3-70b") diff --git a/examples/foundational/14l-function-calling-deepseek.py b/examples/foundational/14l-function-calling-deepseek.py index e468f1ef0..0360f0b84 100644 --- a/examples/foundational/14l-function-calling-deepseek.py +++ b/examples/foundational/14l-function-calling-deepseek.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = DeepSeekLLMService(api_key=os.getenv("DEEPSEEK_API_KEY"), model="deepseek-chat") diff --git a/examples/foundational/14n-function-calling-perplexity.py b/examples/foundational/14n-function-calling-perplexity.py index 6a823b0ba..9d2439fb0 100644 --- a/examples/foundational/14n-function-calling-perplexity.py +++ b/examples/foundational/14n-function-calling-perplexity.py @@ -55,7 +55,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = PerplexityLLMService(api_key=os.getenv("PERPLEXITY_API_KEY"), model="sonar") diff --git a/examples/foundational/15-switch-voices.py b/examples/foundational/15-switch-voices.py index 466e54545..15bd2f954 100644 --- a/examples/foundational/15-switch-voices.py +++ b/examples/foundational/15-switch-voices.py @@ -78,7 +78,7 @@ async def main(): british_lady = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) barbershop_man = CartesiaTTSService( @@ -125,7 +125,10 @@ async def main(): llm, # LLM ParallelPipeline( # TTS (one of the following vocies) [FunctionFilter(news_lady_filter), news_lady], # News Lady voice - [FunctionFilter(british_lady_filter), british_lady], # British Lady voice + [ + FunctionFilter(british_lady_filter), + british_lady, + ], # British Reading Lady voice [FunctionFilter(barbershop_man_filter), barbershop_man], # Barbershop Man voice ), transport.output(), # Transport bot output diff --git a/examples/foundational/15a-switch-languages.py b/examples/foundational/15a-switch-languages.py index beb9929e0..4e05efcb0 100644 --- a/examples/foundational/15a-switch-languages.py +++ b/examples/foundational/15a-switch-languages.py @@ -71,7 +71,7 @@ async def main(): english_tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) spanish_tts = CartesiaTTSService( diff --git a/examples/foundational/17-detect-user-idle.py b/examples/foundational/17-detect-user-idle.py index 0d60acd72..0a41f9d84 100644 --- a/examples/foundational/17-detect-user-idle.py +++ b/examples/foundational/17-detect-user-idle.py @@ -48,7 +48,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/20a-persistent-context-openai.py b/examples/foundational/20a-persistent-context-openai.py index 67f703e06..0be0536cd 100644 --- a/examples/foundational/20a-persistent-context-openai.py +++ b/examples/foundational/20a-persistent-context-openai.py @@ -184,7 +184,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index 29eb85bc7..65d45c3d6 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -179,7 +179,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index f5746716c..740bdc68f 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -234,7 +234,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = GoogleLLMService(model="gemini-2.0-flash-001", api_key=os.getenv("GOOGLE_API_KEY")) diff --git a/examples/foundational/22-natural-conversation.py b/examples/foundational/22-natural-conversation.py index d4580c712..d1f84bb90 100644 --- a/examples/foundational/22-natural-conversation.py +++ b/examples/foundational/22-natural-conversation.py @@ -56,7 +56,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) # This is the LLM that will be used to detect if the user has finished a diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 17c396bdb..c97289217 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -229,7 +229,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) # This is the LLM that will be used to detect if the user has finished a diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index 8485be2f3..d342a2765 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -433,7 +433,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) # This is the LLM that will be used to detect if the user has finished a diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 503b0f5c9..9e3b8c9a3 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -644,7 +644,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) # This is the LLM that will transcribe user speech. diff --git a/examples/foundational/23-bot-background-sound.py b/examples/foundational/23-bot-background-sound.py index f1653b655..92d6cdc42 100644 --- a/examples/foundational/23-bot-background-sound.py +++ b/examples/foundational/23-bot-background-sound.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/25-google-audio-in.py b/examples/foundational/25-google-audio-in.py index fb73b70f2..830cfb769 100644 --- a/examples/foundational/25-google-audio-in.py +++ b/examples/foundational/25-google-audio-in.py @@ -294,7 +294,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) conversation_llm = GoogleLLMService( diff --git a/examples/foundational/28a-transcription-processor-openai.py b/examples/foundational/28a-transcription-processor-openai.py index 884197ff9..a3707e347 100644 --- a/examples/foundational/28a-transcription-processor-openai.py +++ b/examples/foundational/28a-transcription-processor-openai.py @@ -113,7 +113,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService( diff --git a/examples/foundational/28b-transcript-processor-anthropic.py b/examples/foundational/28b-transcript-processor-anthropic.py index acad368d3..c9f2672e0 100644 --- a/examples/foundational/28b-transcript-processor-anthropic.py +++ b/examples/foundational/28b-transcript-processor-anthropic.py @@ -113,7 +113,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/28c-transcription-processor-gemini.py b/examples/foundational/28c-transcription-processor-gemini.py index 62426fa85..558edc76d 100644 --- a/examples/foundational/28c-transcription-processor-gemini.py +++ b/examples/foundational/28c-transcription-processor-gemini.py @@ -134,7 +134,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = GoogleLLMService( diff --git a/examples/foundational/29-livekit-audio-chat.py b/examples/foundational/29-livekit-audio-chat.py index 1965477e2..e5afb5ffb 100644 --- a/examples/foundational/29-livekit-audio-chat.py +++ b/examples/foundational/29-livekit-audio-chat.py @@ -131,7 +131,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) messages = [ diff --git a/examples/foundational/30-observer.py b/examples/foundational/30-observer.py index f44d47e9f..129407628 100644 --- a/examples/foundational/30-observer.py +++ b/examples/foundational/30-observer.py @@ -89,7 +89,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/32-gemini-grounding-metadata.py b/examples/foundational/32-gemini-grounding-metadata.py index 0432bb830..5f0a497a5 100644 --- a/examples/foundational/32-gemini-grounding-metadata.py +++ b/examples/foundational/32-gemini-grounding-metadata.py @@ -81,7 +81,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) # Initialize the Gemini Multimodal Live model diff --git a/examples/moondream-chatbot/bot.py b/examples/moondream-chatbot/bot.py index d44b3a34e..5dd88f4d1 100644 --- a/examples/moondream-chatbot/bot.py +++ b/examples/moondream-chatbot/bot.py @@ -154,7 +154,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/news-chatbot/server/news_bot.py b/examples/news-chatbot/server/news_bot.py index f6787e933..2a389094e 100644 --- a/examples/news-chatbot/server/news_bot.py +++ b/examples/news-chatbot/server/news_bot.py @@ -96,7 +96,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady text_filter=MarkdownTextFilter(), ) diff --git a/examples/patient-intake/bot.py b/examples/patient-intake/bot.py index 5ac2ebe93..b6f7fb941 100644 --- a/examples/patient-intake/bot.py +++ b/examples/patient-intake/bot.py @@ -303,7 +303,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) # tts = CartesiaTTSService( diff --git a/examples/telnyx-chatbot/bot.py b/examples/telnyx-chatbot/bot.py index a31e4ac91..94f44e2b1 100644 --- a/examples/telnyx-chatbot/bot.py +++ b/examples/telnyx-chatbot/bot.py @@ -54,7 +54,7 @@ async def run_bot( tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) messages = [ diff --git a/examples/twilio-chatbot/bot.py b/examples/twilio-chatbot/bot.py index be3a0c40d..7b05e6dfe 100644 --- a/examples/twilio-chatbot/bot.py +++ b/examples/twilio-chatbot/bot.py @@ -74,7 +74,7 @@ async def run_bot(websocket_client: WebSocket, stream_sid: str, testing: bool): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady push_silence_after_stop=testing, ) diff --git a/examples/websocket-server/bot.py b/examples/websocket-server/bot.py index 8f2e98e7d..489605aef 100644 --- a/examples/websocket-server/bot.py +++ b/examples/websocket-server/bot.py @@ -97,7 +97,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) messages = [ From 5f97f6ff94417733f812caf9300f1ab4232b873e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 7 Mar 2025 16:08:30 -0500 Subject: [PATCH 008/132] Add flush_audio() to LmntTTSService --- CHANGELOG.md | 2 +- src/pipecat/services/lmnt.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1de5e50a..cbafd4de7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added support for Chirp voices in `GoogleTTSService`. -- Added a `flush_audio()` method to `FishTTSService`. +- Added a `flush_audio()` method to `FishTTSService` and `LmntTTSService`. - Added `on_user_turn_audio_data` and `on_bot_turn_audio_data` to `AudioBufferProcessor`. This gives the ability to grab the audio of only that diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py index 31ddaf123..cfdf8e6cd 100644 --- a/src/pipecat/services/lmnt.py +++ b/src/pipecat/services/lmnt.py @@ -170,6 +170,11 @@ class LmntTTSService(InterruptibleTTSService): return self._websocket raise Exception("Websocket not connected") + async def flush_audio(self): + if not self._websocket: + return + await self._get_websocket().send(json.dumps({"flush": True})) + async def _receive_messages(self): """Receive messages from LMNT websocket.""" async for message in self._get_websocket(): From 4a363bebf0f86b6d773741066f25c662fa80634e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 7 Mar 2025 16:04:41 -0500 Subject: [PATCH 009/132] Add a set_language convenience method for GoogleSTTService --- CHANGELOG.md | 4 ++++ src/pipecat/services/google/google.py | 12 +++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbafd4de7..4e730b70d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added a `flush_audio()` method to `FishTTSService` and `LmntTTSService`. +- Added a `set_language` convenience method for `GoogleSTTService`, allowing + you to set a single language. This is in addition to the `set_languages` + method which allows you to set a list of languages. + - Added `on_user_turn_audio_data` and `on_bot_turn_audio_data` to `AudioBufferProcessor`. This gives the ability to grab the audio of only that turn for both the user and the bot. diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 009fd772f..9d4a0a8a8 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -58,7 +58,6 @@ from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import ImageGenService, LLMService, STTService, TTSService from pipecat.services.google.frames import LLMSearchResponseFrame from pipecat.services.openai import ( - BaseOpenAILLMService, OpenAIAssistantContextAggregator, OpenAILLMService, OpenAIUnhandledFunctionException, @@ -1730,6 +1729,17 @@ class GoogleSTTService(STTService): await self._disconnect() await self._connect() + async def set_language(self, language: Language): + """Update the service's recognition language. + + A convenience method for setting a single language. + + Args: + language: New language for recognition. + """ + logger.debug(f"Switching STT language to: {language}") + await self.set_languages([language]) + async def set_languages(self, languages: List[Language]): """Update the service's recognition languages. From d9ef19233a59e0cd050d399827a03233bf03ae00 Mon Sep 17 00:00:00 2001 From: Michael Louis Date: Wed, 12 Mar 2025 10:30:23 -0400 Subject: [PATCH 010/132] Added foundational example for ultravox --- .../07u-interruptible-ultravox.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 examples/foundational/07u-interruptible-ultravox.py diff --git a/examples/foundational/07u-interruptible-ultravox.py b/examples/foundational/07u-interruptible-ultravox.py new file mode 100644 index 000000000..195eaf197 --- /dev/null +++ b/examples/foundational/07u-interruptible-ultravox.py @@ -0,0 +1,93 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.services.ultravox import UltravoxSTTService +from pipecat.services.cartesia import CartesiaTTSService + +load_dotenv(override=True) + +# NOTE: This example requires GPU resources to run efficiently. +# The Ultravox model is compute-intensive and performs best with GPU acceleration. +# This can be deployed on cloud GPU providers like Cerebrium.ai for optimal performance. + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + +#Want to initialize the ultravox processor since it takes time to load the model and dont +#want to load it every time the pipeline is run +ultravox_processor = UltravoxSTTService( + model_size="fixie-ai/ultravox-v0_4_1-llama-3_1-8b", + hf_token=os.getenv("HF_TOKEN"), +) + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=False, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + vad_audio_passthrough=True, + ), + ) + + tts = CartesiaTTSService( + api_key=os.environ.get("CARTESIA_API_KEY"), + voice_id='97f4b8fb-f2fe-444b-bb9a-c109783a857a', + + ) + + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + ultravox_processor, + tts, # TTS + transport.output(), # Transport bot output + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + ), + ) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + await task.cancel() + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) + + From 23a4933af9d58828efd62b867f63b6045b9f632c Mon Sep 17 00:00:00 2001 From: Adnan Siddiquei Date: Wed, 12 Mar 2025 17:15:31 +0000 Subject: [PATCH 011/132] Initial implementation of Neuphonic service. A TTS provider. --- dot-env.template | 3 + pyproject.toml | 1 + src/pipecat/services/neuphonic.py | 364 ++++++++++++++++++++++++++++++ 3 files changed, 368 insertions(+) create mode 100644 src/pipecat/services/neuphonic.py diff --git a/dot-env.template b/dot-env.template index e31681235..331eba79c 100644 --- a/dot-env.template +++ b/dot-env.template @@ -29,6 +29,9 @@ DAILY_SAMPLE_ROOM_URL=https://... ELEVENLABS_API_KEY=... ELEVENLABS_VOICE_ID=... +# Neuphonic +NEUPHONIC_API_TOKEN=... + # Fal FAL_KEY=... diff --git a/pyproject.toml b/pyproject.toml index dd58b1bfe..b5a870941 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ aws = [ "boto3~=1.35.99" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.3.1", "websockets~=13.1" ] +neuphonic = [ "pyneuphonic~=1.5.10", "websockets>=12.0,<14.0" ] cerebras = [] deepseek = [] daily = [ "daily-python~=0.15.0" ] diff --git a/src/pipecat/services/neuphonic.py b/src/pipecat/services/neuphonic.py new file mode 100644 index 000000000..b14c325f8 --- /dev/null +++ b/src/pipecat/services/neuphonic.py @@ -0,0 +1,364 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import base64 +import json +from typing import Any, AsyncGenerator, Mapping, Optional + +from loguru import logger +from pydantic import BaseModel + +from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, + CancelFrame, + EndFrame, + ErrorFrame, + Frame, + LLMFullResponseEndFrame, + StartFrame, + StartInterruptionFrame, + TTSAudioRawFrame, + TTSSpeakFrame, + TTSStartedFrame, + TTSStoppedFrame, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.ai_services import TTSService, WordTTSService +from pipecat.services.websocket_service import WebsocketService +from pipecat.transcriptions.language import Language + +# See .env.example for Neuphonic configuration needed +try: + import websockets + from pyneuphonic import Neuphonic, TTSConfig +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use Neuphonic, you need to `pip install pipecat-ai[neuphonic]`. Also, set `NEUPHONIC_API_TOKEN` environment variable." + ) + raise Exception(f"Missing module: {e}") + +# Models that support language codes +NEUPHONIC_MULTILINGUAL_MODELS = { + "neu_fast", + "neu_hq", +} + + +def language_to_neuphonic_lang_code(language: Language) -> Optional[str]: + BASE_LANGUAGES = { + Language.DE: "de", + Language.EN: "en", + Language.ES: "es", + Language.NL: "nl", + Language.AR: "ar", + } + + result = BASE_LANGUAGES.get(language) + + # If not found in base languages, try to find the base language from a variant + if not result: + # Convert enum value to string and get the base language part (e.g. es-ES -> es) + lang_str = str(language.value) + base_code = lang_str.split("-")[0].lower() + # Look up the base code in our supported languages + result = base_code if base_code in BASE_LANGUAGES.values() else None + + return result + + +class NeuphonicTTSService(WordTTSService, WebsocketService): + class InputParams(BaseModel): + language: Optional[Language] = Language.EN + speed: Optional[float] = 1.0 + + def __init__( + self, + *, + api_key: str, + voice_id: Optional[str] = None, + model: str = "neu_hq", + url: str = "wss://api.neuphonic.com", + sample_rate: Optional[int] = 22050, + encoding: str = "pcm_linear", + params: InputParams = InputParams(), + **kwargs, + ): + WordTTSService.__init__( + self, + aggregate_sentences=True, + push_text_frames=False, + push_stop_frames=True, + stop_frame_timeout_s=2.0, + sample_rate=sample_rate, + **kwargs, + ) + WebsocketService.__init__(self) + + self._api_key = api_key + self._url = url + self._settings = { + "lang_code": self.language_to_service_language(params.language), + "speed": params.speed, + "encoding": encoding, + "model": model, + "sampling_rate": sample_rate, + } + self.set_model_name(model) + self.set_voice(voice_id) + + # Indicates if we have sent TTSStartedFrame. It will reset to False when + # there's an interruption or TTSStoppedFrame. + self._started = False + self._cumulative_time = 0 + + def can_generate_metrics(self) -> bool: + return True + + def language_to_service_language(self, language: Language) -> Optional[str]: + return language_to_neuphonic_lang_code(language) + + async def set_model(self, model: str): + await super().set_model(model) + logger.info(f"Switching TTS model to: [{model}]") + await self._disconnect() + await self._connect() + + async def _update_settings(self, settings: Mapping[str, Any]): + if "voice_id" in settings: + self.set_voice(settings["voice_id"]) + + await super()._update_settings(settings) + await self._disconnect() + await self._connect() + logger.info(f"Switching TTS to settings: [{self._settings}]") + + async def start(self, frame: StartFrame): + await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + await super().cancel(frame) + await self._disconnect() + + async def flush_audio(self): + if self._websocket: + msg = {"text": ""} + await self._websocket.send(json.dumps(msg)) + + async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + await super().push_frame(frame, direction) + if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): + self._started = False + if isinstance(frame, TTSStoppedFrame): + await self.add_word_timestamps([("LLMFullResponseEndFrame", 0), ("Reset", 0)]) + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + # If we received a TTSSpeakFrame and the LLM response included text (it + # might be that it's only a function calling response) we pause + # processing more frames until we receive a BotStoppedSpeakingFrame. + if isinstance(frame, TTSSpeakFrame): + await self.pause_processing_frames() + elif isinstance(frame, LLMFullResponseEndFrame) and self._started: + await self.pause_processing_frames() + elif isinstance(frame, BotStoppedSpeakingFrame): + await self.resume_processing_frames() + + async def _connect(self): + await self._connect_websocket() + + self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._keepalive_task = self.create_task(self._keepalive_task_handler()) + + async def _disconnect(self): + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None + + if self._keepalive_task: + await self.cancel_task(self._keepalive_task) + self._keepalive_task = None + + await self._disconnect_websocket() + + async def _connect_websocket(self): + try: + logger.debug("Connecting to Neuphonic") + + tts_config = { + **self._settings, + "voice_id": self._voice_id, + "model": self.model_name, + } + + query_params = [f"api_key={self._api_key}"] + for key, value in tts_config.items(): + if value is not None: + query_params.append(f"{key}={value}") + + url = f"{self._url}/speak/{self._settings['lang_code']}?{'&'.join(query_params)}" + + self._websocket = await websockets.connect(url) + + except Exception as e: + logger.error(f"{self} initialization error: {e}") + self._websocket = None + + async def _disconnect_websocket(self): + try: + await self.stop_all_metrics() + + if self._websocket: + logger.debug("Disconnecting from Neuphonic") + await self._websocket.close() + self._websocket = None + + self._started = False + except Exception as e: + logger.error(f"{self} error closing websocket: {e}") + + async def _receive_messages(self): + async for message in self._websocket: + if isinstance(message, str): + msg = json.loads(message) + if msg.get("data", {}).get("audio") is not None: + await self.stop_ttfb_metrics() + self.start_word_timestamps() + + audio = base64.b64decode(msg["data"]["audio"]) + frame = TTSAudioRawFrame(audio, self.sample_rate, 1) + await self.push_frame(frame) + + async def _keepalive_task_handler(self): + while True: + await asyncio.sleep(10) + await self._send_text("") + + async def _send_text(self, text: str): + if self._websocket: + msg = {"text": text} + logger.debug(f"Sending text to websocket: {msg}") + await self._websocket.send(json.dumps(msg)) + + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + logger.debug(f"Generating TTS: [{text}]") + + try: + if not self._websocket: + await self._connect() + + try: + if not self._started: + await self.start_ttfb_metrics() + yield TTSStartedFrame() + self._started = True + self._cumulative_time = 0 + + await self._send_text(text) + await self.start_tts_usage_metrics(text) + except Exception as e: + logger.error(f"{self} error sending message: {e}") + yield TTSStoppedFrame() + await self._disconnect() + await self._connect() + return + yield None + except Exception as e: + logger.error(f"{self} exception: {e}") + + +class NeuphonicHttpTTSService(TTSService): + """Neuphonic Text-to-Speech service using HTTP streaming. + + Args: + api_key: Neuphonic API key + voice_id: ID of the voice to use + model: Neuphonic model to use (default: "neu_hq") + url: Base URL for the Neuphonic API (default: "https://api.neuphonic.com") + sample_rate: Sample rate for audio output (default: 22050Hz) + encoding: Audio encoding format (default: "pcm_linear") + params: Additional parameters for TTS generation including language and speed + **kwargs: Additional keyword arguments passed to the parent class + """ + + class InputParams(BaseModel): + language: Optional[Language] = Language.EN + speed: Optional[float] = 1.0 + + def __init__( + self, + *, + api_key: str, + voice_id: Optional[str] = None, + model: str = "neu_hq", + url: str = "https://api.neuphonic.com", + sample_rate: Optional[int] = 22050, + encoding: str = "pcm_linear", + params: InputParams = InputParams(), + **kwargs, + ): + super().__init__(sample_rate=sample_rate, **kwargs) + + self._api_key = api_key + self._url = url + self._settings = { + "lang_code": self.language_to_service_language(params.language), + "speed": params.speed, + "encoding": encoding, + "model": model, + "sampling_rate": sample_rate, + } + self.set_model_name(model) + self.set_voice(voice_id) + + def can_generate_metrics(self) -> bool: + return True + + async def start(self, frame: StartFrame): + await super().start(frame) + + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Neuphonic streaming API. + + Args: + text: The text to convert to speech + Yields: + Frames containing audio data and status information + """ + logger.debug(f"Generating TTS: [{text}]") + + client = Neuphonic(api_key=self._api_key, base_url=self._url.replace("https://", "")) + + sse = client.tts.AsyncSSEClient() + + try: + await self.start_ttfb_metrics() + response = sse.send( + text, TTSConfig(**self._settings, model=self.model_name, voice_id=self._voice_id) + ) + + await self.start_tts_usage_metrics(text) + yield TTSStartedFrame() + + async for message in response: + if response.status_code != 200: + logger.error(f"{self} error: {message.errors}") + yield ErrorFrame(error=f"Neuphonic API error: {message.errors}") + + await self.stop_ttfb_metrics() + yield TTSAudioRawFrame(message.data.audio, self.sample_rate, 1) + except Exception as e: + logger.error(f"Error in run_tts: {e}") + yield ErrorFrame(error=str(e)) + finally: + yield TTSStoppedFrame() From f84348296816920f5fef87aadbd228186249e02d Mon Sep 17 00:00:00 2001 From: macaki Date: Wed, 12 Mar 2025 11:26:43 -0600 Subject: [PATCH 012/132] [rime client] Sending over trailing space to help indicate end of utterance after a punctuation. --- src/pipecat/services/rime.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index 60083f55d..935d87de2 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -249,7 +249,9 @@ class RimeTTSService(AudioContextWordTTSService): async def flush_audio(self): if not self._context_id or not self._websocket: return + logger.trace(f"{self}: flushing audio") + await self._get_websocket().send(json.dumps({"text": " "})) self._context_id = None async def _receive_messages(self): From ead555eb4bca2c6db01beefc9ec2a097c9186109 Mon Sep 17 00:00:00 2001 From: Adnan Siddiquei Date: Wed, 12 Mar 2025 17:39:04 +0000 Subject: [PATCH 013/132] Corrected versions on pyproject.toml. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b5a870941..e698ab134 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ aws = [ "boto3~=1.35.99" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.3.1", "websockets~=13.1" ] -neuphonic = [ "pyneuphonic~=1.5.10", "websockets>=12.0,<14.0" ] +neuphonic = [ "pyneuphonic~=1.5.11", "websockets~=13.1" ] cerebras = [] deepseek = [] daily = [ "daily-python~=0.15.0" ] From effb5f6cd8fbd012a14a8d92cceda27ff0f2e35d Mon Sep 17 00:00:00 2001 From: macaki Date: Wed, 12 Mar 2025 11:57:25 -0600 Subject: [PATCH 014/132] added changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e730b70d..1ca6ed7e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated the default mode for `CartesiaTTSService` and `CartesiaHttpTTSService` to `sonic-2`. +### Fixed + +- Fixed an issue in `RimeTTSService` where the last line of text sent didn't result in an audio output being generated. + ## [0.0.58] - 2025-02-26 ### Added From 0b9c4b2255e81753e4a37ef405950f6d00d0894e Mon Sep 17 00:00:00 2001 From: Adnan Siddiquei Date: Wed, 12 Mar 2025 18:04:48 +0000 Subject: [PATCH 015/132] Fixed a couple of small bugs. --- src/pipecat/services/neuphonic.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/neuphonic.py b/src/pipecat/services/neuphonic.py index b14c325f8..f86280850 100644 --- a/src/pipecat/services/neuphonic.py +++ b/src/pipecat/services/neuphonic.py @@ -105,7 +105,6 @@ class NeuphonicTTSService(WordTTSService, WebsocketService): "lang_code": self.language_to_service_language(params.language), "speed": params.speed, "encoding": encoding, - "model": model, "sampling_rate": sample_rate, } self.set_model_name(model) @@ -315,7 +314,6 @@ class NeuphonicHttpTTSService(TTSService): "lang_code": self.language_to_service_language(params.language), "speed": params.speed, "encoding": encoding, - "model": model, "sampling_rate": sample_rate, } self.set_model_name(model) @@ -327,6 +325,9 @@ class NeuphonicHttpTTSService(TTSService): async def start(self, frame: StartFrame): await super().start(frame) + async def flush_audio(self): + pass + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Neuphonic streaming API. @@ -351,7 +352,7 @@ class NeuphonicHttpTTSService(TTSService): yield TTSStartedFrame() async for message in response: - if response.status_code != 200: + if message.status_code != 200: logger.error(f"{self} error: {message.errors}") yield ErrorFrame(error=f"Neuphonic API error: {message.errors}") From 5c9e33bc7a1ab24feb4ab3f0a265a15a05748ec8 Mon Sep 17 00:00:00 2001 From: macaki Date: Wed, 12 Mar 2025 12:20:18 -0600 Subject: [PATCH 016/132] formatting --- src/pipecat/services/rime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index 935d87de2..b2610b06c 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -249,7 +249,7 @@ class RimeTTSService(AudioContextWordTTSService): async def flush_audio(self): if not self._context_id or not self._websocket: return - + logger.trace(f"{self}: flushing audio") await self._get_websocket().send(json.dumps({"text": " "})) self._context_id = None From 08fb931ef6a295bd4b32a86b1e279f6f079374a8 Mon Sep 17 00:00:00 2001 From: Adnan Siddiquei Date: Thu, 13 Mar 2025 12:10:03 +0000 Subject: [PATCH 017/132] Swapped NEUPHONIC_API_TOKEN for NEUPHONIC_API_KEY. --- dot-env.template | 2 +- src/pipecat/services/neuphonic.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dot-env.template b/dot-env.template index 331eba79c..2da20fc0b 100644 --- a/dot-env.template +++ b/dot-env.template @@ -30,7 +30,7 @@ ELEVENLABS_API_KEY=... ELEVENLABS_VOICE_ID=... # Neuphonic -NEUPHONIC_API_TOKEN=... +NEUPHONIC_API_KEY=... # Fal FAL_KEY=... diff --git a/src/pipecat/services/neuphonic.py b/src/pipecat/services/neuphonic.py index f86280850..757d60c3e 100644 --- a/src/pipecat/services/neuphonic.py +++ b/src/pipecat/services/neuphonic.py @@ -38,7 +38,7 @@ try: except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error( - "In order to use Neuphonic, you need to `pip install pipecat-ai[neuphonic]`. Also, set `NEUPHONIC_API_TOKEN` environment variable." + "In order to use Neuphonic, you need to `pip install pipecat-ai[neuphonic]`. Also, set `NEUPHONIC_API_KEY` environment variable." ) raise Exception(f"Missing module: {e}") From 1bf964a667f2a756ef82a22088864adf205c00e8 Mon Sep 17 00:00:00 2001 From: Adnan Siddiquei Date: Thu, 13 Mar 2025 14:42:42 +0000 Subject: [PATCH 018/132] Added two examples on how to use Neuphonic as a TTS (07u). --- .../07u-interruptible-neuphonic-http.py | 102 ++++++++++++++++++ .../07u-interruptible-neuphonic.py | 102 ++++++++++++++++++ 2 files changed, 204 insertions(+) create mode 100644 examples/foundational/07u-interruptible-neuphonic-http.py create mode 100644 examples/foundational/07u-interruptible-neuphonic.py diff --git a/examples/foundational/07u-interruptible-neuphonic-http.py b/examples/foundational/07u-interruptible-neuphonic-http.py new file mode 100644 index 000000000..8377c3baf --- /dev/null +++ b/examples/foundational/07u-interruptible-neuphonic-http.py @@ -0,0 +1,102 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.neuphonic import NeuphonicHttpTTSService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = NeuphonicHttpTTSService( + api_key=os.getenv("NEUPHONIC_API_KEY"), + voice_id="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + await task.cancel() + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/foundational/07u-interruptible-neuphonic.py b/examples/foundational/07u-interruptible-neuphonic.py new file mode 100644 index 000000000..b5d666511 --- /dev/null +++ b/examples/foundational/07u-interruptible-neuphonic.py @@ -0,0 +1,102 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.neuphonic import NeuphonicTTSService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = NeuphonicTTSService( + api_key=os.getenv("NEUPHONIC_API_KEY"), + voice_id="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + await task.cancel() + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) From 25dcf7def6b4ab4eb4529e017d27e22ecb2e99aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 12 Mar 2025 19:33:40 -0700 Subject: [PATCH 019/132] PipelineTask: add on_frame_reached_upstream/on_frame_reached_downstream --- CHANGELOG.md | 7 ++++++- src/pipecat/pipeline/task.py | 22 ++++++++++++++++++++++ tests/test_pipeline.py | 36 +++++++++++++++++++++++++++++++++++- 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ca6ed7e6..776041481 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `on_frame_reached_upstream` and `on_frame_reached_downstream` event + handlers to `PipelineTask`. Those events will be called when a frame reaches + the beginning or end of the pipeline respectively. + - Added support for Chirp voices in `GoogleTTSService`. - Added a `flush_audio()` method to `FishTTSService` and `LmntTTSService`. @@ -80,7 +84,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- Fixed an issue in `RimeTTSService` where the last line of text sent didn't result in an audio output being generated. +- Fixed an issue in `RimeTTSService` where the last line of text sent didn't + result in an audio output being generated. ## [0.0.58] - 2025-02-26 diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 1c9d2dff9..f20ef9d1f 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -119,12 +119,25 @@ class PipelineTaskSink(FrameProcessor): class PipelineTask(BaseTask): """Manages the execution of a pipeline, handling frame processing and task lifecycle. + It has a couple of event handlers `on_frame_reached_upstream` and + `on_frame_reached_downstream` that are called when upstream frames or + downstream frames reach both ends of pipeline. + + @task.event_handler("on_frame_reached_upstream") + async def on_frame_reached_upstream(task, frame): + ... + + @task.event_handler("on_frame_reached_downstream") + async def on_frame_reached_downstream(task, frame): + ... + Args: pipeline: The pipeline to execute. params: Configuration parameters for the pipeline. observers: List of observers for monitoring pipeline execution. clock: Clock implementation for timing operations. check_dangling_tasks: Whether to check for processors' tasks finishing properly. + """ def __init__( @@ -177,6 +190,9 @@ class PipelineTask(BaseTask): self._observer = TaskObserver(observers=observers, task_manager=self._task_manager) + self._register_event_handler("on_frame_reached_upstream") + self._register_event_handler("on_frame_reached_downstream") + @property def params(self) -> PipelineParams: """Returns the pipeline parameters of this task.""" @@ -356,6 +372,9 @@ class PipelineTask(BaseTask): """ while True: frame = await self._up_queue.get() + + await self._call_event_handler("on_frame_reached_upstream", frame) + if isinstance(frame, EndTaskFrame): # Tell the task we should end nicely. await self.queue_frame(EndFrame()) @@ -383,6 +402,9 @@ class PipelineTask(BaseTask): """ while True: frame = await self._down_queue.get() + + await self._call_event_handler("on_frame_reached_downstream", frame) + if isinstance(frame, (EndFrame, StopFrame)): self._pipeline_end_event.set() elif isinstance(frame, HeartbeatFrame): diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 07abef48e..a375812b2 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -12,7 +12,7 @@ from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.filters.identity_filter import IdentityFilter -from pipecat.processors.frame_processor import FrameProcessor +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.tests.utils import HeartbeatsObserver, run_test @@ -94,6 +94,40 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): await task.run() assert task.has_finished() + async def test_task_event_handlers(self): + upstream_received = False + downstream_received = False + + identity = IdentityFilter() + pipeline = Pipeline([identity]) + task = PipelineTask(pipeline) + task.set_event_loop(asyncio.get_event_loop()) + + @task.event_handler("on_frame_reached_upstream") + async def on_frame_reached_upstream(task, frame): + nonlocal upstream_received + if isinstance(frame, TextFrame) and frame.text == "Hello Upstream!": + upstream_received = True + + @task.event_handler("on_frame_reached_downstream") + async def on_frame_reached_downstream(task, frame): + nonlocal downstream_received + if isinstance(frame, TextFrame) and frame.text == "Hello Downstream!": + downstream_received = True + await identity.push_frame( + TextFrame(text="Hello Upstream!"), FrameDirection.UPSTREAM + ) + + await task.queue_frame(TextFrame(text="Hello Downstream!")) + + try: + await asyncio.wait_for(task.run(), timeout=1.0) + except asyncio.TimeoutError: + pass + + assert upstream_received + assert downstream_received + async def test_task_heartbeats(self): heartbeats_counter = 0 From 87004937be423c741741acef2a2085cb467418f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 13 Mar 2025 10:47:47 -0700 Subject: [PATCH 020/132] services(ultravox): CHANGELOG, formatting and minor changes --- CHANGELOG.md | 3 + README.md | 4 +- .../07u-interruptible-ultravox.py | 15 +- pyproject.toml | 1 + src/pipecat/services/ultravox.py | 208 +++++++++--------- 5 files changed, 120 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 776041481..deda5de9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added new `UltravoxSTTService`. + (see https://github.com/fixie-ai/ultravox) + - Added `on_frame_reached_upstream` and `on_frame_reached_downstream` event handlers to `PipelineTask`. Those events will be called when a frame reaches the beginning or end of the pipeline respectively. diff --git a/README.md b/README.md index ec09c8f72..599412673 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,8 @@ pip install "pipecat-ai[option,...]" ### Available services | Category | Services | Install Command Example | -| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | +|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------| +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"` | | Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | | Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[google]"` | diff --git a/examples/foundational/07u-interruptible-ultravox.py b/examples/foundational/07u-interruptible-ultravox.py index 195eaf197..3ae4540f0 100644 --- a/examples/foundational/07u-interruptible-ultravox.py +++ b/examples/foundational/07u-interruptible-ultravox.py @@ -17,9 +17,9 @@ from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.transports.services.daily import DailyParams, DailyTransport -from pipecat.services.ultravox import UltravoxSTTService from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.ultravox import UltravoxSTTService +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) @@ -30,13 +30,14 @@ load_dotenv(override=True) logger.remove(0) logger.add(sys.stderr, level="DEBUG") -#Want to initialize the ultravox processor since it takes time to load the model and dont -#want to load it every time the pipeline is run +# Want to initialize the ultravox processor since it takes time to load the model and dont +# want to load it every time the pipeline is run ultravox_processor = UltravoxSTTService( model_size="fixie-ai/ultravox-v0_4_1-llama-3_1-8b", hf_token=os.getenv("HF_TOKEN"), ) + async def main(): async with aiohttp.ClientSession() as session: (room_url, token) = await configure(session) @@ -56,11 +57,9 @@ async def main(): tts = CartesiaTTSService( api_key=os.environ.get("CARTESIA_API_KEY"), - voice_id='97f4b8fb-f2fe-444b-bb9a-c109783a857a', - + voice_id="97f4b8fb-f2fe-444b-bb9a-c109783a857a", ) - pipeline = Pipeline( [ transport.input(), # Transport user input @@ -89,5 +88,3 @@ async def main(): if __name__ == "__main__": asyncio.run(main()) - - diff --git a/pyproject.toml b/pyproject.toml index dd58b1bfe..eab7b60f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ silero = [ "onnxruntime~=1.20.1" ] simli = [ "simli-ai~=0.1.10"] soundfile = [ "soundfile~=0.13.0" ] together = [] +ultravox = [ "transformers~=4.48.0", "vllm~=0.7.3" ] websocket = [ "websockets~=13.1", "fastapi~=0.115.6" ] whisper = [ "faster-whisper~=1.1.1" ] openrouter = [] diff --git a/src/pipecat/services/ultravox.py b/src/pipecat/services/ultravox.py index f39dc348f..40029e673 100644 --- a/src/pipecat/services/ultravox.py +++ b/src/pipecat/services/ultravox.py @@ -1,132 +1,140 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + """This module implements Ultravox speech-to-text with a locally-loaded model.""" import json -import time import os +import time +from typing import AsyncGenerator, List, Optional + import numpy as np -from enum import Enum -from typing import AsyncGenerator, Optional, List -from loguru import logger -from pydantic import BaseModel from huggingface_hub import login +from loguru import logger from pipecat.frames.frames import ( - Frame, AudioRawFrame, - TranscriptionFrame, - TextFrame, - StartFrame, - EndFrame, CancelFrame, + EndFrame, + ErrorFrame, + Frame, + StartFrame, + TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, - ErrorFrame ) -from pipecat.services.ai_services import AIService from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.ai_services import AIService from pipecat.utils.time import time_now_iso8601 try: - from vllm import SamplingParams, AsyncLLMEngine - from vllm.engine.arg_utils import AsyncEngineArgs from transformers import AutoTokenizer + from vllm import AsyncLLMEngine, SamplingParams + from vllm.engine.arg_utils import AsyncEngineArgs except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error("In order to use Ultravox, you need to `pip install pipecat-ai[ultravox]`.") raise Exception(f"Missing module: {e}") + class AudioBuffer: """Buffer to collect audio frames before processing. - + Attributes: frames: List of AudioRawFrames to process started_at: Timestamp when speech started is_processing: Flag to prevent concurrent processing """ + def __init__(self): self.frames: List[AudioRawFrame] = [] self.started_at: Optional[float] = None self.is_processing: bool = False + class UltravoxModel: """Model wrapper for the Ultravox multimodal model. - + This class handles loading and running the Ultravox model for speech-to-text. - + Args: model_name: The name or path of the Ultravox model to load - + Attributes: model_name: The name of the loaded model engine: The vLLM engine for model inference tokenizer: The tokenizer for the model stop_token_ids: Optional token IDs to stop generation """ + def __init__(self, model_name: str = "fixie-ai/ultravox-v0_4_1-llama-3_1-8b"): self.model_name = model_name self._initialize_engine() self._initialize_tokenizer() self.stop_token_ids = None - + def _initialize_engine(self): """Initialize the vLLM engine for inference.""" engine_args = AsyncEngineArgs( model=self.model_name, gpu_memory_utilization=0.9, max_model_len=8192, - trust_remote_code=True + trust_remote_code=True, ) self.engine = AsyncLLMEngine.from_engine_args(engine_args) - + def _initialize_tokenizer(self): """Initialize the tokenizer for the model.""" self.tokenizer = AutoTokenizer.from_pretrained(self.model_name) - + def format_prompt(self, messages: list): """Format chat messages into a prompt for the model. - + Args: messages: List of message dictionaries with 'role' and 'content' - + Returns: str: Formatted prompt string """ return self.tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=True + messages, tokenize=False, add_generation_prompt=True ) - async def generate(self, messages: list, temperature: float = 0.7, max_tokens: int = 100, audio: np.ndarray = None): + async def generate( + self, + messages: list, + temperature: float = 0.7, + max_tokens: int = 100, + audio: np.ndarray = None, + ): """Generate text from audio input using the model. - + Args: messages: List of message dictionaries temperature: Sampling temperature max_tokens: Maximum tokens to generate audio: Audio data as numpy array - + Yields: str: JSON chunks of the generated response """ sampling_params = SamplingParams( - temperature=temperature, - max_tokens=max_tokens, - stop_token_ids=self.stop_token_ids + temperature=temperature, max_tokens=max_tokens, stop_token_ids=self.stop_token_ids ) - - mm_data = { - "audio": audio - } + + mm_data = {"audio": audio} inputs = {"prompt": self.format_prompt(messages), "multi_modal_data": mm_data} results_generator = self.engine.generate(inputs, sampling_params, str(time.time())) - + previous_text = "" first_chunk = True async for output in results_generator: prompt_output = output.outputs - new_text = prompt_output[0].text[len(previous_text):] + new_text = prompt_output[0].text[len(previous_text) :] previous_text = prompt_output[0].text # Construct OpenAI-compatible chunk @@ -160,19 +168,20 @@ class UltravoxModel: yield json.dumps(chunk) + class UltravoxSTTService(AIService): """Service to transcribe audio using the Ultravox multimodal model. - + This service collects audio frames and processes them with Ultravox to generate text transcriptions. - + Args: model_size: The Ultravox model to use (ModelSize enum or string) hf_token: Hugging Face token for model access temperature: Sampling temperature for generation max_tokens: Maximum tokens to generate **kwargs: Additional arguments passed to AIService - + Attributes: model: The UltravoxModel instance buffer: Buffer to collect audio frames @@ -180,17 +189,18 @@ class UltravoxSTTService(AIService): max_tokens: Maximum tokens to generate _connection_active: Flag indicating if service is active """ + def __init__( self, *, - model_size: str = "fixie-ai/ultravox-v0_4_1-llama-3_1-8b", + model_size: str = "fixie-ai/ultravox-v0_4_1-llama-3_1-8b", hf_token: Optional[str] = None, temperature: float = 0.7, max_tokens: int = 100, **kwargs, ): super().__init__(**kwargs) - + # Authenticate with Hugging Face if token provided if hf_token: login(token=hf_token) @@ -198,22 +208,22 @@ class UltravoxSTTService(AIService): login(token=os.environ.get("HF_TOKEN")) else: logger.warning("No Hugging Face token provided. Model may not load correctly.") - + # Initialize model model_name = model_size if isinstance(model_size, str) else model_size.value - self.model = UltravoxModel(model_name=model_name) - + self._model = UltravoxModel(model_name=model_name) + # Initialize service state - self.buffer = AudioBuffer() - self.temperature = temperature - self.max_tokens = max_tokens + self._buffer = AudioBuffer() + self._temperature = temperature + self._max_tokens = max_tokens self._connection_active = False - + logger.info(f"Initialized UltravoxSTTService with model: {model_name}") - + def can_generate_metrics(self) -> bool: """Indicates whether this service can generate metrics. - + Returns: bool: True, as this service supports metric generation. """ @@ -221,7 +231,7 @@ class UltravoxSTTService(AIService): async def start(self, frame: StartFrame): """Handle service start. - + Args: frame: StartFrame that triggered this method """ @@ -231,7 +241,7 @@ class UltravoxSTTService(AIService): async def stop(self, frame: EndFrame): """Handle service stop. - + Args: frame: EndFrame that triggered this method """ @@ -241,20 +251,20 @@ class UltravoxSTTService(AIService): async def cancel(self, frame: CancelFrame): """Handle service cancellation. - + Args: frame: CancelFrame that triggered this method """ await super().cancel(frame) self._connection_active = False - self.buffer = AudioBuffer() + self._buffer = AudioBuffer() logger.info("UltravoxSTTService cancelled") async def process_frame(self, frame: Frame, direction: FrameDirection): """Process incoming frames. - + This method collects audio frames and processes them when speech ends. - + Args: frame: The frame to process direction: Direction of the frame (input/output) @@ -263,44 +273,44 @@ class UltravoxSTTService(AIService): if isinstance(frame, UserStartedSpeakingFrame): logger.info("Speech started") - self.buffer = AudioBuffer() - self.buffer.started_at = time.time() - - elif isinstance(frame, AudioRawFrame) and self.buffer.started_at is not None: - self.buffer.frames.append(frame) - + self._buffer = AudioBuffer() + self._buffer.started_at = time.time() + + elif isinstance(frame, AudioRawFrame) and self._buffer.started_at is not None: + self._buffer.frames.append(frame) + elif isinstance(frame, UserStoppedSpeakingFrame): - if self.buffer.frames and not self.buffer.is_processing: + if self._buffer.frames and not self._buffer.is_processing: logger.info("Speech ended, processing buffer...") await self.process_generator(self._process_audio_buffer()) return # Return early to avoid pushing None frame - + # Only push the original frame if we haven't processed audio if frame is not None: await self.push_frame(frame, direction) async def _process_audio_buffer(self) -> AsyncGenerator[Frame, None]: """Process collected audio frames with Ultravox. - + This method concatenates audio frames, processes them with the model, and yields the resulting text frames. - + Yields: Frame: TextFrame containing the transcribed text """ try: - self.buffer.is_processing = True - + self._buffer.is_processing = True + # Check if we have valid frames before processing - if not self.buffer.frames: + if not self._buffer.frames: logger.warning("No audio frames to process") yield ErrorFrame("No audio frames to process") return - + # Process audio frames audio_arrays = [] - for f in self.buffer.frames: - if hasattr(f, 'audio') and f.audio: + for f in self._buffer.frames: + if hasattr(f, "audio") and f.audio: # Handle bytes data - these are int16 PCM samples if isinstance(f.audio, bytes): try: @@ -319,75 +329,73 @@ class UltravoxSTTService(AIService): audio_arrays.append(f.audio.astype(np.int16)) else: audio_arrays.append(f.audio) - + # Only proceed if we have valid audio arrays if not audio_arrays: logger.warning("No valid audio data found in frames") yield ErrorFrame("No valid audio data found in frames") return - + # Concatenate audio frames - all should be int16 now audio_data = np.concatenate(audio_arrays) - + # Generate text using the model - if self.model: + if self._model: try: logger.info("Generating text from audio using model...") full_response = "" - + # Start metrics tracking await self.start_ttfb_metrics() await self.start_processing_metrics() - - async for response in self.model.generate( - messages=[{ - 'role': 'user', - 'content': "<|audio|>\n" - }], - temperature=self.temperature, - max_tokens=self.max_tokens, - audio=audio_data + + async for response in self._model.generate( + messages=[{"role": "user", "content": "<|audio|>\n"}], + temperature=self._temperature, + max_tokens=self._max_tokens, + audio=audio_data, ): # Stop TTFB metrics after first response await self.stop_ttfb_metrics() - + chunk = json.loads(response) if "choices" in chunk and len(chunk["choices"]) > 0: delta = chunk["choices"][0]["delta"] if "content" in delta: new_text = delta["content"] full_response += new_text - + # Stop processing metrics after completion await self.stop_processing_metrics() - + logger.info(f"Generated text: {full_response}") - + # Create a transcription frame with the generated text transcription = full_response.strip() if transcription: yield TranscriptionFrame( + user_id="", text=transcription, - interim_text="", - timestamp=time_now_iso8601() + timestamp=time_now_iso8601(), ) else: logger.warning("Empty transcription result") yield ErrorFrame("Empty transcription result") - + except Exception as e: logger.error(f"Error generating text from model: {e}") yield ErrorFrame(f"Error generating text: {str(e)}") else: logger.warning("No model available for text generation") yield ErrorFrame("No model available for text generation") - + except Exception as e: logger.error(f"Error processing audio buffer: {e}") import traceback + logger.error(traceback.format_exc()) yield ErrorFrame(f"Error processing audio: {str(e)}") finally: - self.buffer.is_processing = False - self.buffer.frames = [] - self.buffer.started_at = None + self._buffer.is_processing = False + self._buffer.frames = [] + self._buffer.started_at = None From c9a31ea5135b0f381ba1995ff2e735ebab18b939 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Thu, 13 Mar 2025 14:35:47 -0700 Subject: [PATCH 021/132] fix for 26-gemini-multimodal-live.py --- examples/foundational/26-gemini-multimodal-live.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/foundational/26-gemini-multimodal-live.py b/examples/foundational/26-gemini-multimodal-live.py index 71d41304b..c26d6ad2f 100644 --- a/examples/foundational/26-gemini-multimodal-live.py +++ b/examples/foundational/26-gemini-multimodal-live.py @@ -77,7 +77,7 @@ async def main(): LLMMessagesAppendFrame( messages=[ { - "role": "assistant", + "role": "user", "content": "Greet the user.", } ] From 6eb3a8409fe9b8bba378041e956bbe654e768dc0 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 13 Mar 2025 18:42:19 -0400 Subject: [PATCH 022/132] README: Add Parakeet and FastPitch --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 599412673..822c75aee 100644 --- a/README.md +++ b/README.md @@ -56,10 +56,10 @@ pip install "pipecat-ai[option,...]" ### Available services | Category | Services | Install Command Example | -|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------| -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Parakeet (NVIDIA)](https://docs.pipecat.ai/server/services/stt/parakeet), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"` | -| Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | +| Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [FastPitch (NVIDIA)](https://docs.pipecat.ai/server/services/tts/fastpitch), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | | Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[google]"` | | Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | `pip install "pipecat-ai[daily]"` | | Video | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | From 1b3b4ee04a837eb8a38506d28af6b50176319e54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 13 Mar 2025 18:39:11 -0700 Subject: [PATCH 023/132] PipelineTask: only call event handlers if a filter is matched --- CHANGELOG.md | 5 +++- src/pipecat/pipeline/task.py | 45 ++++++++++++++++++++++++++++++++---- tests/test_pipeline.py | 2 ++ 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index deda5de9d..4feca2771 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `on_frame_reached_upstream` and `on_frame_reached_downstream` event handlers to `PipelineTask`. Those events will be called when a frame reaches - the beginning or end of the pipeline respectively. + the beginning or end of the pipeline respectively. Note that by default, the + event handlers will not be called unless a filter is set with + `PipelineTask.set_reached_upstream_filter()` or + `PipelineTask.set_reached_downstream_filter()`. - Added support for Chirp voices in `GoogleTTSService`. diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index f20ef9d1f..c10b33189 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -5,7 +5,7 @@ # import asyncio -from typing import Any, AsyncIterable, Dict, Iterable, List, Optional +from typing import Any, AsyncIterable, Dict, Iterable, List, Optional, Tuple, Type from loguru import logger from pydantic import BaseModel, ConfigDict @@ -121,7 +121,9 @@ class PipelineTask(BaseTask): It has a couple of event handlers `on_frame_reached_upstream` and `on_frame_reached_downstream` that are called when upstream frames or - downstream frames reach both ends of pipeline. + downstream frames reach both ends of pipeline. By default, the events + handlers will not be called unless some filters are set using + `set_reached_upstream_filter` and `set_reached_downstream_filter`. @task.event_handler("on_frame_reached_upstream") async def on_frame_reached_upstream(task, frame): @@ -180,16 +182,35 @@ class PipelineTask(BaseTask): # StopFrame) has been received in the down queue. self._pipeline_end_event = asyncio.Event() + # This is a source processor that we connect to the provided + # pipeline. This source processor allows up to receive and react to + # upstream frames. self._source = PipelineTaskSource(self._up_queue) self._source.link(pipeline) + # This is a sink processor that we connect to the provided + # pipeline. This sink processor allows up to receive and react to + # downstream frames. self._sink = PipelineTaskSink(self._down_queue) pipeline.link(self._sink) + # This task maneger will handle all the asyncio tasks created by this + # PipelineTask and its frame processors. self._task_manager = task_manager or TaskManager() + # The task observer acts as a proxy to the provided observers. This way, + # we only need to pass a single observer (using the StartFrame) which + # then just acts as a proxy. self._observer = TaskObserver(observers=observers, task_manager=self._task_manager) + # These events can be used to check which frames make it to the source + # or sink processors. Instead of calling the event handlers for every + # frame the user needs to specify which events they are interested + # in. This is mainly for efficiency reason because each event handler + # creates a task and most likely you only care about one or two frame + # types. + self._reached_upstream_types: Tuple[Type[Frame], ...] = () + self._reached_downstream_types: Tuple[Type[Frame], ...] = () self._register_event_handler("on_frame_reached_upstream") self._register_event_handler("on_frame_reached_downstream") @@ -201,6 +222,20 @@ class PipelineTask(BaseTask): def set_event_loop(self, loop: asyncio.AbstractEventLoop): self._task_manager.set_event_loop(loop) + def set_reached_upstream_filter(self, types: Tuple[Type[Frame], ...]): + """Sets which frames will be checked before calling the + on_frame_reached_upstream event handler. + + """ + self._reached_upstream_types = types + + def set_reached_downstream_filter(self, types: Tuple[Type[Frame], ...]): + """Sets which frames will be checked before calling the + on_frame_reached_downstream event handler. + + """ + self._reached_downstream_types = types + def has_finished(self) -> bool: """Indicates whether the tasks has finished. That is, all processors have stopped. @@ -373,7 +408,8 @@ class PipelineTask(BaseTask): while True: frame = await self._up_queue.get() - await self._call_event_handler("on_frame_reached_upstream", frame) + if isinstance(frame, self._reached_upstream_types): + await self._call_event_handler("on_frame_reached_upstream", frame) if isinstance(frame, EndTaskFrame): # Tell the task we should end nicely. @@ -403,7 +439,8 @@ class PipelineTask(BaseTask): while True: frame = await self._down_queue.get() - await self._call_event_handler("on_frame_reached_downstream", frame) + if isinstance(frame, self._reached_downstream_types): + await self._call_event_handler("on_frame_reached_downstream", frame) if isinstance(frame, (EndFrame, StopFrame)): self._pipeline_end_event.set() diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index a375812b2..c3811a672 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -102,6 +102,8 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): pipeline = Pipeline([identity]) task = PipelineTask(pipeline) task.set_event_loop(asyncio.get_event_loop()) + task.set_reached_upstream_filter((TextFrame,)) + task.set_reached_downstream_filter((TextFrame,)) @task.event_handler("on_frame_reached_upstream") async def on_frame_reached_upstream(task, frame): From 7dec8431e16a6675d869c057ef7e4181a23a2ea3 Mon Sep 17 00:00:00 2001 From: Adnan Siddiquei Date: Fri, 14 Mar 2025 10:52:13 +0000 Subject: [PATCH 024/132] Review comments by aconchillo. --- pyproject.toml | 2 +- src/pipecat/services/neuphonic.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e698ab134..1b324c3a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ aws = [ "boto3~=1.35.99" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.3.1", "websockets~=13.1" ] -neuphonic = [ "pyneuphonic~=1.5.11", "websockets~=13.1" ] +neuphonic = [ "pyneuphonic~=1.5.12", "websockets~=13.1" ] cerebras = [] deepseek = [] daily = [ "daily-python~=0.15.0" ] diff --git a/src/pipecat/services/neuphonic.py b/src/pipecat/services/neuphonic.py index 757d60c3e..c0d35fb40 100644 --- a/src/pipecat/services/neuphonic.py +++ b/src/pipecat/services/neuphonic.py @@ -27,8 +27,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.ai_services import TTSService, WordTTSService -from pipecat.services.websocket_service import WebsocketService +from pipecat.services.ai_services import InterruptibleTTSService, TTSService from pipecat.transcriptions.language import Language # See .env.example for Neuphonic configuration needed @@ -71,7 +70,7 @@ def language_to_neuphonic_lang_code(language: Language) -> Optional[str]: return result -class NeuphonicTTSService(WordTTSService, WebsocketService): +class NeuphonicTTSService(InterruptibleTTSService): class InputParams(BaseModel): language: Optional[Language] = Language.EN speed: Optional[float] = 1.0 @@ -88,7 +87,7 @@ class NeuphonicTTSService(WordTTSService, WebsocketService): params: InputParams = InputParams(), **kwargs, ): - WordTTSService.__init__( + super().__init__( self, aggregate_sentences=True, push_text_frames=False, @@ -97,7 +96,6 @@ class NeuphonicTTSService(WordTTSService, WebsocketService): sample_rate=sample_rate, **kwargs, ) - WebsocketService.__init__(self) self._api_key = api_key self._url = url @@ -232,7 +230,6 @@ class NeuphonicTTSService(WordTTSService, WebsocketService): msg = json.loads(message) if msg.get("data", {}).get("audio") is not None: await self.stop_ttfb_metrics() - self.start_word_timestamps() audio = base64.b64decode(msg["data"]["audio"]) frame = TTSAudioRawFrame(audio, self.sample_rate, 1) From 11b13d053be82eca507f79284692cb8fb1d4ecb2 Mon Sep 17 00:00:00 2001 From: Adnan Siddiquei Date: Fri, 14 Mar 2025 11:17:22 +0000 Subject: [PATCH 025/132] Fixed a bug from previous commit. Removed the concept of model from Neuphonic. --- pyproject.toml | 2 +- src/pipecat/services/neuphonic.py | 25 +------------------------ 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1b324c3a5..f87aa7572 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ aws = [ "boto3~=1.35.99" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.3.1", "websockets~=13.1" ] -neuphonic = [ "pyneuphonic~=1.5.12", "websockets~=13.1" ] +neuphonic = [ "pyneuphonic~=1.5.13", "websockets~=13.1" ] cerebras = [] deepseek = [] daily = [ "daily-python~=0.15.0" ] diff --git a/src/pipecat/services/neuphonic.py b/src/pipecat/services/neuphonic.py index c0d35fb40..a2ea98f42 100644 --- a/src/pipecat/services/neuphonic.py +++ b/src/pipecat/services/neuphonic.py @@ -41,12 +41,6 @@ except ModuleNotFoundError as e: ) raise Exception(f"Missing module: {e}") -# Models that support language codes -NEUPHONIC_MULTILINGUAL_MODELS = { - "neu_fast", - "neu_hq", -} - def language_to_neuphonic_lang_code(language: Language) -> Optional[str]: BASE_LANGUAGES = { @@ -80,7 +74,6 @@ class NeuphonicTTSService(InterruptibleTTSService): *, api_key: str, voice_id: Optional[str] = None, - model: str = "neu_hq", url: str = "wss://api.neuphonic.com", sample_rate: Optional[int] = 22050, encoding: str = "pcm_linear", @@ -88,7 +81,6 @@ class NeuphonicTTSService(InterruptibleTTSService): **kwargs, ): super().__init__( - self, aggregate_sentences=True, push_text_frames=False, push_stop_frames=True, @@ -105,7 +97,6 @@ class NeuphonicTTSService(InterruptibleTTSService): "encoding": encoding, "sampling_rate": sample_rate, } - self.set_model_name(model) self.set_voice(voice_id) # Indicates if we have sent TTSStartedFrame. It will reset to False when @@ -119,12 +110,6 @@ class NeuphonicTTSService(InterruptibleTTSService): def language_to_service_language(self, language: Language) -> Optional[str]: return language_to_neuphonic_lang_code(language) - async def set_model(self, model: str): - await super().set_model(model) - logger.info(f"Switching TTS model to: [{model}]") - await self._disconnect() - await self._connect() - async def _update_settings(self, settings: Mapping[str, Any]): if "voice_id" in settings: self.set_voice(settings["voice_id"]) @@ -155,8 +140,6 @@ class NeuphonicTTSService(InterruptibleTTSService): await super().push_frame(frame, direction) if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): self._started = False - if isinstance(frame, TTSStoppedFrame): - await self.add_word_timestamps([("LLMFullResponseEndFrame", 0), ("Reset", 0)]) async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -195,7 +178,6 @@ class NeuphonicTTSService(InterruptibleTTSService): tts_config = { **self._settings, "voice_id": self._voice_id, - "model": self.model_name, } query_params = [f"api_key={self._api_key}"] @@ -279,7 +261,6 @@ class NeuphonicHttpTTSService(TTSService): Args: api_key: Neuphonic API key voice_id: ID of the voice to use - model: Neuphonic model to use (default: "neu_hq") url: Base URL for the Neuphonic API (default: "https://api.neuphonic.com") sample_rate: Sample rate for audio output (default: 22050Hz) encoding: Audio encoding format (default: "pcm_linear") @@ -296,7 +277,6 @@ class NeuphonicHttpTTSService(TTSService): *, api_key: str, voice_id: Optional[str] = None, - model: str = "neu_hq", url: str = "https://api.neuphonic.com", sample_rate: Optional[int] = 22050, encoding: str = "pcm_linear", @@ -313,7 +293,6 @@ class NeuphonicHttpTTSService(TTSService): "encoding": encoding, "sampling_rate": sample_rate, } - self.set_model_name(model) self.set_voice(voice_id) def can_generate_metrics(self) -> bool: @@ -341,9 +320,7 @@ class NeuphonicHttpTTSService(TTSService): try: await self.start_ttfb_metrics() - response = sse.send( - text, TTSConfig(**self._settings, model=self.model_name, voice_id=self._voice_id) - ) + response = sse.send(text, TTSConfig(**self._settings, voice_id=self._voice_id)) await self.start_tts_usage_metrics(text) yield TTSStartedFrame() From f8610a69a5a9af67d8228de09630650bb9ca07f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 13 Mar 2025 19:41:33 -0700 Subject: [PATCH 026/132] introduce text aggregators --- CHANGELOG.md | 5 ++ src/pipecat/services/ai_services.py | 20 +++---- .../utils/text/base_text_aggregator.py | 57 +++++++++++++++++++ .../utils/text/simple_text_aggregator.py | 42 ++++++++++++++ tests/test_simple_text_aggregator.py | 29 ++++++++++ 5 files changed, 143 insertions(+), 10 deletions(-) create mode 100644 src/pipecat/utils/text/base_text_aggregator.py create mode 100644 src/pipecat/utils/text/simple_text_aggregator.py create mode 100644 tests/test_simple_text_aggregator.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4feca2771..a48fa9c3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added new `BaseTextAggregator`. Text aggregators are used by the TTS service + to aggregate LLM tokens and decide when the aggregated text should be pushed + to the TTS service. It also allows for the text to be manipulated while it's + being aggregated. + - Added new `UltravoxSTTService`. (see https://github.com/fixie-ai/ultravox) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 8533a23fc..1701f9829 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -45,8 +45,9 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.websocket_service import WebsocketService from pipecat.transcriptions.language import Language -from pipecat.utils.string import match_endofsentence +from pipecat.utils.text.base_text_aggregator import BaseTextAggregator from pipecat.utils.text.base_text_filter import BaseTextFilter +from pipecat.utils.text.simple_text_aggregator import SimpleTextAggregator from pipecat.utils.time import seconds_to_nanoseconds @@ -237,6 +238,9 @@ class TTSService(AIService): pause_frame_processing: bool = False, # TTS output sample rate sample_rate: Optional[int] = None, + # Text aggregator to aggregate incoming tokens and decide when to push to the TTS. + text_aggregator: Optional[BaseTextAggregator] = None, + # Text filter executed after text has been aggregated. text_filter: Optional[BaseTextFilter] = None, **kwargs, ): @@ -252,12 +256,12 @@ class TTSService(AIService): self._sample_rate = 0 self._voice_id: str = "" self._settings: Dict[str, Any] = {} + self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() self._text_filter: Optional[BaseTextFilter] = text_filter self._stop_frame_task: Optional[asyncio.Task] = None self._stop_frame_queue: asyncio.Queue = asyncio.Queue() - self._current_sentence: str = "" self._processing_text: bool = False @property @@ -336,8 +340,8 @@ class TTSService(AIService): # pause to avoid audio overlapping. await self._maybe_pause_frame_processing() - sentence = self._current_sentence - self._current_sentence = "" + sentence = self._text_aggregator.text + self._text_aggregator.reset() self._processing_text = False await self._push_tts_frames(sentence) if isinstance(frame, LLMFullResponseEndFrame): @@ -382,8 +386,8 @@ class TTSService(AIService): await self._stop_frame_queue.put(frame) async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): - self._current_sentence = "" self._processing_text = False + self._text_aggregator.handle_interruption() if self._text_filter: self._text_filter.handle_interruption() @@ -400,11 +404,7 @@ class TTSService(AIService): if not self._aggregate_sentences: text = frame.text else: - self._current_sentence += frame.text - eos_end_marker = match_endofsentence(self._current_sentence) - if eos_end_marker: - text = self._current_sentence[:eos_end_marker] - self._current_sentence = self._current_sentence[eos_end_marker:] + text = self._text_aggregator.aggregate(frame.text) if text: await self._push_tts_frames(text) diff --git a/src/pipecat/utils/text/base_text_aggregator.py b/src/pipecat/utils/text/base_text_aggregator.py new file mode 100644 index 000000000..452e5598e --- /dev/null +++ b/src/pipecat/utils/text/base_text_aggregator.py @@ -0,0 +1,57 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from abc import ABC, abstractmethod +from typing import Optional + + +class BaseTextAggregator(ABC): + """This is the base class for text aggregators. Text aggregators are usually + used by the TTS service to aggregate LLM tokens and decide when the + aggregated text should be pushed to the TTS service. + + Text aggregators can also be used to manipulate text while it's being + aggregated (e.g. reasoning blocks can be removed). + + """ + + @property + @abstractmethod + def text(self) -> str: + """Returns the currently aggregated text.""" + pass + + @abstractmethod + def aggregate(self, text: str) -> Optional[str]: + """Aggregates the specified text with the currently accumulated text. + + This method should be implemented to define how the new text contributes + to the aggregation process. It returns the updated aggregated text if + it's ready to be processed, or None otherwise. + + Args: + text (str): The text to be aggregated. + + Returns: + Optional[str]: The updated aggregated text or None if aggregated + text is not ready. + + """ + pass + + @abstractmethod + def handle_interruption(self): + """Handles interruptions. When an interruption occurs it is possible + that we might want to discard the aggregated text or do some internal + modifications to the aggregated text. + + """ + pass + + @abstractmethod + def reset(self): + """Clears the internally aggregated text.""" + pass diff --git a/src/pipecat/utils/text/simple_text_aggregator.py b/src/pipecat/utils/text/simple_text_aggregator.py new file mode 100644 index 000000000..9022fc25a --- /dev/null +++ b/src/pipecat/utils/text/simple_text_aggregator.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from typing import Optional + +from pipecat.utils.string import match_endofsentence +from pipecat.utils.text.base_text_aggregator import BaseTextAggregator + + +class SimpleTextAggregator(BaseTextAggregator): + """This is a simple text aggregator. It aggregates text until an end of + sentence is found. + + """ + + def __init__(self): + self._text = "" + + @property + def text(self) -> str: + return self._text + + def aggregate(self, text: str) -> Optional[str]: + result: Optional[str] = None + + self._text += text + + eos_end_marker = match_endofsentence(self._text) + if eos_end_marker: + result = self._text[:eos_end_marker] + self._text = self._text[eos_end_marker:] + + return result + + def handle_interruption(self): + self._text = "" + + def reset(self): + self._text = "" diff --git a/tests/test_simple_text_aggregator.py b/tests/test_simple_text_aggregator.py new file mode 100644 index 000000000..10c4c6a88 --- /dev/null +++ b/tests/test_simple_text_aggregator.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2024-2025 Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import unittest + +from pipecat.utils.text.simple_text_aggregator import SimpleTextAggregator + + +class TestSimpleTextAggregator(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.aggregator = SimpleTextAggregator() + + async def test_reset_aggregations(self): + assert self.aggregator.aggregate("Hello ") == None + assert self.aggregator.text == "Hello " + self.aggregator.reset() + assert self.aggregator.text == "" + + async def test_simple_sentence(self): + assert self.aggregator.aggregate("Hello ") == None + assert self.aggregator.aggregate("Pipecat!") == "Hello Pipecat!" + assert self.aggregator.text == "" + + async def test_multiple_sentences(self): + assert self.aggregator.aggregate("Hello Pipecat! How are ") == "Hello Pipecat!" + assert self.aggregator.aggregate("you?") == " How are you?" From b632d71465a8821e541b3fcc0ba1deaf13488b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 13 Mar 2025 19:41:47 -0700 Subject: [PATCH 027/132] TTSService: flush_audio() should be in the base class --- src/pipecat/services/ai_services.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 1701f9829..3fe33d69e 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -285,6 +285,9 @@ class TTSService(AIService): async def update_setting(self, key: str, value: Any): pass + async def flush_audio(self): + pass + async def start(self, frame: StartFrame): await super().start(frame) self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate @@ -535,9 +538,6 @@ class WebsocketTTSService(TTSService, WebsocketService): TTSService.__init__(self, **kwargs) WebsocketService.__init__(self) - async def flush_audio(self): - pass - class InterruptibleTTSService(WebsocketTTSService): """This is a base class for websocket-based TTS services that don't support From 16d7df1c9f6bd8e9128346dd1091943c1ce464e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 13 Mar 2025 18:53:24 -0700 Subject: [PATCH 028/132] removed most deprecations --- CHANGELOG.md | 16 + src/pipecat/audio/utils.py | 17 -- .../processors/filters/stt_mute_filter.py | 13 +- src/pipecat/processors/frameworks/rtvi.py | 273 ------------------ src/pipecat/services/aws.py | 13 - src/pipecat/transports/services/daily.py | 15 +- src/pipecat/vad/__init__.py | 0 src/pipecat/vad/silero.py | 16 - src/pipecat/vad/vad_analyzer.py | 15 - 9 files changed, 18 insertions(+), 360 deletions(-) delete mode 100644 src/pipecat/vad/__init__.py delete mode 100644 src/pipecat/vad/silero.py delete mode 100644 src/pipecat/vad/vad_analyzer.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a48fa9c3b..3d56a323f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated the default mode for `CartesiaTTSService` and `CartesiaHttpTTSService` to `sonic-2`. +### Removed + +- Removed deprecated `audio.resample_audio()`, use `create_default_resampler()` + instead. + +- Removed deprecated`stt_service` parameter from `STTMuteFilter`. + +- Removed deprecated RTVI processors, use an `RTVIObserver` instead. + +- Removed deprecated `AWSTTSService`, use `PollyTTSService` instead. + +- Removed deprecated field `tier` from `DailyTranscriptionSettings`, use `model` + instead. + +- Removed deprecated `pipecat.vad` package, use `pipecat.audio.vad` instead. + ### Fixed - Fixed an issue in `RimeTTSService` where the last line of text sent didn't diff --git a/src/pipecat/audio/utils.py b/src/pipecat/audio/utils.py index 7c4c25fcc..1f7db648f 100644 --- a/src/pipecat/audio/utils.py +++ b/src/pipecat/audio/utils.py @@ -18,23 +18,6 @@ def create_default_resampler(**kwargs) -> BaseAudioResampler: return SOXRAudioResampler(**kwargs) -def resample_audio(audio: bytes, original_rate: int, target_rate: int) -> bytes: - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'resample_audio()' is deprecated, use 'create_default_resampler()' instead.", - DeprecationWarning, - ) - - if original_rate == target_rate: - return audio - audio_data = np.frombuffer(audio, dtype=np.int16) - resampled_audio = soxr.resample(audio_data, original_rate, target_rate) - return resampled_audio.astype(np.int16).tobytes() - - def mix_audio(audio1: bytes, audio2: bytes) -> bytes: data1 = np.frombuffer(audio1, dtype=np.int16) data2 = np.frombuffer(audio2, dtype=np.int16) diff --git a/src/pipecat/processors/filters/stt_mute_filter.py b/src/pipecat/processors/filters/stt_mute_filter.py index c74e495ab..cce087f22 100644 --- a/src/pipecat/processors/filters/stt_mute_filter.py +++ b/src/pipecat/processors/filters/stt_mute_filter.py @@ -92,20 +92,9 @@ class STTMuteFilter(FrameProcessor): **kwargs: Additional arguments passed to parent class """ - def __init__( - self, *, config: STTMuteConfig, stt_service: Optional[FrameProcessor] = None, **kwargs - ): + def __init__(self, *, config: STTMuteConfig, **kwargs): super().__init__(**kwargs) self._config = config - if stt_service is not None: - import warnings - - warnings.warn( - "The stt_service parameter is deprecated and will be removed in a future version. " - "STTMuteFilter now manages mute state internally.", - DeprecationWarning, - stacklevel=2, - ) self._first_speech_handled = False self._bot_is_speaking = False self._function_call_in_progress = False diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index dac903862..29747a582 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -391,267 +391,6 @@ class RTVIServerMessageFrame(SystemFrame): return f"{self.name}(data: {self.data})" -class RTVIFrameProcessor(FrameProcessor): - def __init__(self, direction: FrameDirection = FrameDirection.DOWNSTREAM, **kwargs): - super().__init__(**kwargs) - self._direction = direction - - async def _push_transport_message_urgent(self, model: BaseModel, exclude_none: bool = True): - frame = TransportMessageUrgentFrame(message=model.model_dump(exclude_none=exclude_none)) - await self.push_frame(frame, self._direction) - - -class RTVISpeakingProcessor(RTVIFrameProcessor): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'RTVISpeakingProcessor' is deprecated, use an 'RTVIObserver' instead.", - DeprecationWarning, - ) - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - await self.push_frame(frame, direction) - - if isinstance(frame, (UserStartedSpeakingFrame, UserStoppedSpeakingFrame)): - await self._handle_interruptions(frame) - elif isinstance(frame, (BotStartedSpeakingFrame, BotStoppedSpeakingFrame)): - await self._handle_bot_speaking(frame) - - async def _handle_interruptions(self, frame: Frame): - message = None - if isinstance(frame, UserStartedSpeakingFrame): - message = RTVIUserStartedSpeakingMessage() - elif isinstance(frame, UserStoppedSpeakingFrame): - message = RTVIUserStoppedSpeakingMessage() - - if message: - await self._push_transport_message_urgent(message) - - async def _handle_bot_speaking(self, frame: Frame): - message = None - if isinstance(frame, BotStartedSpeakingFrame): - message = RTVIBotStartedSpeakingMessage() - elif isinstance(frame, BotStoppedSpeakingFrame): - message = RTVIBotStoppedSpeakingMessage() - - if message: - await self._push_transport_message_urgent(message) - - -class RTVIUserTranscriptionProcessor(RTVIFrameProcessor): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'RTVIUserTranscriptionProcessor' is deprecated, use an 'RTVIObserver' instead.", - DeprecationWarning, - ) - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - await self.push_frame(frame, direction) - - if isinstance(frame, (TranscriptionFrame, InterimTranscriptionFrame)): - await self._handle_user_transcriptions(frame) - - async def _handle_user_transcriptions(self, frame: Frame): - message = None - if isinstance(frame, TranscriptionFrame): - message = RTVIUserTranscriptionMessage( - data=RTVIUserTranscriptionMessageData( - text=frame.text, user_id=frame.user_id, timestamp=frame.timestamp, final=True - ) - ) - elif isinstance(frame, InterimTranscriptionFrame): - message = RTVIUserTranscriptionMessage( - data=RTVIUserTranscriptionMessageData( - text=frame.text, user_id=frame.user_id, timestamp=frame.timestamp, final=False - ) - ) - - if message: - await self._push_transport_message_urgent(message) - - -class RTVIUserLLMTextProcessor(RTVIFrameProcessor): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'RTVIUserLLMTextProcessor' is deprecated, use an 'RTVIObserver' instead.", - DeprecationWarning, - ) - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - await self.push_frame(frame, direction) - - if isinstance(frame, OpenAILLMContextFrame): - await self._handle_context(frame) - - async def _handle_context(self, frame: OpenAILLMContextFrame): - messages = frame.context.messages - if len(messages) > 0: - message = messages[-1] - if message["role"] == "user": - content = message["content"] - if isinstance(content, list): - text = " ".join(item["text"] for item in content if "text" in item) - else: - text = content - rtvi_message = RTVIUserLLMTextMessage(data=RTVITextMessageData(text=text)) - await self._push_transport_message_urgent(rtvi_message) - - -class RTVIBotTranscriptionProcessor(RTVIFrameProcessor): - def __init__(self): - super().__init__() - self._aggregation = "" - - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'RTVIBotTranscriptionProcessor' is deprecated, use an 'RTVIObserver' instead.", - DeprecationWarning, - ) - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - await self.push_frame(frame, direction) - - if isinstance(frame, UserStartedSpeakingFrame): - await self._push_aggregation() - elif isinstance(frame, LLMTextFrame): - self._aggregation += frame.text - if match_endofsentence(self._aggregation): - await self._push_aggregation() - - async def _push_aggregation(self): - if len(self._aggregation) > 0: - message = RTVIBotTranscriptionMessage(data=RTVITextMessageData(text=self._aggregation)) - await self._push_transport_message_urgent(message) - self._aggregation = "" - - -class RTVIBotLLMProcessor(RTVIFrameProcessor): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'RTVIBotLLMProcessor' is deprecated, use an 'RTVIObserver' instead.", - DeprecationWarning, - ) - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - await self.push_frame(frame, direction) - - if isinstance(frame, LLMFullResponseStartFrame): - await self._push_transport_message_urgent(RTVIBotLLMStartedMessage()) - elif isinstance(frame, LLMFullResponseEndFrame): - await self._push_transport_message_urgent(RTVIBotLLMStoppedMessage()) - elif isinstance(frame, LLMTextFrame): - message = RTVIBotLLMTextMessage(data=RTVITextMessageData(text=frame.text)) - await self._push_transport_message_urgent(message) - - -class RTVIBotTTSProcessor(RTVIFrameProcessor): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'RTVIBotTTSProcessor' is deprecated, use an 'RTVIObserver' instead.", - DeprecationWarning, - ) - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - await self.push_frame(frame, direction) - - if isinstance(frame, TTSStartedFrame): - await self._push_transport_message_urgent(RTVIBotTTSStartedMessage()) - elif isinstance(frame, TTSStoppedFrame): - await self._push_transport_message_urgent(RTVIBotTTSStoppedMessage()) - elif isinstance(frame, TTSTextFrame): - message = RTVIBotTTSTextMessage(data=RTVITextMessageData(text=frame.text)) - await self._push_transport_message_urgent(message) - - -class RTVIMetricsProcessor(RTVIFrameProcessor): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'RTVIMetricsProcessor' is deprecated, use an 'RTVIObserver' instead.", - DeprecationWarning, - ) - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - await self.push_frame(frame, direction) - - if isinstance(frame, MetricsFrame): - await self._handle_metrics(frame) - - async def _handle_metrics(self, frame: MetricsFrame): - metrics = {} - for d in frame.data: - if isinstance(d, TTFBMetricsData): - if "ttfb" not in metrics: - metrics["ttfb"] = [] - metrics["ttfb"].append(d.model_dump(exclude_none=True)) - elif isinstance(d, ProcessingMetricsData): - if "processing" not in metrics: - metrics["processing"] = [] - metrics["processing"].append(d.model_dump(exclude_none=True)) - elif isinstance(d, LLMUsageMetricsData): - if "tokens" not in metrics: - metrics["tokens"] = [] - metrics["tokens"].append(d.value.model_dump(exclude_none=True)) - elif isinstance(d, TTSUsageMetricsData): - if "characters" not in metrics: - metrics["characters"] = [] - metrics["characters"].append(d.model_dump(exclude_none=True)) - - message = RTVIMetricsMessage(data=metrics) - await self._push_transport_message_urgent(message) - - class RTVIObserver(BaseObserver): """Pipeline frame observer for RTVI server message handling. @@ -876,18 +615,6 @@ class RTVIProcessor(FrameProcessor): self._input_transport = input_transport self._input_transport.enable_audio_in_stream_on_start(False) - def observer(self) -> RTVIObserver: - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'RTVI.observer()' is deprecated, instantiate an 'RTVIObserver' directly instead.", - DeprecationWarning, - ) - - return RTVIObserver(self) - def register_action(self, action: RTVIAction): id = self._action_id(action.service, action.action) self._registered_actions[id] = action diff --git a/src/pipecat/services/aws.py b/src/pipecat/services/aws.py index b83df2f77..4c0417d72 100644 --- a/src/pipecat/services/aws.py +++ b/src/pipecat/services/aws.py @@ -248,16 +248,3 @@ class PollyTTSService(TTSService): finally: yield TTSStoppedFrame() - - -class AWSTTSService(PollyTTSService): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'AWSTTSService' is deprecated, use 'PollyTTSService' instead.", DeprecationWarning - ) diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 7735ae91f..873b13cd1 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -6,7 +6,6 @@ import asyncio import time -import warnings from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from typing import Any, Awaitable, Callable, Mapping, Optional @@ -18,7 +17,7 @@ from daily import ( VirtualSpeakerDevice, ) from loguru import logger -from pydantic import BaseModel, model_validator +from pydantic import BaseModel from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams from pipecat.frames.frames import ( @@ -124,7 +123,6 @@ class DailyTranscriptionSettings(BaseModel): Attributes: language: ISO language code for transcription (e.g. "en"). - tier: Deprecated. Use model instead. model: Transcription model to use (e.g. "nova-2-general"). profanity_filter: Whether to filter profanity from transcripts. redact: Whether to redact sensitive information. @@ -135,7 +133,6 @@ class DailyTranscriptionSettings(BaseModel): """ language: str = "en" - tier: Optional[str] = None model: str = "nova-2-general" profanity_filter: bool = True redact: bool = False @@ -144,16 +141,6 @@ class DailyTranscriptionSettings(BaseModel): includeRawResponse: bool = True extra: Mapping[str, Any] = {"interim_results": True} - @model_validator(mode="before") - def check_deprecated_fields(cls, values): - with warnings.catch_warnings(): - warnings.simplefilter("always") - if "tier" in values: - warnings.warn( - "Field 'tier' is deprecated, use 'model' instead.", DeprecationWarning - ) - return values - class DailyParams(TransportParams): """Configuration parameters for Daily transport. diff --git a/src/pipecat/vad/__init__.py b/src/pipecat/vad/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/pipecat/vad/silero.py b/src/pipecat/vad/silero.py deleted file mode 100644 index 285e2e0e8..000000000 --- a/src/pipecat/vad/silero.py +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import warnings - -with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "Package `pipecat.vad` is deprecated, use `pipecat.audio.vad` instead", DeprecationWarning - ) - -from ..audio.vad.silero import SileroVADAnalyzer -from ..processors.audio.vad.silero import SileroVAD diff --git a/src/pipecat/vad/vad_analyzer.py b/src/pipecat/vad/vad_analyzer.py deleted file mode 100644 index 7c8fc4c31..000000000 --- a/src/pipecat/vad/vad_analyzer.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import warnings - -with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "Package `pipecat.vad` is deprecated, use `pipecat.audio.vad` instead", DeprecationWarning - ) - -from ..audio.vad.vad_analyzer import VADAnalyzer, VADParams, VADState From 24220f38f0880ea6f6e955d590acf5ae5bc675c9 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 7 Mar 2025 16:43:10 -0500 Subject: [PATCH 029/132] Add a Pipecat Cloud deployment example --- CHANGELOG.md | 4 + .../pipecat-cloud-example/.gitignore | 94 +++++++++ .../pipecat-cloud-example/Dockerfile | 7 + .../pipecat-cloud-example/README.md | 191 ++++++++++++++++++ .../deployment/pipecat-cloud-example/bot.py | 167 +++++++++++++++ .../pipecat-cloud-example/env.example | 2 + .../pipecat-cloud-example/local_runner.py | 46 +++++ .../pipecat-cloud-example/pcc-deploy.toml | 6 + .../pipecat-cloud-example/requirements.txt | 2 + 9 files changed, 519 insertions(+) create mode 100644 examples/deployment/pipecat-cloud-example/.gitignore create mode 100644 examples/deployment/pipecat-cloud-example/Dockerfile create mode 100644 examples/deployment/pipecat-cloud-example/README.md create mode 100644 examples/deployment/pipecat-cloud-example/bot.py create mode 100644 examples/deployment/pipecat-cloud-example/env.example create mode 100644 examples/deployment/pipecat-cloud-example/local_runner.py create mode 100644 examples/deployment/pipecat-cloud-example/pcc-deploy.toml create mode 100644 examples/deployment/pipecat-cloud-example/requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d56a323f..e47f1010a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -114,6 +114,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed an issue in `RimeTTSService` where the last line of text sent didn't result in an audio output being generated. +### Other + +- Added a Pipecat Cloud deployment example to the `examples` directory. + ## [0.0.58] - 2025-02-26 ### Added diff --git a/examples/deployment/pipecat-cloud-example/.gitignore b/examples/deployment/pipecat-cloud-example/.gitignore new file mode 100644 index 000000000..8a5b4de3d --- /dev/null +++ b/examples/deployment/pipecat-cloud-example/.gitignore @@ -0,0 +1,94 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +dist/ +*.egg-info/ +*.egg +.installed.cfg +.eggs/ +downloads/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +MANIFEST + +# Virtual Environments +venv/ +env/ +.env +.venv/ +ENV/ +env.bak/ +venv.bak/ + +# IDE +.idea/ +.vscode/ +.spyderproject +.spyproject +.ropeproject + +# Testing and Coverage +.coverage +.coverage.* +htmlcov/ +.pytest_cache/ +.tox/ +.nox/ +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +cover/ + +# Logs and Databases +*.log +*.db +db.sqlite3 +db.sqlite3-journal +pip-log.txt + +# System Files +.DS_Store +Thumbs.db +desktop.ini +*.swp +*.swo +*.bak +*.tmp +*~ + +# Build and Documentation +docs/_build/ +.pybuilder/ +target/ +instance/ +.webassets-cache +.pdm.toml +.pdm-python +.pdm-build/ +__pypackages__/ + +# Other +*.mo +*.pot +*.sage.py +.mypy_cache/ +.dmypy.json +dmypy.json +.pyre/ +.pytype/ +cython_debug/ +.ipynb_checkpoints + +# Pipecat cloud +.pcc-deploy.toml \ No newline at end of file diff --git a/examples/deployment/pipecat-cloud-example/Dockerfile b/examples/deployment/pipecat-cloud-example/Dockerfile new file mode 100644 index 000000000..290899e5c --- /dev/null +++ b/examples/deployment/pipecat-cloud-example/Dockerfile @@ -0,0 +1,7 @@ +FROM dailyco/pipecat-base:latest + +COPY ./requirements.txt requirements.txt + +RUN pip install --no-cache-dir --upgrade -r requirements.txt + +COPY ./bot.py bot.py \ No newline at end of file diff --git a/examples/deployment/pipecat-cloud-example/README.md b/examples/deployment/pipecat-cloud-example/README.md new file mode 100644 index 000000000..1950a0f94 --- /dev/null +++ b/examples/deployment/pipecat-cloud-example/README.md @@ -0,0 +1,191 @@ +# Pipecat Cloud Starter Project + +[![Docs](https://img.shields.io/badge/Documentation-blue)](https://docs.pipecat.daily.co) [![Discord](https://img.shields.io/discord/1217145424381743145)](https://discord.gg/dailyco) + +A template voice agent for [Pipecat Cloud](https://www.daily.co/products/pipecat-cloud/) that demonstrates building and deploying a conversational AI agent. + +## Prerequisites + +- Python 3.10+ +- Linux, MacOS, or Windows Subsystem for Linux (WSL) +- [Docker](https://www.docker.com) and a Docker repository (e.g., [Docker Hub](https://hub.docker.com)) +- A Docker Hub account (or other container registry account) +- [Pipecat Cloud](https://pipecat.daily.co) account + +> **Note**: If you haven't installed Docker yet, follow the official installation guides for your platform ([Linux](https://docs.docker.com/engine/install/), [Mac](https://docs.docker.com/desktop/setup/install/mac-install/), [Windows](https://docs.docker.com/desktop/setup/install/windows-install/)). For Docker Hub, [create a free account](https://hub.docker.com/signup) and log in via terminal with `docker login`. + +## Getting Started + +### 1. Set up Python environment + +We recommend using a virtual environment to manage your Python dependencies. + +```bash +# Create a virtual environment +python -m venv venv + +# Activate it +source venv/bin/activate # On Windows: venv\Scripts\activate + +# Install dependencies +pip install -r requirements.txt +pip install pipecatcloud +``` + +### 2. Get the starter project + +Clone the starter project from GitHub: + +```bash +git clone https://github.com/daily-co/pipecat-cloud-starter +cd pipecat-cloud-starter +``` + +or use the Pipecat Cloud CLI to initialize a new project: + +```bash +mkdir pipecat-cloud-starter && cd pipecat-cloud-starter +pcc init +``` + +### 3. Authenticate with Pipecat Cloud + +```bash +pcc auth login +``` + +### 4. Acquire required API keys + +This starter requires the following API keys: + +- **OpenAI API Key**: Get from [platform.openai.com/api-keys](https://platform.openai.com/api-keys) +- **Cartesia API Key**: Get from [play.cartesia.ai/keys](https://play.cartesia.ai/keys) +- **Daily API Key**: Automatically provided through your Pipecat Cloud account + +### 5. Configure to run locally (optional) + +You can test your agent locally before deploying to Pipecat Cloud: + +- `DAILY_API_KEY` value can be found at [https://pipecat.daily.co](https://pipecat.daily.co) Under the `Settings` menu of your agent, in the `Daily` tab. + +```bash +# Set environment variables with your API keys +export CARTESIA_API_KEY="your_cartesia_key" +export DAILY_API_KEY="your_daily_key" +export OPENAI_API_KEY="your_openai_key" +LOCAL_RUN=1 python bot.py +``` + +## Deploy & Run + +### 1. Build and push your Docker image + +```bash +# Build the image (targeting ARM architecture for cloud deployment) +docker build --platform=linux/arm64 -t my-first-agent:latest . + +# Tag with your Docker username and version +docker tag my-first-agent:latest your-username/my-first-agent:0.1 + +# Push to Docker Hub +docker push your-username/my-first-agent:0.1 +``` + +### 2. Create a secret set for your API keys + +The starter project requires API keys for OpenAI and Cartesia: + +```bash +# Copy the example env file +cp env.example .env + +# Edit .env to add your API keys: +# CARTESIA_API_KEY=your_cartesia_key +# OPENAI_API_KEY=your_openai_key + +# Create a secret set from your .env file +pcc secrets set my-first-agent-secrets --file .env +``` + +Alternatively, you can create secrets directly via CLI: + +```bash +pcc secrets set my-first-agent-secrets \ + CARTESIA_API_KEY=your_cartesia_key \ + OPENAI_API_KEY=your_openai_key +``` + +### 3. Deploy to Pipecat Cloud + +```bash +pcc deploy my-first-agent your-username/my-first-agent:0.1 +``` + +> **Note (Optional)**: For a more maintainable approach, you can use the included `pcc-deploy.toml` file: +> +> ```toml +> agent_name = "my-first-agent" +> image = "your-username/my-first-agent:0.1" +> secret_set = "my-first-agent-secrets" +> +> [scaling] +> min_instances = 0 +> ``` +> +> Then simply run `pcc deploy` without additional arguments. + +> **Note**: If your repository is private, you'll need to add credentials: +> +> ```bash +> # Create pull secret (you'll be prompted for credentials) +> pcc secrets image-pull-secret pull-secret https://index.docker.io/v1/ +> +> # Deploy with credentials +> pcc deploy my-first-agent your-username/my-first-agent:0.1 --credentials pull-secret +> ``` + +### 4. Check deployment and scaling (optional) + +By default, your agent will use "scale-to-zero" configuration, which means it may have a cold start of around 10 seconds when first used. By default, idle instances are maintained for 5 minutes before being terminated when using scale-to-zero. + +For more responsive testing, you can scale your deployment to keep a minimum of one instance warm: + +```bash +# Ensure at least one warm instance is always available +pcc deploy my-first-agent your-username/my-first-agent:0.1 --min-instances 1 + +# Check the status of your deployment +pcc agent status my-first-agent +``` + +By default, idle instances are maintained for 5 minutes before being terminated when using scale-to-zero. + +### 5. Create an API key + +```bash +# Create a public API key for accessing your agent +pcc organizations keys create + +# Set it as the default key to use with your agent +pcc organizations keys use +``` + +### 6. Start your agent + +```bash +# Start a session with your agent in a Daily room +pcc agent start my-first-agent --use-daily +``` + +This will return a URL, which you can use to connect to your running agent. + +## Documentation + +For more details on Pipecat Cloud and its capabilities: + +- [Pipecat Cloud Documentation](https://docs.pipecat.daily.co) +- [Pipecat Project Documentation](https://docs.pipecat.ai) + +## Support + +Join our [Discord community](https://discord.gg/dailyco) for help and discussions. diff --git a/examples/deployment/pipecat-cloud-example/bot.py b/examples/deployment/pipecat-cloud-example/bot.py new file mode 100644 index 000000000..fdb3eb712 --- /dev/null +++ b/examples/deployment/pipecat-cloud-example/bot.py @@ -0,0 +1,167 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMMessagesFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +# Check if we're in local development mode +LOCAL_RUN = os.getenv("LOCAL_RUN") +if LOCAL_RUN: + import asyncio + import webbrowser + + try: + from local_runner import configure + except ImportError: + logger.error("Could not import local_runner module. Local development mode may not work.") + +# Load environment variables +load_dotenv(override=True) + + +async def main(room_url: str, token: str, session_logger=None): + """Main pipeline setup and execution function. + + Args: + room_url: The Daily room URL + token: The Daily room token + session_logger: Optional logger instance + """ + log = session_logger or logger + + log.debug("Starting bot in room: {}", room_url) + + async with aiohttp.ClientSession() as session: + transport = DailyTransport( + room_url, + token, + "bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22" + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + log.info("First participant joined: {}", participant["id"]) + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append( + { + "role": "system", + "content": "Please start with 'Hello World' and introduce yourself to the user.", + } + ) + await task.queue_frames([LLMMessagesFrame(messages)]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + log.info("Participant left: {}", participant) + await task.cancel() + + runner = PipelineRunner() + + await runner.run(task) + + +async def bot(config, room_url: str, token: str, session_id=None, session_logger=None): + """Main bot entry point compatible with the FastAPI route handler. + + Args: + config: The configuration object from the request body + room_url: The Daily room URL + token: The Daily room token + session_id: The session ID for logging + session_logger: The session-specific logger + """ + log = session_logger or logger + log.info(f"Bot process initialized {room_url} {token}") + log.info(f"Bot config {config}") + + try: + await main(room_url, token, session_logger) + log.info("Bot process completed") + except Exception as e: + log.exception(f"Error in bot process: {str(e)}") + raise + + +# Local development functions +async def local_main(): + """Function for local development testing.""" + try: + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + logger.warning("_") + logger.warning("_") + logger.warning(f"Talk to your voice agent here: {room_url}") + logger.warning("_") + logger.warning("_") + webbrowser.open(room_url) + await main(room_url, token) + except Exception as e: + logger.exception(f"Error in local development mode: {e}") + + +# Local development entry point +if LOCAL_RUN and __name__ == "__main__": + try: + asyncio.run(local_main()) + except Exception as e: + logger.exception(f"Failed to run in local mode: {e}") diff --git a/examples/deployment/pipecat-cloud-example/env.example b/examples/deployment/pipecat-cloud-example/env.example new file mode 100644 index 000000000..1cbb5c4f6 --- /dev/null +++ b/examples/deployment/pipecat-cloud-example/env.example @@ -0,0 +1,2 @@ +CARTESIA_API_KEY= +OPENAI_API_KEY= \ No newline at end of file diff --git a/examples/deployment/pipecat-cloud-example/local_runner.py b/examples/deployment/pipecat-cloud-example/local_runner.py new file mode 100644 index 000000000..432592534 --- /dev/null +++ b/examples/deployment/pipecat-cloud-example/local_runner.py @@ -0,0 +1,46 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +import aiohttp + +from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper, DailyRoomParams + + +async def configure(aiohttp_session: aiohttp.ClientSession): + (url, token) = await configure_with_args(aiohttp_session) + return (url, token) + + +async def configure_with_args(aiohttp_session: aiohttp.ClientSession = None): + key = os.getenv("DAILY_API_KEY") + if not key: + raise Exception( + "No Daily API key specified. set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers." + ) + + daily_rest_helper = DailyRESTHelper( + daily_api_key=key, + daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"), + aiohttp_session=aiohttp_session, + ) + + room = await daily_rest_helper.create_room( + DailyRoomParams(properties={"enable_prejoin_ui": False}) + ) + if not room.url: + raise HTTPException(status_code=500, detail="Failed to create room") + + url = room.url + + # Create a meeting token for the given room with an expiration 1 hour in + # the future. + expiry_time: float = 60 * 60 + + token = await daily_rest_helper.get_token(url, expiry_time) + + return (url, token) diff --git a/examples/deployment/pipecat-cloud-example/pcc-deploy.toml b/examples/deployment/pipecat-cloud-example/pcc-deploy.toml new file mode 100644 index 000000000..063ed5929 --- /dev/null +++ b/examples/deployment/pipecat-cloud-example/pcc-deploy.toml @@ -0,0 +1,6 @@ +agent_name = "my-first-agent" +image = "your-username/my-first-agent:0.1" +secret_set = "my-first-agent-secrets" + +[scaling] + min_instances = 0 diff --git a/examples/deployment/pipecat-cloud-example/requirements.txt b/examples/deployment/pipecat-cloud-example/requirements.txt new file mode 100644 index 000000000..f5abaf91f --- /dev/null +++ b/examples/deployment/pipecat-cloud-example/requirements.txt @@ -0,0 +1,2 @@ +pipecat-ai[cartesia,daily,openai,silero]>=0.0.58 +python-dotenv~=1.0.1 \ No newline at end of file From d3cd1a6c597104685dad56d2d807fea07e799201 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 14 Mar 2025 20:37:55 -0400 Subject: [PATCH 030/132] Update with latest starter --- .../pipecat-cloud-example/README.md | 55 ++++--- .../deployment/pipecat-cloud-example/bot.py | 144 +++++++++--------- .../pipecat-cloud-example/requirements.txt | 3 +- 3 files changed, 101 insertions(+), 101 deletions(-) diff --git a/examples/deployment/pipecat-cloud-example/README.md b/examples/deployment/pipecat-cloud-example/README.md index 1950a0f94..7d5e4bd6f 100644 --- a/examples/deployment/pipecat-cloud-example/README.md +++ b/examples/deployment/pipecat-cloud-example/README.md @@ -4,6 +4,8 @@ A template voice agent for [Pipecat Cloud](https://www.daily.co/products/pipecat-cloud/) that demonstrates building and deploying a conversational AI agent. +> **For a detailed step-by-step guide, see our [Quickstart Documentation](https://docs.pipecat.daily.co/quickstart).** + ## Prerequisites - Python 3.10+ @@ -14,25 +16,9 @@ A template voice agent for [Pipecat Cloud](https://www.daily.co/products/pipecat > **Note**: If you haven't installed Docker yet, follow the official installation guides for your platform ([Linux](https://docs.docker.com/engine/install/), [Mac](https://docs.docker.com/desktop/setup/install/mac-install/), [Windows](https://docs.docker.com/desktop/setup/install/windows-install/)). For Docker Hub, [create a free account](https://hub.docker.com/signup) and log in via terminal with `docker login`. -## Getting Started +## Get Started -### 1. Set up Python environment - -We recommend using a virtual environment to manage your Python dependencies. - -```bash -# Create a virtual environment -python -m venv venv - -# Activate it -source venv/bin/activate # On Windows: venv\Scripts\activate - -# Install dependencies -pip install -r requirements.txt -pip install pipecatcloud -``` - -### 2. Get the starter project +### 1. Get the starter project Clone the starter project from GitHub: @@ -41,11 +27,19 @@ git clone https://github.com/daily-co/pipecat-cloud-starter cd pipecat-cloud-starter ``` -or use the Pipecat Cloud CLI to initialize a new project: +### 2. Set up your Python environment + +We recommend using a virtual environment to manage your Python dependencies. ```bash -mkdir pipecat-cloud-starter && cd pipecat-cloud-starter -pcc init +# Create a virtual environment +python -m venv .venv + +# Activate it +source .venv/bin/activate # On Windows: .venv\Scripts\activate + +# Install the Pipecat Cloud CLI +pip install pipecatcloud ``` ### 3. Authenticate with Pipecat Cloud @@ -66,13 +60,24 @@ This starter requires the following API keys: You can test your agent locally before deploying to Pipecat Cloud: -- `DAILY_API_KEY` value can be found at [https://pipecat.daily.co](https://pipecat.daily.co) Under the `Settings` menu of your agent, in the `Daily` tab. - ```bash # Set environment variables with your API keys export CARTESIA_API_KEY="your_cartesia_key" export DAILY_API_KEY="your_daily_key" export OPENAI_API_KEY="your_openai_key" +``` + +> Your `DAILY_API_KEY` can be found at [https://pipecat.daily.co](https://pipecat.daily.co) under the `Settings` in the `Daily (WebRTC)` tab. + +First install requirements: + +```bash +pip install -r requirements.txt +``` + +Then, launch the bot.py script locally: + +```bash LOCAL_RUN=1 python bot.py ``` @@ -118,7 +123,7 @@ pcc secrets set my-first-agent-secrets \ ### 3. Deploy to Pipecat Cloud ```bash -pcc deploy my-first-agent your-username/my-first-agent:0.1 +pcc deploy my-first-agent your-username/my-first-agent:0.1 --secrets my-first-agent-secrets ``` > **Note (Optional)**: For a more maintainable approach, you can use the included `pcc-deploy.toml` file: @@ -137,7 +142,7 @@ pcc deploy my-first-agent your-username/my-first-agent:0.1 > **Note**: If your repository is private, you'll need to add credentials: > > ```bash -> # Create pull secret (you'll be prompted for credentials) +> # Create pull secret (you’ll be prompted for credentials) > pcc secrets image-pull-secret pull-secret https://index.docker.io/v1/ > > # Deploy with credentials diff --git a/examples/deployment/pipecat-cloud-example/bot.py b/examples/deployment/pipecat-cloud-example/bot.py index fdb3eb712..89d4973b7 100644 --- a/examples/deployment/pipecat-cloud-example/bot.py +++ b/examples/deployment/pipecat-cloud-example/bot.py @@ -9,6 +9,7 @@ import os import aiohttp from dotenv import load_dotenv from loguru import logger +from pipecatcloud.agent import DailySessionArguments from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame @@ -35,110 +36,103 @@ if LOCAL_RUN: load_dotenv(override=True) -async def main(room_url: str, token: str, session_logger=None): +async def main(room_url: str, token: str): """Main pipeline setup and execution function. Args: room_url: The Daily room URL token: The Daily room token - session_logger: Optional logger instance """ - log = session_logger or logger + logger.debug("Starting bot in room: {}", room_url) - log.debug("Starting bot in room: {}", room_url) + transport = DailyTransport( + room_url, + token, + "bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) - async with aiohttp.ClientSession() as session: - transport = DailyTransport( - room_url, - token, - "bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22" + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22" - ) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] - messages = [ + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + logger.info("First participant joined: {}", participant["id"]) + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append( { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, - ] - - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] + "content": "Please start with 'Hello World' and introduce yourself to the user.", + } ) + await task.queue_frames([LLMMessagesFrame(messages)]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + logger.info("Participant left: {}", participant) + await task.cancel() - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - log.info("First participant joined: {}", participant["id"]) - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append( - { - "role": "system", - "content": "Please start with 'Hello World' and introduce yourself to the user.", - } - ) - await task.queue_frames([LLMMessagesFrame(messages)]) + runner = PipelineRunner() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - log.info("Participant left: {}", participant) - await task.cancel() - - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) -async def bot(config, room_url: str, token: str, session_id=None, session_logger=None): +async def bot(args: DailySessionArguments): """Main bot entry point compatible with the FastAPI route handler. Args: - config: The configuration object from the request body room_url: The Daily room URL token: The Daily room token + body: The configuration object from the request body session_id: The session ID for logging - session_logger: The session-specific logger """ - log = session_logger or logger - log.info(f"Bot process initialized {room_url} {token}") - log.info(f"Bot config {config}") + logger.info(f"Bot process initialized {args.room_url} {args.token}") try: - await main(room_url, token, session_logger) - log.info("Bot process completed") + await main(args.room_url, args.token) + logger.info("Bot process completed") except Exception as e: - log.exception(f"Error in bot process: {str(e)}") + logger.exception(f"Error in bot process: {str(e)}") raise diff --git a/examples/deployment/pipecat-cloud-example/requirements.txt b/examples/deployment/pipecat-cloud-example/requirements.txt index f5abaf91f..2e9fd4e9d 100644 --- a/examples/deployment/pipecat-cloud-example/requirements.txt +++ b/examples/deployment/pipecat-cloud-example/requirements.txt @@ -1,2 +1,3 @@ +pipecatcloud pipecat-ai[cartesia,daily,openai,silero]>=0.0.58 -python-dotenv~=1.0.1 \ No newline at end of file +python-dotenv~=1.0.1 From fa7da8f5f6b8acaaf6daeee205e9606cbc936bbb Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Sat, 1 Mar 2025 00:20:16 +0530 Subject: [PATCH 031/132] adding vertex llm --- CHANGELOG.md | 3 ++ src/pipecat/services/google/google.py | 73 +++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e47f1010a..b6001f82e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `AzureRealtimeBetaLLMService` to support Azure's OpeanAI Realtime API. Added foundational example `19a-azure-realtime-beta.py`. +- Introduced `GoogleVertexAIService`, a new class for integrating with Vertex AI + Gemini models. + ### Changed - Updated the default mode for `CartesiaTTSService` and diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 9d4a0a8a8..44a677136 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -71,6 +71,7 @@ try: import google.generativeai as gai from google import genai from google.api_core.client_options import ClientOptions + from google.auth.transport.requests import Request from google.cloud import speech_v2, texttospeech_v1 from google.cloud.speech_v2.types import cloud_speech from google.genai import types @@ -1333,6 +1334,78 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService): ) +class GoogleVertexAIService(OpenAILLMService): + """Implements inference with Google's AI models via Vertex AI while maintaining OpenAI API compatibility. + Reference: + https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library + """ + + class InputParams(OpenAILLMService.InputParams): + """Input parameters specific to Vertex AI.""" + + project_id: str + location: str + + def __init__( + self, + *, + credentials: Optional[str] = None, + credentials_path: Optional[str] = None, + model: str = "google/gemini-1.5-flash", + params: InputParams = OpenAILLMService.InputParams(), + **kwargs, + ): + """Initializes the VertexLLMService. + Args: + credentials (Optional[str]): JSON string of service account credentials. + credentials_path (Optional[str]): Path to the service account JSON file. + model (str): Model identifier. Defaults to "google/gemini-1.5-flash". + params (InputParams): Vertex AI input parameters. + **kwargs: Additional arguments for OpenAILLMService. + """ + base_url = self._get_base_url(params) + self._api_key = self._get_api_token(credentials, credentials_path) + + super().__init__(api_key=self._api_key, base_url=base_url, model=model, **kwargs) + + @staticmethod + def _get_base_url(params: InputParams) -> str: + """Constructs the base URL for Vertex AI API.""" + return ( + f"https://{params.location}-aiplatform.googleapis.com/v1/" + f"projects/{params.project_id}/locations/{params.location}/endpoints/openapi" + ) + + @staticmethod + def _get_api_token(credentials: Optional[str], credentials_path: Optional[str]) -> str: + """Retrieves an authentication token using Google service account credentials. + Args: + credentials (Optional[str]): JSON string of service account credentials. + credentials_path (Optional[str]): Path to the service account JSON file. + Returns: + str: OAuth token for API authentication. + """ + creds: Optional[service_account.Credentials] = None + + if credentials: + # Parse and load credentials from JSON string + creds = service_account.Credentials.from_service_account_info( + json.loads(credentials), scopes=["https://www.googleapis.com/auth/cloud-platform"] + ) + elif credentials_path: + # Load credentials from JSON file + creds = service_account.Credentials.from_service_account_file( + credentials_path, scopes=["https://www.googleapis.com/auth/cloud-platform"] + ) + + if not creds: + raise ValueError("No valid credentials provided.") + + creds.refresh(Request()) # Ensure token is up-to-date, lifetime is 1 hour. + + return creds.token + + class GoogleTTSService(TTSService): class InputParams(BaseModel): pitch: Optional[str] = None From 5f000efc611a3e7514df53e190abd6414296122d Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Sat, 15 Mar 2025 10:36:26 +0530 Subject: [PATCH 032/132] adding example --- CHANGELOG.md | 3 +- .../14p-function-calling-gemini-vertex-ai.py | 137 ++++++++++++++++++ src/pipecat/services/google/google.py | 7 +- 3 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 examples/foundational/14p-function-calling-gemini-vertex-ai.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b6001f82e..b3ee2646b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,7 +89,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 foundational example `19a-azure-realtime-beta.py`. - Introduced `GoogleVertexAIService`, a new class for integrating with Vertex AI - Gemini models. + Gemini models. Added foundational example + `14p-function-calling-gemini-vertex-ai.py`. ### Changed diff --git a/examples/foundational/14p-function-calling-gemini-vertex-ai.py b/examples/foundational/14p-function-calling-gemini-vertex-ai.py new file mode 100644 index 000000000..68608e932 --- /dev/null +++ b/examples/foundational/14p-function-calling-gemini-vertex-ai.py @@ -0,0 +1,137 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.google import GoogleVertexAIService +from pipecat.services.openai import OpenAILLMContext +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def start_fetch_weather(function_name, llm, context): + """Push a frame to the LLM; this is handy when the LLM response might take a while.""" + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) + logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") + + +async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await result_callback({"conditions": "nice", "temperature": "75"}) + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = ElevenLabsTTSService( + api_key=os.getenv("ELEVENLABS_API_KEY", ""), + voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), + ) + + llm = GoogleVertexAIService( + # credentials="", + params=GoogleVertexAIService.InputParams( + project_id="", + ) + ) + # Register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function( + "get_current_weather", fetch_weather_from_api, start_callback=start_fetch_weather + ) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + + messages = [ + { + "role": "user", + "content": "Start a conversation with 'Hey there' to get the current weather.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 44a677136..f741154b9 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -1343,15 +1343,16 @@ class GoogleVertexAIService(OpenAILLMService): class InputParams(OpenAILLMService.InputParams): """Input parameters specific to Vertex AI.""" + # https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations + location: str = "us-east4" project_id: str - location: str def __init__( self, *, credentials: Optional[str] = None, credentials_path: Optional[str] = None, - model: str = "google/gemini-1.5-flash", + model: str = "google/gemini-2.0-flash-001", params: InputParams = OpenAILLMService.InputParams(), **kwargs, ): @@ -1359,7 +1360,7 @@ class GoogleVertexAIService(OpenAILLMService): Args: credentials (Optional[str]): JSON string of service account credentials. credentials_path (Optional[str]): Path to the service account JSON file. - model (str): Model identifier. Defaults to "google/gemini-1.5-flash". + model (str): Model identifier. Defaults to "google/gemini-2.0-flash-001". params (InputParams): Vertex AI input parameters. **kwargs: Additional arguments for OpenAILLMService. """ From c1382b06911d32982807059fbb9a3ecef25a376f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 15 Mar 2025 20:30:35 -0400 Subject: [PATCH 033/132] Update the 34-audio-recording.py example to include an STT processor --- CHANGELOG.md | 2 ++ examples/foundational/34-audio-recording.py | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e47f1010a..d0e9f8713 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Other +- Update the `34-audio-recording.py` example to include an STT processor. + - Added a Pipecat Cloud deployment example to the `examples` directory. ## [0.0.58] - 2025-02-26 diff --git a/examples/foundational/34-audio-recording.py b/examples/foundational/34-audio-recording.py index 94877c722..24f009570 100644 --- a/examples/foundational/34-audio-recording.py +++ b/examples/foundational/34-audio-recording.py @@ -32,6 +32,7 @@ Requirements: OPENAI_API_KEY=your_openai_key CARTESIA_API_KEY=your_cartesia_key DAILY_API_KEY=your_daily_key + DEEPGRAM_API_KEY=your_deepgram_key The recordings will be saved in a 'recordings' directory with timestamps: recordings/ @@ -65,6 +66,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -98,13 +100,14 @@ async def main(): DailyParams( # audio_in_enabled=True, audio_out_enabled=True, - transcription_enabled=True, vad_enabled=True, vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, # Enable audio passthrough for recording + vad_audio_passthrough=True, ), ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"), audio_passthrough=True) + tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", @@ -128,6 +131,7 @@ async def main(): pipeline = Pipeline( [ transport.input(), + stt, context_aggregator.user(), llm, tts, From 02dbef8f5a4453212ceeb74e51bc9603cff2b522 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 17 Mar 2025 11:28:51 -0400 Subject: [PATCH 034/132] Add Neuphonic TTS to README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 822c75aee..f75e9d386 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ pip install "pipecat-ai[option,...]" | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | | Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Parakeet (NVIDIA)](https://docs.pipecat.ai/server/services/stt/parakeet), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"` | -| Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [FastPitch (NVIDIA)](https://docs.pipecat.ai/server/services/tts/fastpitch), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | +| Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [FastPitch (NVIDIA)](https://docs.pipecat.ai/server/services/tts/fastpitch), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | | Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[google]"` | | Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | `pip install "pipecat-ai[daily]"` | | Video | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | From 3458f1b6dede365574cc91f11f00a964b08b96b1 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 17 Mar 2025 11:43:40 -0400 Subject: [PATCH 035/132] Add Google Imagen to README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f75e9d386..9522ce175 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ pip install "pipecat-ai[option,...]" | Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[google]"` | | Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | `pip install "pipecat-ai[daily]"` | | Video | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | -| Vision & Image | [Moondream](https://docs.pipecat.ai/server/services/vision/moondream), [fal](https://docs.pipecat.ai/server/services/image-generation/fal) | `pip install "pipecat-ai[moondream]"` | +| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/fal), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) | `pip install "pipecat-ai[moondream]"` | | Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [Noisereduce](https://docs.pipecat.ai/server/utilities/audio/noisereduce-filter) | `pip install "pipecat-ai[silero]"` | | Analytics & Metrics | [Canonical AI](https://docs.pipecat.ai/server/services/analytics/canonical), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | `pip install "pipecat-ai[canonical]"` | From 1ad8e28025dbe04272608cc00636f2ff76d90bd8 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 13 Mar 2025 14:21:36 -0400 Subject: [PATCH 036/132] Update TranscriptProcessor to more robustly handle different TTSTextFrame outputs --- .../processors/transcript_processor.py | 45 ++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py index 614fe176c..6a7793335 100644 --- a/src/pipecat/processors/transcript_processor.py +++ b/src/pipecat/processors/transcript_processor.py @@ -90,9 +90,50 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): self._aggregation_start_time: Optional[str] = None async def _emit_aggregated_text(self): - """Emit aggregated text as a transcript message.""" + """Emit aggregated text as a transcript message. + + This method intelligently joins text fragments to create natural spacing, + handling both word-by-word and pre-spaced text fragments appropriately. + + The implementation handles two common patterns from TTS services: + + 1. Word-by-word fragments without spacing: + ``` + TTSTextFrame: ['Hello.'] + TTSTextFrame: ['How'] + TTSTextFrame: ['can'] + TTSTextFrame: ['I'] + TTSTextFrame: ['assist'] + TTSTextFrame: ['you'] + TTSTextFrame: ['today?'] + ``` + Result: "Hello. How can I assist you today?" + + 2. Pre-spaced fragments: + ``` + TTSTextFrame: ['Hello'] + TTSTextFrame: [' there'] + TTSTextFrame: ['!'] + TTSTextFrame: [' How'] + TTSTextFrame: ["'s"] + TTSTextFrame: [' it'] + TTSTextFrame: [' going'] + TTSTextFrame: ['?'] + ``` + Result: "Hello there! How's it going?" + """ if self._current_text_parts and self._aggregation_start_time: - content = " ".join(self._current_text_parts).strip() + # Build content with intelligent spacing + content = "" + for i, part in enumerate(self._current_text_parts): + # Add a space only when the current part doesn't start with + # whitespace or punctuation/special characters + if i > 0 and not part.startswith((" ", ".", ",", "!", "?", ";", ":", "'", '"')): + content += " " + content += part + + content = content.strip() + if content: logger.debug(f"Emitting aggregated assistant message: {content}") message = TranscriptionMessage( From 5b6b700214b308319730263dd6854a6e99b22efe Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 13 Mar 2025 14:22:13 -0400 Subject: [PATCH 037/132] OpenAIRealtimeBetaLLMService outputs a TTSTextFrame --- src/pipecat/services/openai_realtime_beta/openai.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 00f8cd840..321c66826 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -43,6 +43,7 @@ from pipecat.frames.frames import ( TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, + TTSTextFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -471,6 +472,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def _handle_evt_audio_transcript_delta(self, evt): if evt.delta: await self.push_frame(LLMTextFrame(evt.delta)) + await self.push_frame(TTSTextFrame(evt.delta)) async def _handle_evt_speech_started(self, evt): await self._truncate_current_audio_response() From 571c10403f9684bdcac175272c51d7e692f98378 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 13 Mar 2025 14:23:00 -0400 Subject: [PATCH 038/132] tests: Add additional coverage to test_transcript_processor --- tests/test_transcript_processor.py | 298 ++++++++++++++++++++++++++++- 1 file changed, 297 insertions(+), 1 deletion(-) diff --git a/tests/test_transcript_processor.py b/tests/test_transcript_processor.py index 1c6db277f..5f80b3ca6 100644 --- a/tests/test_transcript_processor.py +++ b/tests/test_transcript_processor.py @@ -275,7 +275,7 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): # First update should be interrupted message first_message = received_updates[0].messages[0] self.assertEqual(first_message.role, "assistant") - self.assertEqual(first_message.content, "Hello world !") + self.assertEqual(first_message.content, "Hello world!") self.assertIsNotNone(first_message.timestamp) # Second update should be new response @@ -426,3 +426,299 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): self.assertEqual(received_updates[0].content, "User message") self.assertEqual(received_updates[1].role, "assistant") self.assertEqual(received_updates[1].content, "Assistant message") + + async def test_text_fragments_with_spaces(self): + """Test aggregating text fragments with various spacing patterns""" + processor = AssistantTranscriptProcessor() + + # Track received updates + received_updates = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + # Test the specific pattern shared + frames_to_send = [ + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), + TTSTextFrame(text="Hello"), + TTSTextFrame(text=" there"), + TTSTextFrame(text="!"), + TTSTextFrame(text=" How"), + TTSTextFrame(text="'s"), + TTSTextFrame(text=" it"), + TTSTextFrame(text=" going"), + TTSTextFrame(text="?"), + BotStoppedSpeakingFrame(), + ] + + expected_down_frames = [ + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TranscriptionUpdateFrame, + ] + + # Run test + received_frames, _ = await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + # Verify result + self.assertEqual(len(received_updates), 1) + message = received_updates[0].messages[0] + self.assertEqual(message.role, "assistant") + # Should be properly joined without extra spaces + self.assertEqual(message.content, "Hello there! How's it going?") + + async def test_mixed_spacing_styles(self): + """Test handling mixed word-by-word and pre-spaced fragments""" + processor = AssistantTranscriptProcessor() + + received_updates = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + # Mix of spacing styles within the same utterance + frames_to_send = [ + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), + # Word-by-word style + TTSTextFrame(text="First"), + TTSTextFrame(text="style."), + # Pre-spaced style + TTSTextFrame(text=" Second"), + TTSTextFrame(text=" style"), + TTSTextFrame(text="!"), + BotStoppedSpeakingFrame(), + ] + + expected_down_frames = [ + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TranscriptionUpdateFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + self.assertEqual(len(received_updates), 1) + message = received_updates[0].messages[0] + self.assertEqual(message.content, "First style. Second style!") + + async def test_punctuation_handling(self): + """Test handling of various punctuation patterns""" + processor = AssistantTranscriptProcessor() + + received_updates = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + # Test various punctuation types + frames_to_send = [ + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), + TTSTextFrame(text="Commas"), + TTSTextFrame(text=","), + TTSTextFrame(text="colons"), + TTSTextFrame(text=":"), + TTSTextFrame(text="semicolons"), + TTSTextFrame(text=";"), + TTSTextFrame(text="quotes"), + TTSTextFrame(text="'"), + TTSTextFrame(text="and"), + TTSTextFrame(text='"'), + TTSTextFrame(text="double quotes"), + TTSTextFrame(text="!"), + BotStoppedSpeakingFrame(), + ] + + expected_down_frames = [ + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TranscriptionUpdateFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + self.assertEqual(len(received_updates), 1) + message = received_updates[0].messages[0] + self.assertEqual( + message.content, "Commas, colons: semicolons; quotes' and\" double quotes!" + ) + + async def test_complex_mixed_case(self): + """Test a complex mix of patterns to ensure robustness""" + processor = AssistantTranscriptProcessor() + + received_updates = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + # Complex mixed case with various patterns + frames_to_send = [ + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), + # Pre-spaced fragments + TTSTextFrame(text="Hello"), + TTSTextFrame(text=" there"), + TTSTextFrame(text="!"), + # Sentence boundary + TTSTextFrame(text=" I'm"), + TTSTextFrame(text=" testing"), + TTSTextFrame(text=" spacing"), + TTSTextFrame(text="."), + # Word-by-word fragments + TTSTextFrame(text="Does"), + TTSTextFrame(text="this"), + TTSTextFrame(text="work"), + TTSTextFrame(text="correctly"), + TTSTextFrame(text="?"), + # Mixed punctuation and spacing + TTSTextFrame(text=" Let's"), + TTSTextFrame(text=" see:"), + TTSTextFrame(text="commas"), + TTSTextFrame(text=","), + TTSTextFrame(text=" semicolons"), + TTSTextFrame(text=";"), + TTSTextFrame(text=" and"), + TTSTextFrame(text=" quotes"), + TTSTextFrame(text="'"), + TTSTextFrame(text="!"), + BotStoppedSpeakingFrame(), + ] + + expected_down_frames = [ + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TranscriptionUpdateFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + self.assertEqual(len(received_updates), 1) + message = received_updates[0].messages[0] + expected = "Hello there! I'm testing spacing. Does this work correctly? Let's see: commas, semicolons; and quotes'!" + self.assertEqual(message.content, expected) + + async def test_multiple_consecutive_punctuation(self): + """Test handling of multiple consecutive punctuation marks""" + processor = AssistantTranscriptProcessor() + + received_updates = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + frames_to_send = [ + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), + TTSTextFrame(text="Wow"), + TTSTextFrame(text="!"), + TTSTextFrame(text="!"), + TTSTextFrame(text="!"), + TTSTextFrame(text=" That's"), + TTSTextFrame(text=" amazing"), + TTSTextFrame(text="..."), + TTSTextFrame(text=" Don't"), + TTSTextFrame(text=" you"), + TTSTextFrame(text=" think"), + TTSTextFrame(text="?"), + TTSTextFrame(text="?"), + BotStoppedSpeakingFrame(), + ] + + expected_down_frames = [ + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TranscriptionUpdateFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + self.assertEqual(len(received_updates), 1) + message = received_updates[0].messages[0] + self.assertEqual(message.content, "Wow!!! That's amazing... Don't you think??") From 6e6905405b48f1548370650a0109419e96cbaebd Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 13 Mar 2025 14:25:02 -0400 Subject: [PATCH 039/132] Update CHANGELOG --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e47f1010a..b24b74bc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,6 +90,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Updated `TranscriptProcessor` to support text output from + `OpenAIRealtimeBetaLLMService`. + +- `OpenAIRealtimeBetaLLMService` now pushes a `TTSTextFrame`. + - Updated the default mode for `CartesiaTTSService` and `CartesiaHttpTTSService` to `sonic-2`. From d5776c27f4cbeb8a576c00e6c32545f994f29a55 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 13 Mar 2025 14:31:57 -0400 Subject: [PATCH 040/132] Update 19-openai-realtime-beta --- examples/foundational/19-openai-realtime-beta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 504f8fbf1..8796e0141 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -147,8 +147,8 @@ Remember, your responses should be short. Just one or two sentences, usually.""" transport.input(), # Transport user input context_aggregator.user(), llm, # LLM - context_aggregator.assistant(), transport.output(), # Transport bot output + context_aggregator.assistant(), ] ) From 3f002f8ffb837f55d0f08a52cf2a28644e75a766 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 13 Mar 2025 14:34:05 -0400 Subject: [PATCH 041/132] Remove unnecessary TranscriptProcessor examples --- CHANGELOG.md | 4 + ...penai.py => 28-transcription-processor.py} | 0 .../28b-transcript-processor-anthropic.py | 177 --------------- .../28c-transcription-processor-gemini.py | 210 ------------------ 4 files changed, 4 insertions(+), 387 deletions(-) rename examples/foundational/{28a-transcription-processor-openai.py => 28-transcription-processor.py} (100%) delete mode 100644 examples/foundational/28b-transcript-processor-anthropic.py delete mode 100644 examples/foundational/28c-transcription-processor-gemini.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b24b74bc5..596a3e1e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,6 +123,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added a Pipecat Cloud deployment example to the `examples` directory. +- Removed foundational examples 28b and 28c as the TranscriptProcessor no + longer has an LLM depedency. Renamed foundational example 28a to + `28-transcript-processor.py`. + ## [0.0.58] - 2025-02-26 ### Added diff --git a/examples/foundational/28a-transcription-processor-openai.py b/examples/foundational/28-transcription-processor.py similarity index 100% rename from examples/foundational/28a-transcription-processor-openai.py rename to examples/foundational/28-transcription-processor.py diff --git a/examples/foundational/28b-transcript-processor-anthropic.py b/examples/foundational/28b-transcript-processor-anthropic.py deleted file mode 100644 index c9f2672e0..000000000 --- a/examples/foundational/28b-transcript-processor-anthropic.py +++ /dev/null @@ -1,177 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os -import sys -from typing import List, Optional - -import aiohttp -from dotenv import load_dotenv -from loguru import logger -from runner import configure - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TranscriptionMessage, TranscriptionUpdateFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.transcript_processor import TranscriptProcessor -from pipecat.services.anthropic import AnthropicLLMService -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.deepgram import DeepgramSTTService -from pipecat.transports.services.daily import DailyParams, DailyTransport - -load_dotenv(override=True) - -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - - -class TranscriptHandler: - """Handles real-time transcript processing and output. - - Maintains a list of conversation messages and outputs them either to a log - or to a file as they are received. Each message includes its timestamp and role. - - Attributes: - messages: List of all processed transcript messages - output_file: Optional path to file where transcript is saved. If None, outputs to log only. - """ - - def __init__(self, output_file: Optional[str] = None): - """Initialize handler with optional file output. - - Args: - output_file: Path to output file. If None, outputs to log only. - """ - self.messages: List[TranscriptionMessage] = [] - self.output_file: Optional[str] = output_file - logger.debug( - f"TranscriptHandler initialized {'with output_file=' + output_file if output_file else 'with log output only'}" - ) - - async def save_message(self, message: TranscriptionMessage): - """Save a single transcript message. - - Outputs the message to the log and optionally to a file. - - Args: - message: The message to save - """ - timestamp = f"[{message.timestamp}] " if message.timestamp else "" - line = f"{timestamp}{message.role}: {message.content}" - - # Always log the message - logger.info(f"Transcript: {line}") - - # Optionally write to file - if self.output_file: - try: - with open(self.output_file, "a", encoding="utf-8") as f: - f.write(line + "\n") - except Exception as e: - logger.error(f"Error saving transcript message to file: {e}") - - async def on_transcript_update( - self, processor: TranscriptProcessor, frame: TranscriptionUpdateFrame - ): - """Handle new transcript messages. - - Args: - processor: The TranscriptProcessor that emitted the update - frame: TranscriptionUpdateFrame containing new messages - """ - logger.debug(f"Received transcript update with {len(frame.messages)} new messages") - - for msg in frame.messages: - self.messages.append(msg) - await self.save_message(msg) - - -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - - transport = DailyTransport( - room_url, - None, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-5-sonnet-20241022" - ) - - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way.", - }, - {"role": "user", "content": "Say hello."}, - ] - - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - - # Create transcript processor and handler - transcript = TranscriptProcessor() - transcript_handler = TranscriptHandler() # Output to log only - # transcript_handler = TranscriptHandler(output_file="transcript.txt") # Output to file and log - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - transcript.user(), # User transcripts - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - transcript.assistant(), # Assistant transcripts - context_aggregator.assistant(), # Assistant spoken responses - ] - ) - - task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) - - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - # Register event handler for transcript updates - @transcript.event_handler("on_transcript_update") - async def on_transcript_update(processor, frame): - await transcript_handler.on_transcript_update(processor, frame) - - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - # Stop the pipeline immediately when the participant leaves - await task.cancel() - - runner = PipelineRunner() - - await runner.run(task) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/foundational/28c-transcription-processor-gemini.py b/examples/foundational/28c-transcription-processor-gemini.py deleted file mode 100644 index 558edc76d..000000000 --- a/examples/foundational/28c-transcription-processor-gemini.py +++ /dev/null @@ -1,210 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os -import sqlite3 -import sys -from typing import List, Optional - -import aiohttp -from dotenv import load_dotenv -from loguru import logger -from runner import configure - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TranscriptionMessage, TranscriptionUpdateFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.transcript_processor import TranscriptProcessor -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.deepgram import DeepgramSTTService -from pipecat.services.google import GoogleLLMService -from pipecat.services.openai import OpenAILLMContext -from pipecat.transports.services.daily import DailyParams, DailyTransport - -load_dotenv(override=True) - -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - - -class TranscriptHandler: - """Handles real-time transcript processing and output. - - Maintains a list of conversation messages and outputs them either to a log - or to a file as they are received. Each message includes its timestamp and role. - - Attributes: - messages: List of all processed transcript messages - output_file: Optional path to file where transcript is saved. If None, outputs to log only. - """ - - def __init__(self, output_file: Optional[str] = None, output_db: Optional[str] = None): - """Initialize handler with optional file or database output. - - Args: - output_file: Path to output file. If None, outputs to log only. - """ - self.messages: List[TranscriptionMessage] = [] - self.output_file: Optional[str] = output_file - self.output_db: Optional[str] = output_db - - if self.output_db: - self.con = sqlite3.connect("example.db") - self.db = self.con.cursor() - - table = self.db.execute("SELECT name FROM sqlite_master WHERE name='messages'") - if not (table.fetchone()): - self.db.execute( - "CREATE TABLE messages(role TEXT, content TEXT, timestamp DATETIME DEFAULT CURRENT_TIMESTAMP )" - ) - logger.debug( - f"TranscriptHandler initialized; output file: {output_file}, output DB: {output_db}" - ) - - async def save_message(self, message: TranscriptionMessage): - """Save a single transcript message. - - Outputs the message to the log and optionally to a SQLite database or file. - - Args: - message: The message to save - """ - timestamp = f"[{message.timestamp}] " if message.timestamp else "" - line = f"{timestamp}{message.role}: {message.content}" - - # Always log the message - logger.info(f"Transcript: {line}") - - # Optionally write to file - if self.output_file: - try: - with open(self.output_file, "a", encoding="utf-8") as f: - f.write(line + "\n") - except Exception as e: - logger.error(f"Error saving transcript message to file: {e}") - - # and/or to a SQLite database - if self.output_db: - self.db.execute( - "INSERT INTO messages VALUES (?, ?, ?)", - (message.role, message.content, message.timestamp), - ) - self.con.commit() - - async def on_transcript_update( - self, processor: TranscriptProcessor, frame: TranscriptionUpdateFrame - ): - """Handle new transcript messages. - - Args: - processor: The TranscriptProcessor that emitted the update - frame: TranscriptionUpdateFrame containing new messages - """ - logger.debug(f"Received transcript update with {len(frame.messages)} new messages") - - for msg in frame.messages: - self.messages.append(msg) - await self.save_message(msg) - - -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - - transport = DailyTransport( - room_url, - None, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = GoogleLLMService( - model="models/gemini-2.0-flash-exp", - # model="gemini-exp-1114", - api_key=os.getenv("GOOGLE_API_KEY"), - ) - - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way.", - }, - {"role": "user", "content": "Say hello."}, - ] - - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - - # Create transcript processor and handler - transcript = TranscriptProcessor() - # Select a TranscriptHandler output method - # Uncomment out only one of the following lines: - transcript_handler = TranscriptHandler() # Output to log only - # transcript_handler = TranscriptHandler(output_file="transcript.txt") # Output to file and log - # transcript_handler = TranscriptHandler(output_db="example.db") # Output to SQLite DB and log - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - transcript.user(), # User transcripts - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - transcript.assistant(), # Assistant transcripts - context_aggregator.assistant(), # Assistant spoken responses - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) - - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - # Register event handler for transcript updates - @transcript.event_handler("on_transcript_update") - async def on_transcript_update(processor, frame): - await transcript_handler.on_transcript_update(processor, frame) - - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - # Stop the pipeline immediately when the participant leaves - await task.cancel() - - runner = PipelineRunner() - - await runner.run(task) - - -if __name__ == "__main__": - asyncio.run(main()) From acd0660f66ccecaabf3089480045faba0c37a586 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 14 Mar 2025 14:00:38 -0400 Subject: [PATCH 042/132] Update GeminiMultimodalLiveLLMService to work with the TranscriptProcessor --- CHANGELOG.md | 3 ++- src/pipecat/services/gemini_multimodal_live/gemini.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 596a3e1e0..7376603c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,7 +93,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated `TranscriptProcessor` to support text output from `OpenAIRealtimeBetaLLMService`. -- `OpenAIRealtimeBetaLLMService` now pushes a `TTSTextFrame`. +- `OpenAIRealtimeBetaLLMService` and `GeminiMultimodalLiveLLMService` now push + a `TTSTextFrame`. - Updated the default mode for `CartesiaTTSService` and `CartesiaHttpTTSService` to `sonic-2`. diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index ef49df329..5fe8a792b 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -38,6 +38,7 @@ from pipecat.frames.frames import ( TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, + TTSTextFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -312,6 +313,7 @@ class GeminiMultimodalLiveLLMService(LLMService): # context.add_message({"role": "assistant", "content": [{"type": "text", "text": text}]}) await self.push_frame(LLMFullResponseStartFrame()) await self.push_frame(LLMTextFrame(text=text)) + await self.push_frame(TTSTextFrame(text=text)) await self.push_frame(LLMFullResponseEndFrame()) async def _transcribe_audio(self, audio, context): From 6885d07e880341d1a5ae46054ae8d64609c3e9eb Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 17 Mar 2025 16:30:46 -0400 Subject: [PATCH 043/132] Simplify the TranscriptProcessor _emit_aggregated_text logic --- .../processors/transcript_processor.py | 86 +++--- tests/test_transcript_processor.py | 248 +----------------- 2 files changed, 50 insertions(+), 284 deletions(-) diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py index 6a7793335..3eaff66ca 100644 --- a/src/pipecat/processors/transcript_processor.py +++ b/src/pipecat/processors/transcript_processor.py @@ -90,52 +90,62 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): self._aggregation_start_time: Optional[str] = None async def _emit_aggregated_text(self): - """Emit aggregated text as a transcript message. + """Aggregates and emits text fragments as a transcript message. - This method intelligently joins text fragments to create natural spacing, - handling both word-by-word and pre-spaced text fragments appropriately. + This method uses a heuristic to automatically detect whether text fragments + use pre-spacing (spaces at the beginning of fragments) or not, and applies + the appropriate joining strategy. It handles fragments from different TTS + services with different formatting patterns. - The implementation handles two common patterns from TTS services: + Examples: + Pre-spaced fragments (concatenated): + ``` + TTSTextFrame: ["Hello"] + TTSTextFrame: [" there"] + TTSTextFrame: ["!"] + TTSTextFrame: [" How"] + TTSTextFrame: ["'s"] + TTSTextFrame: [" it"] + TTSTextFrame: [" going"] + TTSTextFrame: ["?"] + ``` + Result: "Hello there! How's it going?" - 1. Word-by-word fragments without spacing: - ``` - TTSTextFrame: ['Hello.'] - TTSTextFrame: ['How'] - TTSTextFrame: ['can'] - TTSTextFrame: ['I'] - TTSTextFrame: ['assist'] - TTSTextFrame: ['you'] - TTSTextFrame: ['today?'] - ``` - Result: "Hello. How can I assist you today?" - - 2. Pre-spaced fragments: - ``` - TTSTextFrame: ['Hello'] - TTSTextFrame: [' there'] - TTSTextFrame: ['!'] - TTSTextFrame: [' How'] - TTSTextFrame: ["'s"] - TTSTextFrame: [' it'] - TTSTextFrame: [' going'] - TTSTextFrame: ['?'] - ``` - Result: "Hello there! How's it going?" + Word-by-word fragments (joined with spaces): + ``` + TTSTextFrame: ["Hello"] + TTSTextFrame: ["there!"] + TTSTextFrame: ["How"] + TTSTextFrame: ["is"] + TTSTextFrame: ["it"] + TTSTextFrame: ["going?"] + ``` + Result: "Hello there! How is it going?" """ if self._current_text_parts and self._aggregation_start_time: - # Build content with intelligent spacing - content = "" - for i, part in enumerate(self._current_text_parts): - # Add a space only when the current part doesn't start with - # whitespace or punctuation/special characters - if i > 0 and not part.startswith((" ", ".", ",", "!", "?", ";", ":", "'", '"')): - content += " " - content += part + # Heuristic to detect pre-spaced fragments + uses_prespacing = False + if len(self._current_text_parts) > 1: + # Check if any fragment after the first one starts with whitespace + has_spaced_parts = any( + part and part[0].isspace() for part in self._current_text_parts[1:] + ) + if has_spaced_parts: + uses_prespacing = True + # Apply appropriate joining method + if uses_prespacing: + # Pre-spaced fragments - just concatenate + content = "".join(self._current_text_parts) + else: + # Word-by-word fragments - join with spaces + content = " ".join(self._current_text_parts) + + # Clean up any excessive whitespace content = content.strip() if content: - logger.debug(f"Emitting aggregated assistant message: {content}") + logger.trace(f"Emitting aggregated assistant message: {content}") message = TranscriptionMessage( role="assistant", content=content, @@ -143,7 +153,7 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): ) await self._emit_update([message]) else: - logger.debug("No content to emit after stripping whitespace") + logger.trace("No content to emit after stripping whitespace") # Reset aggregation state self._current_text_parts = [] diff --git a/tests/test_transcript_processor.py b/tests/test_transcript_processor.py index 5f80b3ca6..d13246b2c 100644 --- a/tests/test_transcript_processor.py +++ b/tests/test_transcript_processor.py @@ -235,8 +235,7 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): BotStartedSpeakingFrame(), SleepFrame(sleep=0.1), TTSTextFrame(text="Hello"), - TTSTextFrame(text="world"), - TTSTextFrame(text="!"), + TTSTextFrame(text="world!"), SleepFrame(sleep=0.1), StartInterruptionFrame(), # User interrupts here BotStartedSpeakingFrame(), @@ -251,8 +250,7 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): expected_down_frames = [ BotStartedSpeakingFrame, TTSTextFrame, # "Hello" - TTSTextFrame, # "world" - TTSTextFrame, # "!" + TTSTextFrame, # "world!" TranscriptionUpdateFrame, # First message (emitted due to interruption) StartInterruptionFrame, # Interruption frame comes after the update BotStartedSpeakingFrame, @@ -480,245 +478,3 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): self.assertEqual(message.role, "assistant") # Should be properly joined without extra spaces self.assertEqual(message.content, "Hello there! How's it going?") - - async def test_mixed_spacing_styles(self): - """Test handling mixed word-by-word and pre-spaced fragments""" - processor = AssistantTranscriptProcessor() - - received_updates = [] - - @processor.event_handler("on_transcript_update") - async def handle_update(proc, frame: TranscriptionUpdateFrame): - received_updates.append(frame) - - # Mix of spacing styles within the same utterance - frames_to_send = [ - BotStartedSpeakingFrame(), - SleepFrame(sleep=0.1), - # Word-by-word style - TTSTextFrame(text="First"), - TTSTextFrame(text="style."), - # Pre-spaced style - TTSTextFrame(text=" Second"), - TTSTextFrame(text=" style"), - TTSTextFrame(text="!"), - BotStoppedSpeakingFrame(), - ] - - expected_down_frames = [ - BotStartedSpeakingFrame, - BotStoppedSpeakingFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TranscriptionUpdateFrame, - ] - - await run_test( - processor, - frames_to_send=frames_to_send, - expected_down_frames=expected_down_frames, - ) - - self.assertEqual(len(received_updates), 1) - message = received_updates[0].messages[0] - self.assertEqual(message.content, "First style. Second style!") - - async def test_punctuation_handling(self): - """Test handling of various punctuation patterns""" - processor = AssistantTranscriptProcessor() - - received_updates = [] - - @processor.event_handler("on_transcript_update") - async def handle_update(proc, frame: TranscriptionUpdateFrame): - received_updates.append(frame) - - # Test various punctuation types - frames_to_send = [ - BotStartedSpeakingFrame(), - SleepFrame(sleep=0.1), - TTSTextFrame(text="Commas"), - TTSTextFrame(text=","), - TTSTextFrame(text="colons"), - TTSTextFrame(text=":"), - TTSTextFrame(text="semicolons"), - TTSTextFrame(text=";"), - TTSTextFrame(text="quotes"), - TTSTextFrame(text="'"), - TTSTextFrame(text="and"), - TTSTextFrame(text='"'), - TTSTextFrame(text="double quotes"), - TTSTextFrame(text="!"), - BotStoppedSpeakingFrame(), - ] - - expected_down_frames = [ - BotStartedSpeakingFrame, - BotStoppedSpeakingFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TranscriptionUpdateFrame, - ] - - await run_test( - processor, - frames_to_send=frames_to_send, - expected_down_frames=expected_down_frames, - ) - - self.assertEqual(len(received_updates), 1) - message = received_updates[0].messages[0] - self.assertEqual( - message.content, "Commas, colons: semicolons; quotes' and\" double quotes!" - ) - - async def test_complex_mixed_case(self): - """Test a complex mix of patterns to ensure robustness""" - processor = AssistantTranscriptProcessor() - - received_updates = [] - - @processor.event_handler("on_transcript_update") - async def handle_update(proc, frame: TranscriptionUpdateFrame): - received_updates.append(frame) - - # Complex mixed case with various patterns - frames_to_send = [ - BotStartedSpeakingFrame(), - SleepFrame(sleep=0.1), - # Pre-spaced fragments - TTSTextFrame(text="Hello"), - TTSTextFrame(text=" there"), - TTSTextFrame(text="!"), - # Sentence boundary - TTSTextFrame(text=" I'm"), - TTSTextFrame(text=" testing"), - TTSTextFrame(text=" spacing"), - TTSTextFrame(text="."), - # Word-by-word fragments - TTSTextFrame(text="Does"), - TTSTextFrame(text="this"), - TTSTextFrame(text="work"), - TTSTextFrame(text="correctly"), - TTSTextFrame(text="?"), - # Mixed punctuation and spacing - TTSTextFrame(text=" Let's"), - TTSTextFrame(text=" see:"), - TTSTextFrame(text="commas"), - TTSTextFrame(text=","), - TTSTextFrame(text=" semicolons"), - TTSTextFrame(text=";"), - TTSTextFrame(text=" and"), - TTSTextFrame(text=" quotes"), - TTSTextFrame(text="'"), - TTSTextFrame(text="!"), - BotStoppedSpeakingFrame(), - ] - - expected_down_frames = [ - BotStartedSpeakingFrame, - BotStoppedSpeakingFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TranscriptionUpdateFrame, - ] - - await run_test( - processor, - frames_to_send=frames_to_send, - expected_down_frames=expected_down_frames, - ) - - self.assertEqual(len(received_updates), 1) - message = received_updates[0].messages[0] - expected = "Hello there! I'm testing spacing. Does this work correctly? Let's see: commas, semicolons; and quotes'!" - self.assertEqual(message.content, expected) - - async def test_multiple_consecutive_punctuation(self): - """Test handling of multiple consecutive punctuation marks""" - processor = AssistantTranscriptProcessor() - - received_updates = [] - - @processor.event_handler("on_transcript_update") - async def handle_update(proc, frame: TranscriptionUpdateFrame): - received_updates.append(frame) - - frames_to_send = [ - BotStartedSpeakingFrame(), - SleepFrame(sleep=0.1), - TTSTextFrame(text="Wow"), - TTSTextFrame(text="!"), - TTSTextFrame(text="!"), - TTSTextFrame(text="!"), - TTSTextFrame(text=" That's"), - TTSTextFrame(text=" amazing"), - TTSTextFrame(text="..."), - TTSTextFrame(text=" Don't"), - TTSTextFrame(text=" you"), - TTSTextFrame(text=" think"), - TTSTextFrame(text="?"), - TTSTextFrame(text="?"), - BotStoppedSpeakingFrame(), - ] - - expected_down_frames = [ - BotStartedSpeakingFrame, - BotStoppedSpeakingFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TranscriptionUpdateFrame, - ] - - await run_test( - processor, - frames_to_send=frames_to_send, - expected_down_frames=expected_down_frames, - ) - - self.assertEqual(len(received_updates), 1) - message = received_updates[0].messages[0] - self.assertEqual(message.content, "Wow!!! That's amazing... Don't you think??") From c57fa93a700206885ea01a42414391708e58792d Mon Sep 17 00:00:00 2001 From: Lucas Rothman Date: Mon, 17 Mar 2025 16:22:36 -0700 Subject: [PATCH 044/132] Renamed to sample_rate --- src/pipecat/services/tavus.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pipecat/services/tavus.py b/src/pipecat/services/tavus.py index abb8b921d..9083dd53d 100644 --- a/src/pipecat/services/tavus.py +++ b/src/pipecat/services/tavus.py @@ -37,7 +37,7 @@ class TavusVideoService(AIService): replica_id: str, persona_id: str = "pipecat0", # Use `pipecat0` so that your TTS voice is used in place of the Tavus persona session: aiohttp.ClientSession, - output_sample_rate: int = 16000, + sample_rate: int = 16000, **kwargs, ) -> None: super().__init__(**kwargs) @@ -45,7 +45,7 @@ class TavusVideoService(AIService): self._replica_id = replica_id self._persona_id = persona_id self._session = session - self._output_sample_rate = output_sample_rate + self._sample_rate = sample_rate self._conversation_id: str @@ -96,7 +96,7 @@ class TavusVideoService(AIService): async def _encode_audio_and_send(self, audio: bytes, in_rate: int, done: bool) -> None: """Encodes audio to base64 and sends it to Tavus""" if not done: - audio = await self._resampler.resample(audio, in_rate, self._output_sample_rate) + audio = await self._resampler.resample(audio, in_rate, self._sample_rate) audio_base64 = base64.b64encode(audio).decode("utf-8") logger.trace(f"{self}: sending {len(audio)} bytes") await self._send_audio_message(audio_base64, done=done) @@ -110,7 +110,7 @@ class TavusVideoService(AIService): elif isinstance(frame, TTSAudioRawFrame): await self._encode_audio_and_send(frame.audio, frame.sample_rate, done=False) elif isinstance(frame, TTSStoppedFrame): - await self._encode_audio_and_send(b"\x00", self._output_sample_rate, done=True) + await self._encode_audio_and_send(b"\x00", self._sample_rate, done=True) await self.stop_ttfb_metrics() await self.stop_processing_metrics() elif isinstance(frame, StartInterruptionFrame): @@ -139,7 +139,7 @@ class TavusVideoService(AIService): "inference_id": self._current_idx_str, "audio": audio_base64, "done": done, - "sample_rate": self._output_sample_rate, + "sample_rate": self._sample_rate, }, } ) From 188677e601658e3e6b0d4c389e8b30e90837431f Mon Sep 17 00:00:00 2001 From: Adnan Siddiquei Date: Tue, 18 Mar 2025 10:35:22 +0000 Subject: [PATCH 045/132] Added 4 new languages: FR, PT, RU, ZH, HI. --- src/pipecat/services/neuphonic.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/pipecat/services/neuphonic.py b/src/pipecat/services/neuphonic.py index a2ea98f42..b935885b6 100644 --- a/src/pipecat/services/neuphonic.py +++ b/src/pipecat/services/neuphonic.py @@ -49,6 +49,11 @@ def language_to_neuphonic_lang_code(language: Language) -> Optional[str]: Language.ES: "es", Language.NL: "nl", Language.AR: "ar", + Language.FR: "fr", + Language.PT: "pt", + Language.RU: "ru", + Language.HI: "HI", + Language.ZH: "zh", } result = BASE_LANGUAGES.get(language) From e731a0d41f96860aaf053e1a64f172b6e81b31b2 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 17 Mar 2025 22:44:12 -0400 Subject: [PATCH 046/132] Add PairPatternAggregator --- .../utils/text/pattern_pair_aggregator.py | 262 ++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 src/pipecat/utils/text/pattern_pair_aggregator.py diff --git a/src/pipecat/utils/text/pattern_pair_aggregator.py b/src/pipecat/utils/text/pattern_pair_aggregator.py new file mode 100644 index 000000000..734e6a9bd --- /dev/null +++ b/src/pipecat/utils/text/pattern_pair_aggregator.py @@ -0,0 +1,262 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import re +from typing import Callable, Dict, Optional, Pattern, Tuple, Union + +from loguru import logger + +from pipecat.utils.string import match_endofsentence +from pipecat.utils.text.base_text_aggregator import BaseTextAggregator + + +class PatternMatch: + """Represents a matched pattern pair with its content. + + A PatternMatch object is created when a complete pattern pair is found + in the text. It contains information about which pattern was matched, + the full matched text (including start and end patterns), and the + content between the patterns. + + Attributes: + pattern_id: The identifier of the matched pattern pair. + full_match: The complete text including start and end patterns. + content: The text content between the start and end patterns. + """ + + def __init__(self, pattern_id: str, full_match: str, content: str): + """Initialize a pattern match. + + Args: + pattern_id: ID of the pattern pair. + full_match: Complete matched text including start and end patterns. + content: Content between the start and end patterns. + """ + self.pattern_id = pattern_id + self.full_match = full_match + self.content = content + + def __str__(self) -> str: + """Return a string representation of the pattern match. + + Returns: + A string describing the pattern match. + """ + return f"PatternMatch(id={self.pattern_id}, content={self.content})" + + +class PatternPairAggregator(BaseTextAggregator): + """Aggregator that identifies and processes content between pattern pairs. + + This aggregator buffers text until it can identify complete pattern pairs + (defined by start and end patterns), processes the content between these + patterns using registered handlers, and returns text at sentence boundaries. + It's particularly useful for processing structured content in streaming text, + such as XML tags, markdown formatting, or custom delimiters. + + The aggregator ensures that patterns spanning multiple text chunks are + correctly identified and handles cases where patterns contain sentence + boundaries. + """ + + def __init__(self): + """Initialize the pattern pair aggregator. + + Creates an empty aggregator with no patterns or handlers registered. + """ + self._text = "" + self._patterns = {} + self._handlers = {} + + @property + def text(self) -> str: + """Get the currently buffered text. + + Returns: + The current text buffer content. + """ + return self._text + + def add_pattern_pair( + self, pattern_id: str, start_pattern: str, end_pattern: str, remove_match: bool = True + ) -> "PatternPairAggregator": + """Add a pattern pair to detect in the text. + + Registers a new pattern pair with a unique identifier. The aggregator + will look for text that starts with the start pattern and ends with + the end pattern, and treat the content between them as a match. + + Args: + pattern_id: Unique identifier for this pattern pair. + start_pattern: Pattern that marks the beginning of content. + end_pattern: Pattern that marks the end of content. + remove_match: Whether to remove the matched content from the text. + + Returns: + Self for method chaining. + """ + self._patterns[pattern_id] = { + "start": start_pattern, + "end": end_pattern, + "remove_match": remove_match, + } + return self + + def on_pattern_match( + self, pattern_id: str, handler: Callable[[PatternMatch], None] + ) -> "PatternPairAggregator": + """Register a handler for when a pattern pair is matched. + + The handler will be called whenever a complete match for the + specified pattern ID is found in the text. + + Args: + pattern_id: ID of the pattern pair to match. + handler: Function to call when pattern is matched. + The function should accept a PatternMatch object. + + Returns: + Self for method chaining. + """ + self._handlers[pattern_id] = handler + return self + + def _process_complete_patterns(self, text: str) -> Tuple[str, bool]: + """Process all complete pattern pairs in the text. + + Searches for all complete pattern pairs in the text, calls the + appropriate handlers, and optionally removes the matches. + + Args: + text: The text to process. + + Returns: + Tuple of (processed_text, was_modified) where: + - processed_text is the text after processing patterns + - was_modified indicates whether any changes were made + """ + processed_text = text + modified = False + + for pattern_id, pattern_info in self._patterns.items(): + # Escape special regex characters in the patterns + start = re.escape(pattern_info["start"]) + end = re.escape(pattern_info["end"]) + remove_match = pattern_info["remove_match"] + + # Create regex to match from start pattern to end pattern + # The .*? is non-greedy to handle nested patterns + regex = f"{start}(.*?){end}" + + # Find all matches + match_iter = re.finditer(regex, processed_text, re.DOTALL) + matches = list(match_iter) # Convert to list for safe iteration + + for match in matches: + content = match.group(1) # Content between patterns + full_match = match.group(0) # Full match including patterns + + # Create pattern match object + pattern_match = PatternMatch( + pattern_id=pattern_id, full_match=full_match, content=content + ) + + # Call the appropriate handler if registered + if pattern_id in self._handlers: + try: + self._handlers[pattern_id](pattern_match) + except Exception as e: + logger.error(f"Error in pattern handler for {pattern_id}: {e}") + + # Remove the pattern from the text if configured + if remove_match: + processed_text = processed_text.replace(full_match, "", 1) + modified = True + + return processed_text, modified + + def _has_incomplete_patterns(self, text: str) -> bool: + """Check if text contains incomplete pattern pairs. + + Determines whether the text contains any start patterns without + matching end patterns, which would indicate incomplete content. + + Args: + text: The text to check. + + Returns: + True if there are incomplete patterns, False otherwise. + """ + for pattern_id, pattern_info in self._patterns.items(): + start = pattern_info["start"] + end = pattern_info["end"] + + # Count occurrences + start_count = text.count(start) + end_count = text.count(end) + + # If there are more starts than ends, we have incomplete patterns + if start_count > end_count: + return True + + return False + + def aggregate(self, text: str) -> Optional[str]: + """Aggregate text and process pattern pairs. + + This method adds the new text to the buffer, processes any complete pattern + pairs, and returns processed text up to sentence boundaries if possible. + If there are incomplete patterns (start without matching end), it will + continue buffering text. + + Args: + text: New text to add to the buffer. + + Returns: + Processed text up to a sentence boundary, or None if more + text is needed to form a complete sentence or pattern. + """ + # Add new text to buffer + self._text += text + + # Process any complete patterns in the buffer + processed_text, modified = self._process_complete_patterns(self._text) + + # Only update the buffer if modifications were made + if modified: + self._text = processed_text + + # Check if we have incomplete patterns + if self._has_incomplete_patterns(self._text): + # Still waiting for complete patterns + return None + + # Find sentence boundary if no incomplete patterns + eos_marker = match_endofsentence(self._text) + if eos_marker: + # Extract text up to the sentence boundary + result = self._text[:eos_marker] + self._text = self._text[eos_marker:] + return result + + # No complete sentence found yet + return None + + def handle_interruption(self): + """Handle interruptions by clearing the buffer. + + Called when an interruption occurs in the processing pipeline, + to reset the state and discard any partially aggregated text. + """ + self._text = "" + + def reset(self): + """Clear the internally aggregated text. + + Resets the aggregator to its initial state, discarding any + buffered text. + """ + self._text = "" From ddcc1fbb2fc28b0547c5025aaecba4d8f04cd89f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 17 Mar 2025 23:05:22 -0400 Subject: [PATCH 047/132] Add foundational example 35 --- examples/foundational/35-voice-switching.py | 192 ++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 examples/foundational/35-voice-switching.py diff --git a/examples/foundational/35-voice-switching.py b/examples/foundational/35-voice-switching.py new file mode 100644 index 000000000..5dd986bc3 --- /dev/null +++ b/examples/foundational/35-voice-switching.py @@ -0,0 +1,192 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.utils.text.pattern_pair_aggregator import PatternMatch, PatternPairAggregator + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + +# Define voice IDs +VOICE_IDS = { + "narrator": "c45bc5ec-dc68-4feb-8829-6e6b2748095d", # Narrator voice + "female": "71a7ad14-091c-4e8e-a314-022ece01c121", # Female character voice + "male": "7cf0e2b1-8daf-4fe4-89ad-f6039398f359", # Male character voice +} + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Storytelling Bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + # Initialize TTS with narrator voice as default + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id=VOICE_IDS["narrator"], + ) + + # Create pattern pair aggregator for voice switching + pattern_aggregator = PatternPairAggregator() + + # Add pattern for voice switching + pattern_aggregator.add_pattern_pair( + pattern_id="voice_tag", + start_pattern="", + end_pattern="", + remove_match=True, + ) + + # Register handler for voice switching + def on_voice_tag(match: PatternMatch): + voice_name = match.content.strip().lower() + if voice_name in VOICE_IDS: + voice_id = VOICE_IDS[voice_name] + tts.set_voice(voice_id) + logger.info(f"Switched to {voice_name} voice") + else: + logger.warning(f"Unknown voice: {voice_name}") + + pattern_aggregator.on_pattern_match("voice_tag", on_voice_tag) + + # Set the pattern aggregator on the TTS service + tts._text_aggregator = pattern_aggregator + + # Initialize LLM + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + # System prompt for storytelling with voice switching + system_prompt = """You are an engaging storyteller that uses different voices to bring stories to life. + +You have three voices to use, but each has a specific purpose: + +narrator +This is the default narrator voice. Use this for all narration, descriptions, and non-dialogue text. + +female +Use this ONLY for direct speech by female characters (just the quoted text). + +male +Use this ONLY for direct speech by male characters (just the quoted text). + +IMPORTANT: Switch back to narrator voice immediately after character dialogue. + +Here's an EXAMPLE of correct voice usage: + +narrator +Sarah spotted her old friend across the café. She couldn't believe her eyes. + +female +"Jacob! It's been so long!" + +narrator +Sarah exclaimed, jumping up from her seat with a radiant smile. + +male +"Sarah, is it really you? I can't believe it!" + +narrator +Jacob replied, grinning widely as he walked over to her. The two friends embraced warmly, as if trying to make up for all the years spent apart. + +female +"What are you doing in town? Last I heard you were in Seattle." + +narrator +She asked, gesturing for him to join her at the table. + +FOLLOW THESE RULES: +1. Always begin with the narrator voice +2. Only use character voices for the EXACT words they speak (in quotes) +3. SWITCH BACK to narrator voice for speech tags and all other text +4. Begin by asking what kind of story the user would like to hear +5. Create engaging dialogue with distinct characters + +Remember: Use narrator voice for EVERYTHING except the actual quoted dialogue.""" + + # Set up LLM context + messages = [ + { + "role": "system", + "content": system_prompt, + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + # Create pipeline + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, # TTS with pattern aggregator + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + logger.info(f"First participant joined: {participant['id']}") + await transport.capture_participant_transcription(participant["id"]) + + # Start conversation - empty prompt to let LLM follow system instructions + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + logger.info(f"Participant left: {participant['id']}") + await task.cancel() + + logger.info(f"Starting storytelling bot at: {room_url}") + logger.info("Join the room to interact with the bot!") + + runner = PipelineRunner() + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) From 6ec4052f29271d8fcbade96b77401c2cc86f0033 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 17 Mar 2025 23:17:41 -0400 Subject: [PATCH 048/132] Add CHANGELOG entries --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 406e9a304..c43971da8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added new `PatternPairAggregator` that extends `BaseTextAggregator` to + identify content between matching pattern pairs in streamed text. This allows + for detection and processing of structured content like XML-style tags that + may span across multiple text chunks or sentence boundaries. + - Added new `BaseTextAggregator`. Text aggregators are used by the TTS service to aggregate LLM tokens and decide when the aggregated text should be pushed to the TTS service. It also allows for the text to be manipulated while it's @@ -124,6 +129,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update the `34-audio-recording.py` example to include an STT processor. +- Added foundational example `35-voice-switching.py` showing how to use the new + `PatternPairAggregator`. + - Added a Pipecat Cloud deployment example to the `examples` directory. - Removed foundational examples 28b and 28c as the TranscriptProcessor no From 2dee882710a771509afd9b4a6db7c4777a431b33 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 17 Mar 2025 23:29:08 -0400 Subject: [PATCH 049/132] Add unit tests --- tests/test_pattern_pair_aggregator.py | 147 ++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 tests/test_pattern_pair_aggregator.py diff --git a/tests/test_pattern_pair_aggregator.py b/tests/test_pattern_pair_aggregator.py new file mode 100644 index 000000000..e1086e577 --- /dev/null +++ b/tests/test_pattern_pair_aggregator.py @@ -0,0 +1,147 @@ +# +# Copyright (c) 2024-2025 Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import unittest +from unittest.mock import Mock + +from pipecat.utils.text.pattern_pair_aggregator import PatternMatch, PatternPairAggregator + + +class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.aggregator = PatternPairAggregator() + self.test_handler = Mock() + + # Add a test pattern + self.aggregator.add_pattern_pair( + pattern_id="test_pattern", + start_pattern="", + end_pattern="", + remove_match=True, + ) + + # Register the mock handler + self.aggregator.on_pattern_match("test_pattern", self.test_handler) + + async def test_pattern_match_and_removal(self): + # First part doesn't complete the pattern + result = self.aggregator.aggregate("Hello pattern") + self.assertIsNone(result) + self.assertEqual(self.aggregator.text, "Hello pattern") + + # Second part completes the pattern and includes an exclamation point + result = self.aggregator.aggregate(" content!") + + # Verify the handler was called with correct PatternMatch object + self.test_handler.assert_called_once() + call_args = self.test_handler.call_args[0][0] + self.assertIsInstance(call_args, PatternMatch) + self.assertEqual(call_args.pattern_id, "test_pattern") + self.assertEqual(call_args.full_match, "pattern content") + self.assertEqual(call_args.content, "pattern content") + + # The exclamation point should be treated as a sentence boundary, + # so the result should include just text up to and including "!" + self.assertEqual(result, "Hello !") + + # Next sentence should be processed separately + result = self.aggregator.aggregate(" This is another sentence.") + self.assertEqual(result, " This is another sentence.") + + # Buffer should be empty after returning a complete sentence + self.assertEqual(self.aggregator.text, "") + + async def test_incomplete_pattern(self): + # Add text with incomplete pattern + result = self.aggregator.aggregate("Hello pattern content") + + # No complete pattern yet, so nothing should be returned + self.assertIsNone(result) + + # The handler should not be called yet + self.test_handler.assert_not_called() + + # Buffer should contain the incomplete text + self.assertEqual(self.aggregator.text, "Hello pattern content") + + # Reset and confirm buffer is cleared + self.aggregator.reset() + self.assertEqual(self.aggregator.text, "") + + async def test_multiple_patterns(self): + # Set up multiple patterns and handlers + voice_handler = Mock() + emphasis_handler = Mock() + + self.aggregator.add_pattern_pair( + pattern_id="voice", start_pattern="", end_pattern="", remove_match=True + ) + + self.aggregator.add_pattern_pair( + pattern_id="emphasis", + start_pattern="", + end_pattern="", + remove_match=False, # Keep emphasis tags + ) + + self.aggregator.on_pattern_match("voice", voice_handler) + self.aggregator.on_pattern_match("emphasis", emphasis_handler) + + # Test with multiple patterns in one text block + text = "Hello female I am very excited to meet you!" + result = self.aggregator.aggregate(text) + + # Both handlers should be called with correct data + voice_handler.assert_called_once() + voice_match = voice_handler.call_args[0][0] + self.assertEqual(voice_match.pattern_id, "voice") + self.assertEqual(voice_match.content, "female") + + emphasis_handler.assert_called_once() + emphasis_match = emphasis_handler.call_args[0][0] + self.assertEqual(emphasis_match.pattern_id, "emphasis") + self.assertEqual(emphasis_match.content, "very") + + # Voice pattern should be removed, emphasis pattern should remain + self.assertEqual(result, "Hello I am very excited to meet you!") + + # Buffer should be empty + self.assertEqual(self.aggregator.text, "") + + async def test_handle_interruption(self): + # Start with incomplete pattern + result = self.aggregator.aggregate("Hello pattern") + self.assertIsNone(result) + + # Simulate interruption + self.aggregator.handle_interruption() + + # Buffer should be cleared + self.assertEqual(self.aggregator.text, "") + + # Handler should not have been called + self.test_handler.assert_not_called() + + async def test_pattern_across_sentences(self): + # Test pattern that spans multiple sentences + result = self.aggregator.aggregate("Hello This is sentence one.") + + # First sentence contains start of pattern but no end, so no complete pattern yet + self.assertIsNone(result) + + # Add second part with pattern end + result = self.aggregator.aggregate(" This is sentence two. Final sentence.") + + # Handler should be called with entire content + self.test_handler.assert_called_once() + call_args = self.test_handler.call_args[0][0] + self.assertEqual(call_args.content, "This is sentence one. This is sentence two.") + + # Pattern should be removed, resulting in text with sentences merged + self.assertEqual(result, "Hello Final sentence.") + + # Buffer should be empty + self.assertEqual(self.aggregator.text, "") From b28276446d15e565645f0a024a01b2893711b09a Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 18 Mar 2025 07:47:18 -0400 Subject: [PATCH 050/132] Code review feedback --- CHANGELOG.md | 5 +- ....py => 35-pattern-pair-voice-switching.py} | 56 ++++++++++++++++--- .../utils/text/pattern_pair_aggregator.py | 2 +- 3 files changed, 52 insertions(+), 11 deletions(-) rename examples/foundational/{35-voice-switching.py => 35-pattern-pair-voice-switching.py} (77%) diff --git a/CHANGELOG.md b/CHANGELOG.md index c43971da8..6d200ab87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,7 +130,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update the `34-audio-recording.py` example to include an STT processor. - Added foundational example `35-voice-switching.py` showing how to use the new - `PatternPairAggregator`. + `PatternPairAggregator`. This example shows how to encode information for the + LLM to instruct TTS voice changes, but this can be used to encode any + information into the LLM response, which you want to parse and use in other + parts of your application. - Added a Pipecat Cloud deployment example to the `examples` directory. diff --git a/examples/foundational/35-voice-switching.py b/examples/foundational/35-pattern-pair-voice-switching.py similarity index 77% rename from examples/foundational/35-voice-switching.py rename to examples/foundational/35-pattern-pair-voice-switching.py index 5dd986bc3..bb9587706 100644 --- a/examples/foundational/35-voice-switching.py +++ b/examples/foundational/35-pattern-pair-voice-switching.py @@ -4,6 +4,46 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Pattern Pair Voice Switching Example with Pipecat. + +This example demonstrates how to use the PatternPairAggregator to dynamically switch +between different voices in a storytelling application. It showcases how pattern matching +can be used to control TTS behavior in streaming text from an LLM. + +The example: + 1. Sets up a storytelling bot with three distinct voices (narrator, male, female) + 2. Uses pattern pairs (name) to trigger voice switching + 3. Processes the patterns in real-time as text streams from the LLM + 4. Removes the pattern tags before sending text to TTS + +The PatternPairAggregator: + - Buffers text until complete patterns are detected + - Identifies content between start/end pattern pairs + - Triggers callbacks when patterns are matched + - Processes patterns that may span across multiple text chunks + - Returns processed text at sentence boundaries + +Example usage (run from pipecat root directory): + $ pip install "pipecat-ai[daily,openai,cartesia,silero]" + $ pip install -r dev-requirements.txt + $ python examples/foundational/35-pattern-pair-voice-switching.py + +Requirements: + - OpenAI API key (for GPT-4o) + - Cartesia API key (for text-to-speech) + - Daily API key (for video/audio transport) + + Environment variables (.env file): + OPENAI_API_KEY=your_openai_key + CARTESIA_API_KEY=your_cartesia_key + DAILY_API_KEY=your_daily_key + +Note: + This example shows one application of PatternPairAggregator (voice switching), + but the same approach can be used for various pattern-based text processing needs, + such as formatting instructions, command recognition, or structured data extraction. +""" + import asyncio import os import sys @@ -43,7 +83,7 @@ async def main(): transport = DailyTransport( room_url, token, - "Storytelling Bot", + "Multi-voice storyteller", DailyParams( audio_out_enabled=True, transcription_enabled=True, @@ -52,12 +92,6 @@ async def main(): ), ) - # Initialize TTS with narrator voice as default - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id=VOICE_IDS["narrator"], - ) - # Create pattern pair aggregator for voice switching pattern_aggregator = PatternPairAggregator() @@ -81,8 +115,12 @@ async def main(): pattern_aggregator.on_pattern_match("voice_tag", on_voice_tag) - # Set the pattern aggregator on the TTS service - tts._text_aggregator = pattern_aggregator + # Initialize TTS with narrator voice as default + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id=VOICE_IDS["narrator"], + text_aggregator=pattern_aggregator, + ) # Initialize LLM llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/src/pipecat/utils/text/pattern_pair_aggregator.py b/src/pipecat/utils/text/pattern_pair_aggregator.py index 734e6a9bd..86f87103b 100644 --- a/src/pipecat/utils/text/pattern_pair_aggregator.py +++ b/src/pipecat/utils/text/pattern_pair_aggregator.py @@ -5,7 +5,7 @@ # import re -from typing import Callable, Dict, Optional, Pattern, Tuple, Union +from typing import Callable, Optional, Tuple from loguru import logger From 4303ed4991a930bb4c4d3d268ea7991db3f78e56 Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Tue, 18 Mar 2025 20:58:21 +0530 Subject: [PATCH 051/132] rename service --- CHANGELOG.md | 2 +- .../foundational/14p-function-calling-gemini-vertex-ai.py | 6 +++--- src/pipecat/services/google/google.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3ee2646b..74cd0b15f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,7 +88,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `AzureRealtimeBetaLLMService` to support Azure's OpeanAI Realtime API. Added foundational example `19a-azure-realtime-beta.py`. -- Introduced `GoogleVertexAIService`, a new class for integrating with Vertex AI +- Introduced `GoogleVertexLLMService`, a new class for integrating with Vertex AI Gemini models. Added foundational example `14p-function-calling-gemini-vertex-ai.py`. diff --git a/examples/foundational/14p-function-calling-gemini-vertex-ai.py b/examples/foundational/14p-function-calling-gemini-vertex-ai.py index 68608e932..d64bae89d 100644 --- a/examples/foundational/14p-function-calling-gemini-vertex-ai.py +++ b/examples/foundational/14p-function-calling-gemini-vertex-ai.py @@ -21,7 +21,7 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.services.elevenlabs import ElevenLabsTTSService -from pipecat.services.google import GoogleVertexAIService +from pipecat.services.google import GoogleVertexLLMService from pipecat.services.openai import OpenAILLMContext from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -62,9 +62,9 @@ async def main(): voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), ) - llm = GoogleVertexAIService( + llm = GoogleVertexLLMService( # credentials="", - params=GoogleVertexAIService.InputParams( + params=GoogleVertexLLMService.InputParams( project_id="", ) ) diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index f741154b9..f9a8d4894 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -1334,7 +1334,7 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService): ) -class GoogleVertexAIService(OpenAILLMService): +class GoogleVertexLLMService(OpenAILLMService): """Implements inference with Google's AI models via Vertex AI while maintaining OpenAI API compatibility. Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library From 32609b1132595340fee838f5762ee32f2f9c0296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 13 Mar 2025 18:04:25 -0700 Subject: [PATCH 052/132] event handlers are now executed in separate tasks --- CHANGELOG.md | 5 +++ src/pipecat/pipeline/runner.py | 6 ++++ src/pipecat/pipeline/task.py | 4 +++ src/pipecat/processors/frame_processor.py | 1 + .../transports/network/fastapi_websocket.py | 18 ++++++++-- .../transports/network/websocket_client.py | 28 +++++++++++++--- .../transports/network/websocket_server.py | 19 +++++++++-- src/pipecat/transports/services/daily.py | 24 +++++++++++--- src/pipecat/transports/services/livekit.py | 33 ++++++++++++++++--- src/pipecat/utils/base_object.py | 29 ++++++++++++++++ 10 files changed, 150 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d200ab87..0a5d29e90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- All event handlers are now executed in separate tasks in order to prevent + blocking the pipeline. It is possible that event handlers take some time to + execute in which case the pipeline would be blocked waiting for the event + handler to complete. + - Updated `TranscriptProcessor` to support text output from `OpenAIRealtimeBetaLLMService`. diff --git a/src/pipecat/pipeline/runner.py b/src/pipecat/pipeline/runner.py index 3209fa92a..7ac07064f 100644 --- a/src/pipecat/pipeline/runner.py +++ b/src/pipecat/pipeline/runner.py @@ -40,12 +40,18 @@ class PipelineRunner(BaseObject): task.set_event_loop(self._loop) await task.run() del self._tasks[task.name] + + # Cleanup base object. + await self.cleanup() + # If we are cancelling through a signal, make sure we wait for it so # everything gets cleaned up nicely. if self._sig_task: await self._sig_task + if self._force_gc: self._gc_collect() + logger.debug(f"Runner {self} finished running {task}") async def stop_when_done(self): diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index c10b33189..12c56ef3f 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -354,6 +354,10 @@ class PipelineTask(BaseTask): self._pipeline_end_event.clear() async def _cleanup(self, cleanup_pipeline: bool): + # Cleanup base object. + await self.cleanup() + + # Cleanup pipeline processors. await self._source.cleanup() if cleanup_pipeline: await self._pipeline.cleanup() diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 6a1669ff1..847cdf175 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -164,6 +164,7 @@ class FrameProcessor(BaseObject): await self._task_manager.wait_for_task(task, timeout) async def cleanup(self): + await super().cleanup() await self.__cancel_input_task() await self.__cancel_push_task() diff --git a/src/pipecat/transports/network/fastapi_websocket.py b/src/pipecat/transports/network/fastapi_websocket.py index bfb1e8146..937dda80c 100644 --- a/src/pipecat/transports/network/fastapi_websocket.py +++ b/src/pipecat/transports/network/fastapi_websocket.py @@ -102,11 +102,13 @@ class FastAPIWebsocketClient: class FastAPIWebsocketInputTransport(BaseInputTransport): def __init__( self, + transport: BaseTransport, client: FastAPIWebsocketClient, params: FastAPIWebsocketParams, **kwargs, ): super().__init__(params, **kwargs) + self._transport = transport self._client = client self._params = params self._receive_task = None @@ -139,6 +141,10 @@ class FastAPIWebsocketInputTransport(BaseInputTransport): await self._stop_tasks() await self._client.disconnect() + async def cleanup(self): + await super().cleanup() + await self._transport.cleanup() + async def _receive_messages(self): try: async for message in self._client.receive(): @@ -165,11 +171,14 @@ class FastAPIWebsocketInputTransport(BaseInputTransport): class FastAPIWebsocketOutputTransport(BaseOutputTransport): def __init__( self, + transport: BaseTransport, client: FastAPIWebsocketClient, params: FastAPIWebsocketParams, **kwargs, ): super().__init__(params, **kwargs) + + self._transport = transport self._client = client self._params = params @@ -194,6 +203,10 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport): await super().cancel(frame) await self._client.disconnect() + async def cleanup(self): + await super().cleanup() + await self._transport.cleanup() + async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -266,6 +279,7 @@ class FastAPIWebsocketTransport(BaseTransport): output_name: Optional[str] = None, ): super().__init__(input_name=input_name, output_name=output_name) + self._params = params self._callbacks = FastAPIWebsocketCallbacks( @@ -278,10 +292,10 @@ class FastAPIWebsocketTransport(BaseTransport): self._client = FastAPIWebsocketClient(websocket, is_binary, self._callbacks) self._input = FastAPIWebsocketInputTransport( - self._client, self._params, name=self._input_name + self, self._client, self._params, name=self._input_name ) self._output = FastAPIWebsocketOutputTransport( - self._client, self._params, name=self._output_name + self, self._client, self._params, name=self._output_name ) # Register supported handlers. The user will only be able to register diff --git a/src/pipecat/transports/network/websocket_client.py b/src/pipecat/transports/network/websocket_client.py index eb2b5cfb8..11a000e69 100644 --- a/src/pipecat/transports/network/websocket_client.py +++ b/src/pipecat/transports/network/websocket_client.py @@ -118,9 +118,15 @@ class WebsocketClientSession: class WebsocketClientInputTransport(BaseInputTransport): - def __init__(self, session: WebsocketClientSession, params: WebsocketClientParams): + def __init__( + self, + transport: BaseTransport, + session: WebsocketClientSession, + params: WebsocketClientParams, + ): super().__init__(params) + self._transport = transport self._session = session self._params = params @@ -138,6 +144,10 @@ class WebsocketClientInputTransport(BaseInputTransport): await super().cancel(frame) await self._session.disconnect() + async def cleanup(self): + await super().cleanup() + await self._transport.cleanup() + async def on_message(self, websocket, message): frame = await self._params.serializer.deserialize(message) if not frame: @@ -149,9 +159,15 @@ class WebsocketClientInputTransport(BaseInputTransport): class WebsocketClientOutputTransport(BaseOutputTransport): - def __init__(self, session: WebsocketClientSession, params: WebsocketClientParams): + def __init__( + self, + transport: BaseTransport, + session: WebsocketClientSession, + params: WebsocketClientParams, + ): super().__init__(params) + self._transport = transport self._session = session self._params = params @@ -178,6 +194,10 @@ class WebsocketClientOutputTransport(BaseOutputTransport): await super().cancel(frame) await self._session.disconnect() + async def cleanup(self): + await super().cleanup() + await self._transport.cleanup() + async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame): await self._write_frame(frame) @@ -250,12 +270,12 @@ class WebsocketClientTransport(BaseTransport): def input(self) -> WebsocketClientInputTransport: if not self._input: - self._input = WebsocketClientInputTransport(self._session, self._params) + self._input = WebsocketClientInputTransport(self, self._session, self._params) return self._input def output(self) -> WebsocketClientOutputTransport: if not self._output: - self._output = WebsocketClientOutputTransport(self._session, self._params) + self._output = WebsocketClientOutputTransport(self, self._session, self._params) return self._output async def _on_connected(self, websocket): diff --git a/src/pipecat/transports/network/websocket_server.py b/src/pipecat/transports/network/websocket_server.py index 19ebe4a45..e542342a2 100644 --- a/src/pipecat/transports/network/websocket_server.py +++ b/src/pipecat/transports/network/websocket_server.py @@ -55,6 +55,7 @@ class WebsocketServerCallbacks(BaseModel): class WebsocketServerInputTransport(BaseInputTransport): def __init__( self, + transport: BaseTransport, host: str, port: int, params: WebsocketServerParams, @@ -63,6 +64,7 @@ class WebsocketServerInputTransport(BaseInputTransport): ): super().__init__(params, **kwargs) + self._transport = transport self._host = host self._port = port self._params = params @@ -102,6 +104,10 @@ class WebsocketServerInputTransport(BaseInputTransport): await self.cancel_task(self._server_task) self._server_task = None + async def cleanup(self): + await super().cleanup() + await self._transport.cleanup() + async def _server_task_handler(self): logger.info(f"Starting websocket server on {self._host}:{self._port}") async with websockets.serve(self._client_handler, self._host, self._port) as server: @@ -163,9 +169,10 @@ class WebsocketServerInputTransport(BaseInputTransport): class WebsocketServerOutputTransport(BaseOutputTransport): - def __init__(self, params: WebsocketServerParams, **kwargs): + def __init__(self, transport: BaseTransport, params: WebsocketServerParams, **kwargs): super().__init__(params, **kwargs) + self._transport = transport self._params = params self._websocket: Optional[websockets.WebSocketServerProtocol] = None @@ -189,6 +196,10 @@ class WebsocketServerOutputTransport(BaseOutputTransport): await self._params.serializer.setup(frame) self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2 + async def cleanup(self): + await super().cleanup() + await self._transport.cleanup() + async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -283,13 +294,15 @@ class WebsocketServerTransport(BaseTransport): def input(self) -> WebsocketServerInputTransport: if not self._input: self._input = WebsocketServerInputTransport( - self._host, self._port, self._params, self._callbacks, name=self._input_name + self, self._host, self._port, self._params, self._callbacks, name=self._input_name ) return self._input def output(self) -> WebsocketServerOutputTransport: if not self._output: - self._output = WebsocketServerOutputTransport(self._params, name=self._output_name) + self._output = WebsocketServerOutputTransport( + self, self._params, name=self._output_name + ) return self._output async def _on_client_connected(self, websocket): diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 873b13cd1..f4b83dfa7 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -811,9 +811,16 @@ class DailyInputTransport(BaseInputTransport): params: Configuration parameters. """ - def __init__(self, client: DailyTransportClient, params: DailyParams, **kwargs): + def __init__( + self, + transport: BaseTransport, + client: DailyTransportClient, + params: DailyParams, + **kwargs, + ): super().__init__(params, **kwargs) + self._transport = transport self._client = client self._params = params @@ -881,6 +888,7 @@ class DailyInputTransport(BaseInputTransport): async def cleanup(self): await super().cleanup() await self._client.cleanup() + await self._transport.cleanup() # # FrameProcessor @@ -971,9 +979,12 @@ class DailyOutputTransport(BaseOutputTransport): params: Configuration parameters. """ - def __init__(self, client: DailyTransportClient, params: DailyParams, **kwargs): + def __init__( + self, transport: BaseTransport, client: DailyTransportClient, params: DailyParams, **kwargs + ): super().__init__(params, **kwargs) + self._transport = transport self._client = client # Whether we have seen a StartFrame already. @@ -1008,6 +1019,7 @@ class DailyOutputTransport(BaseOutputTransport): async def cleanup(self): await super().cleanup() await self._client.cleanup() + await self._transport.cleanup() async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame): await self._client.send_message(frame) @@ -1109,12 +1121,16 @@ class DailyTransport(BaseTransport): def input(self) -> DailyInputTransport: if not self._input: - self._input = DailyInputTransport(self._client, self._params, name=self._input_name) + self._input = DailyInputTransport( + self, self._client, self._params, name=self._input_name + ) return self._input def output(self) -> DailyOutputTransport: if not self._output: - self._output = DailyOutputTransport(self._client, self._params, name=self._output_name) + self._output = DailyOutputTransport( + self, self._client, self._params, name=self._output_name + ) return self._output # diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py index 7018ea520..149ca4b7c 100644 --- a/src/pipecat/transports/services/livekit.py +++ b/src/pipecat/transports/services/livekit.py @@ -345,9 +345,17 @@ class LiveKitTransportClient: class LiveKitInputTransport(BaseInputTransport): - def __init__(self, client: LiveKitTransportClient, params: LiveKitParams, **kwargs): + def __init__( + self, + transport: BaseTransport, + client: LiveKitTransportClient, + params: LiveKitParams, + **kwargs, + ): super().__init__(params, **kwargs) + self._transport = transport self._client = client + self._audio_in_task = None self._vad_analyzer: Optional[VADAnalyzer] = params.vad_analyzer self._resampler = create_default_resampler() @@ -377,6 +385,10 @@ class LiveKitInputTransport(BaseInputTransport): if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled): await self.cancel_task(self._audio_in_task) + async def cleanup(self): + await super().cleanup() + await self._transport.cleanup() + async def push_app_message(self, message: Any, sender: str): frame = LiveKitTransportMessageUrgentFrame(message=message, participant_id=sender) await self.push_frame(frame) @@ -414,8 +426,15 @@ class LiveKitInputTransport(BaseInputTransport): class LiveKitOutputTransport(BaseOutputTransport): - def __init__(self, client: LiveKitTransportClient, params: LiveKitParams, **kwargs): + def __init__( + self, + transport: BaseTransport, + client: LiveKitTransportClient, + params: LiveKitParams, + **kwargs, + ): super().__init__(params, **kwargs) + self._transport = transport self._client = client async def start(self, frame: StartFrame): @@ -433,6 +452,10 @@ class LiveKitOutputTransport(BaseOutputTransport): await super().cancel(frame) await self._client.disconnect() + async def cleanup(self): + await super().cleanup() + await self._transport.cleanup() + async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame): if isinstance(frame, (LiveKitTransportMessageFrame, LiveKitTransportMessageUrgentFrame)): await self._client.send_data(frame.message.encode(), frame.participant_id) @@ -499,13 +522,15 @@ class LiveKitTransport(BaseTransport): def input(self) -> LiveKitInputTransport: if not self._input: - self._input = LiveKitInputTransport(self._client, self._params, name=self._input_name) + self._input = LiveKitInputTransport( + self, self._client, self._params, name=self._input_name + ) return self._input def output(self) -> LiveKitOutputTransport: if not self._output: self._output = LiveKitOutputTransport( - self._client, self._params, name=self._output_name + self, self._client, self._params, name=self._output_name ) return self._output diff --git a/src/pipecat/utils/base_object.py b/src/pipecat/utils/base_object.py index e51eac35d..1dee24ce7 100644 --- a/src/pipecat/utils/base_object.py +++ b/src/pipecat/utils/base_object.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # +import asyncio import inspect from abc import ABC from typing import Optional @@ -17,8 +18,15 @@ class BaseObject(ABC): def __init__(self, *, name: Optional[str] = None): self._id: int = obj_id() self._name = name or f"{self.__class__.__name__}#{obj_count(self)}" + + # Registered event handlers. self._event_handlers: dict = {} + # Set of tasks being executed. When a task finishes running it gets + # automatically removed from the set. When we cleanup we wait for all + # event tasks still being executed. + self._event_tasks = set() + @property def id(self) -> int: return self._id @@ -27,6 +35,12 @@ class BaseObject(ABC): def name(self) -> str: return self._name + async def cleanup(self): + if self._event_tasks: + event_names, tasks = zip(*self._event_tasks) + logger.debug(f"{self} wating on event handlers to finish {list(event_names)}...") + await asyncio.wait(tasks) + def event_handler(self, event_name: str): def decorator(handler): self.add_event_handler(event_name, handler) @@ -45,6 +59,16 @@ class BaseObject(ABC): self._event_handlers[event_name] = [] async def _call_event_handler(self, event_name: str, *args, **kwargs): + # Create the task. + task = asyncio.create_task(self._run_task(event_name, *args, **kwargs)) + + # Add it to our list of event tasks. + self._event_tasks.add((event_name, task)) + + # Remove the task from the event tasks list when the task completes. + task.add_done_callback(self._event_task_finished) + + async def _run_task(self, event_name: str, *args, **kwargs): try: for handler in self._event_handlers[event_name]: if inspect.iscoroutinefunction(handler): @@ -54,5 +78,10 @@ class BaseObject(ABC): except Exception as e: logger.exception(f"Exception in event handler {event_name}: {e}") + def _event_task_finished(self, task: asyncio.Task): + tuple_to_remove = next((t for t in self._event_tasks if t[1] == task), None) + if tuple_to_remove: + self._event_tasks.discard(tuple_to_remove) + def __str__(self): return self.name From 5dc8b48fbe353b97048a018320eaa5f83fe3dda4 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 18 Mar 2025 09:42:03 -0400 Subject: [PATCH 053/132] Fix an issue where GoogleSTTService would timeout due to stream inactivity --- CHANGELOG.md | 6 ++++++ src/pipecat/services/google/google.py | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d200ab87..7afea60be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -122,6 +122,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue with the `GoogleSTTService` where stream timeouts during + periods of inactivity were causing connection failures. The service now + properly detects timeout errors and handles reconnection gracefully, + ensuring continuous operation even after periods of silence or when using an + `STTMuteFilter`. + - Fixed an issue in `RimeTTSService` where the last line of text sent didn't result in an audio output being generated. diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 9d4a0a8a8..b2af64b57 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -1972,7 +1972,8 @@ class GoogleSTTService(STTService): break except Exception as e: - logger.error(f"Stream error, attempting to reconnect: {e}") + logger.warning(f"{self} Reconnecting: {e}") + await asyncio.sleep(1) # Brief delay before reconnecting self._stream_start_time = int(time.time() * 1000) continue @@ -2025,3 +2026,6 @@ class GoogleSTTService(STTService): except Exception as e: logger.error(f"Error processing Google STT responses: {e}") + + # Re-raise the exception to let it propagate (e.g. in the case of a timeout, propagate to _stream_audio to reconnect) + raise From 514ecda7552c1d26a648477de1d8001e498c8615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Mar 2025 17:31:01 -0700 Subject: [PATCH 054/132] TTSService: allow passing multiple text filters and aggregators --- CHANGELOG.md | 11 +++- .../foundational/14j-function-calling-nim.py | 2 +- .../35-pattern-pair-voice-switching.py | 2 +- examples/news-chatbot/server/news_bot.py | 2 +- src/pipecat/services/ai_services.py | 64 ++++++++++++++----- 5 files changed, 61 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c29b8e98..f90dd80dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added new `BaseTextAggregator`. Text aggregators are used by the TTS service to aggregate LLM tokens and decide when the aggregated text should be pushed - to the TTS service. It also allows for the text to be manipulated while it's - being aggregated. + to the TTS service. They also allow for the text to be manipulated while it's + being aggregated. Multiple text aggregators can be passed with + `text_aggregators` to the TTS service. - Added new `UltravoxSTTService`. (see https://github.com/fixie-ai/ultravox) @@ -113,6 +114,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated the default mode for `CartesiaTTSService` and `CartesiaHttpTTSService` to `sonic-2`. +### Deprecated + +- `TTSService` parameter `text_filter` is now deprecated, use `text_filters` + instead which is now a list. This allows passing multiple filters that will be + executed in order. + ### Removed - Removed deprecated `audio.resample_audio()`, use `create_default_resampler()` diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index d703d637a..ea8e25cf6 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -60,7 +60,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - # text_filter=MarkdownTextFilter(), + # text_filters=[MarkdownTextFilter()], ) llm = NimLLMService( diff --git a/examples/foundational/35-pattern-pair-voice-switching.py b/examples/foundational/35-pattern-pair-voice-switching.py index bb9587706..7d0094132 100644 --- a/examples/foundational/35-pattern-pair-voice-switching.py +++ b/examples/foundational/35-pattern-pair-voice-switching.py @@ -119,7 +119,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id=VOICE_IDS["narrator"], - text_aggregator=pattern_aggregator, + text_aggregators=[pattern_aggregator], ) # Initialize LLM diff --git a/examples/news-chatbot/server/news_bot.py b/examples/news-chatbot/server/news_bot.py index 2a389094e..b9f60200f 100644 --- a/examples/news-chatbot/server/news_bot.py +++ b/examples/news-chatbot/server/news_bot.py @@ -97,7 +97,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - text_filter=MarkdownTextFilter(), + text_filters=[MarkdownTextFilter()], ) llm = GoogleLLMService( diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 3fe33d69e..904e5cf90 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -8,7 +8,7 @@ import asyncio import io import wave from abc import abstractmethod -from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Tuple, Type +from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Sequence, Tuple, Type from loguru import logger @@ -239,8 +239,9 @@ class TTSService(AIService): # TTS output sample rate sample_rate: Optional[int] = None, # Text aggregator to aggregate incoming tokens and decide when to push to the TTS. - text_aggregator: Optional[BaseTextAggregator] = None, + text_aggregators: Sequence[BaseTextAggregator] = [], # Text filter executed after text has been aggregated. + text_filters: Sequence[BaseTextFilter] = [], text_filter: Optional[BaseTextFilter] = None, **kwargs, ): @@ -256,8 +257,21 @@ class TTSService(AIService): self._sample_rate = 0 self._voice_id: str = "" self._settings: Dict[str, Any] = {} - self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() - self._text_filter: Optional[BaseTextFilter] = text_filter + # Ensure there's at least one text aggregator. + self._text_aggregators: Sequence[BaseTextAggregator] = text_aggregators or [ + SimpleTextAggregator() + ] + self._text_filters: Sequence[BaseTextFilter] = text_filters + if text_filter: + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Parameter 'text_filter' is deprecated, use 'text_filters' instead.", + DeprecationWarning, + ) + self._text_filters = [text_filter] self._stop_frame_task: Optional[asyncio.Task] = None self._stop_frame_queue: asyncio.Queue = asyncio.Queue() @@ -317,8 +331,9 @@ class TTSService(AIService): self.set_model_name(value) elif key == "voice": self.set_voice(value) - elif key == "text_filter" and self._text_filter: - self._text_filter.update_settings(value) + elif key == "text_filter": + for filter in self._text_filters: + filter.update_settings(value) else: logger.warning(f"Unknown setting for TTS service: {key}") @@ -343,8 +358,8 @@ class TTSService(AIService): # pause to avoid audio overlapping. await self._maybe_pause_frame_processing() - sentence = self._text_aggregator.text - self._text_aggregator.reset() + sentence = self._text_aggregators[-1].text + self._reset_aggregators() self._processing_text = False await self._push_tts_frames(sentence) if isinstance(frame, LLMFullResponseEndFrame): @@ -390,9 +405,10 @@ class TTSService(AIService): async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): self._processing_text = False - self._text_aggregator.handle_interruption() - if self._text_filter: - self._text_filter.handle_interruption() + for aggregator in self._text_aggregators: + aggregator.handle_interruption() + for filter in self._text_filters: + filter.handle_interruption() async def _maybe_pause_frame_processing(self): if self._processing_text and self._pause_frame_processing: @@ -402,12 +418,25 @@ class TTSService(AIService): if self._pause_frame_processing: await self.resume_processing_frames() + def _reset_aggregators(self): + for aggregator in self._text_aggregators: + aggregator.reset() + async def _process_text_frame(self, frame: TextFrame): text: Optional[str] = None if not self._aggregate_sentences: text = frame.text else: - text = self._text_aggregator.aggregate(frame.text) + current_text = frame.text + + # Process all aggregators except the last one. + for aggregator in self._text_aggregators[:-1]: + aggregator.aggregate(current_text) + current_text = aggregator.text + + # The last aggregator decides whether we are sending text to the + # TTS or not. + text = self._text_aggregators[-1].aggregate(current_text) if text: await self._push_tts_frames(text) @@ -427,11 +456,16 @@ class TTSService(AIService): self._processing_text = True await self.start_processing_metrics() - if self._text_filter: - self._text_filter.reset_interruption() - text = self._text_filter.filter(text) + + # Process all filter. + for filter in self._text_filters: + filter.reset_interruption() + text = filter.filter(text) + await self.process_generator(self.run_tts(text)) + await self.stop_processing_metrics() + if self._push_text_frames: # We send the original text after the audio. This way, if we are # interrupted, the text is not added to the assistant context. From 7f1ccab445c4246a0e0758dab7da3c2fcd602048 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 19 Mar 2025 07:07:45 -0400 Subject: [PATCH 055/132] Fix: RTVI message disconnect-bot now pushes EndTaskFrame --- CHANGELOG.md | 4 ++++ src/pipecat/processors/frameworks/rtvi.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f90dd80dc..249474080 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,6 +138,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue where the RTVI message `disconnect-bot` was pushing an + `EndFrame`, resulting in the pipeline not shutting down. It now pushes an + `EndTaskFrame` upstream to shutdown the pipeline. + - Fixed an issue with the `GoogleSTTService` where stream timeouts during periods of inactivity were causing connection failures. The service now properly detects timeout errors and handles reconnection gracefully, diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 29747a582..bb97c2098 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -29,6 +29,7 @@ from pipecat.frames.frames import ( CancelFrame, DataFrame, EndFrame, + EndTaskFrame, ErrorFrame, Frame, FunctionCallResultFrame, @@ -766,7 +767,7 @@ class RTVIProcessor(FrameProcessor): update_config = RTVIUpdateConfig.model_validate(message.data) await self._handle_update_config(message.id, update_config) case "disconnect-bot": - await self.push_frame(EndFrame()) + await self.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM) case "action": action = RTVIActionRun.model_validate(message.data) action_frame = RTVIActionFrame(message_id=message.id, rtvi_action_run=action) From 5f28834588dba5ddd1f8466bab598886d5b6664f Mon Sep 17 00:00:00 2001 From: Nico <105345946+nicougou@users.noreply.github.com> Date: Wed, 19 Mar 2025 14:49:51 +0100 Subject: [PATCH 056/132] feature: add custom headers to AsyncOpenAI --- src/pipecat/services/openai.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 5a3a993aa..feddc8c44 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -116,6 +116,7 @@ class BaseOpenAILLMService(LLMService): base_url=None, organization=None, project=None, + default_headers: Mapping[str, str] | None = None, params: InputParams = InputParams(), **kwargs, ): @@ -132,10 +133,10 @@ class BaseOpenAILLMService(LLMService): } self.set_model_name(model) self._client = self.create_client( - api_key=api_key, base_url=base_url, organization=organization, project=project, **kwargs + api_key=api_key, base_url=base_url, organization=organization, project=project, default_headers=default_headers, **kwargs ) - def create_client(self, api_key=None, base_url=None, organization=None, project=None, **kwargs): + def create_client(self, api_key=None, base_url=None, organization=None, project=None, default_headers=None, **kwargs): return AsyncOpenAI( api_key=api_key, base_url=base_url, @@ -146,6 +147,7 @@ class BaseOpenAILLMService(LLMService): max_keepalive_connections=100, max_connections=1000, keepalive_expiry=None ) ), + default_headers=default_headers ) def can_generate_metrics(self) -> bool: From 1dbad2326aeeb2fccda94e392be6cdd8393dba4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 26 Feb 2025 17:23:06 -0800 Subject: [PATCH 057/132] utils(string): support email addresses in end of sentence matching --- CHANGELOG.md | 3 +++ src/pipecat/utils/string.py | 19 ++++++++++++++++++- tests/test_utils_string.py | 4 ++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 249474080..6e626a7b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,6 +138,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `match_endofsentence` issue that would result in emails to be + considered an end of sentence. + - Fixed an issue where the RTVI message `disconnect-bot` was pushing an `EndFrame`, resulting in the pipeline not shutting down. It now pushes an `EndTaskFrame` upstream to shutdown the pipeline. diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index 154d03174..06f2fc175 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -17,9 +17,26 @@ ENDOFSENTENCE_PATTERN_STR = r""" (\。\s*\。\s*\。|[。？！；।]) # the full-width version (mainly used in East Asian languages such as Chinese, Hindi) $ # End of string """ + ENDOFSENTENCE_PATTERN = re.compile(ENDOFSENTENCE_PATTERN_STR, re.VERBOSE) +EMAIL_PATTERN = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}") + def match_endofsentence(text: str) -> int: - match = ENDOFSENTENCE_PATTERN.search(text.rstrip()) + text = text.rstrip() + + # Find all emails. + emails = list(EMAIL_PATTERN.finditer(text)) + + # Replace email dots by ampersands so we can find the end of sentence. + for email_match in emails: + start = email_match.start() + end = email_match.end() + new_email = text[start:end].replace(".", "&") + text = text[:start] + new_email + text[end:] + + # Match against the new text. + match = ENDOFSENTENCE_PATTERN.search(text) + return match.end() if match else 0 diff --git a/tests/test_utils_string.py b/tests/test_utils_string.py index e4cdb4cb4..24519f724 100644 --- a/tests/test_utils_string.py +++ b/tests/test_utils_string.py @@ -18,6 +18,9 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): assert match_endofsentence("This is a sentence...") == 21 assert match_endofsentence("This is a sentence . . .") == 24 assert match_endofsentence("This is a sentence. ..") == 22 + assert match_endofsentence("This is for Mr. and Mrs. Jones.") == 31 + assert match_endofsentence("U.S.A and U.S.A..") == 17 + assert match_endofsentence("My emails are foo@pipecat.ai and bar@pipecat.ai.") == 48 assert not match_endofsentence("This is not a sentence") assert not match_endofsentence("This is not a sentence,") assert not match_endofsentence("This is not a sentence, ") @@ -28,6 +31,7 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): assert not match_endofsentence("Heute ist Dienstag, der 3.") # 3. Juli 2024 assert not match_endofsentence("America, or the U.") # U.S.A. assert not match_endofsentence("It still early, it's 3:00 a.") # 3:00 a.m. + assert not match_endofsentence("My emails are foo@pipecat.ai and bar@pipecat.ai") async def test_endofsentence_zh(self): chinese_sentences = [ From 11984b89b79661d6d7ae2e0987446434dfe8021b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 26 Feb 2025 18:22:37 -0800 Subject: [PATCH 058/132] utils(string): add support for floating point numbers --- CHANGELOG.md | 3 +++ src/pipecat/utils/string.py | 28 ++++++++++++++++++++-------- tests/test_utils_string.py | 6 ++++++ 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e626a7b1..c530b0986 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,6 +138,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `match_endofsentence` issue that would result in floating point + numbers to be considered an end of sentence. + - Fixed a `match_endofsentence` issue that would result in emails to be considered an end of sentence. diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index 06f2fc175..a89127de9 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -8,7 +8,8 @@ import re ENDOFSENTENCE_PATTERN_STR = r""" (? str: + start = match.start() + end = match.end() + replacement = text[start:end].replace(old, new) + text = text[:start] + replacement + text[end:] + return text + def match_endofsentence(text: str) -> int: text = text.rstrip() - # Find all emails. + # Replace email dots by ampersands so we can find the end of sentence. For + # example, first.last@email.com becomes first&last@email&com. emails = list(EMAIL_PATTERN.finditer(text)) - - # Replace email dots by ampersands so we can find the end of sentence. for email_match in emails: - start = email_match.start() - end = email_match.end() - new_email = text[start:end].replace(".", "&") - text = text[:start] + new_email + text[end:] + text = replace_match(text, email_match, ".", "&") + + # Replace number dots by ampersands so we can find the end of sentence. + numbers = list(NUMBER_PATTERN.finditer(text)) + for number_match in numbers: + text = replace_match(text, number_match, ".", "&") # Match against the new text. match = ENDOFSENTENCE_PATTERN.search(text) diff --git a/tests/test_utils_string.py b/tests/test_utils_string.py index 24519f724..ee0946d69 100644 --- a/tests/test_utils_string.py +++ b/tests/test_utils_string.py @@ -21,6 +21,11 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): assert match_endofsentence("This is for Mr. and Mrs. Jones.") == 31 assert match_endofsentence("U.S.A and U.S.A..") == 17 assert match_endofsentence("My emails are foo@pipecat.ai and bar@pipecat.ai.") == 48 + assert match_endofsentence("My email is foo.bar@pipecat.ai.") == 31 + assert match_endofsentence("My email is spell(foo.bar@pipecat.ai).") == 38 + assert match_endofsentence("The number pi is 3.14159.") == 25 + assert match_endofsentence("Valid scientific notation 1.23e4.") == 33 + assert match_endofsentence("Valid scientific notation 0.e4.") == 31 assert not match_endofsentence("This is not a sentence") assert not match_endofsentence("This is not a sentence,") assert not match_endofsentence("This is not a sentence, ") @@ -32,6 +37,7 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): assert not match_endofsentence("America, or the U.") # U.S.A. assert not match_endofsentence("It still early, it's 3:00 a.") # 3:00 a.m. assert not match_endofsentence("My emails are foo@pipecat.ai and bar@pipecat.ai") + assert not match_endofsentence("The number pi is 3.14159") async def test_endofsentence_zh(self): chinese_sentences = [ From 1a3a268c9de7d3a0d59ea43da1cc7718c9c430bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 28 Feb 2025 17:23:54 -0800 Subject: [PATCH 059/132] utils(string): add new function parse_start_end_tags() --- src/pipecat/utils/string.py | 76 +++++++++++++++++++++++++++++++++++++ tests/test_utils_string.py | 50 +++++++++++++++++++++++- 2 files changed, 125 insertions(+), 1 deletion(-) diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index a89127de9..69036a665 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -5,6 +5,7 @@ # import re +from typing import Optional, Sequence, Tuple ENDOFSENTENCE_PATTERN_STR = r""" (? str: + """Replace occurrences of a substring within a matched section of a given + text. + + Args: + text (str): The input text in which replacements will be made. + match (re.Match): A regex match object representing the section of text to modify. + old (str): The substring to be replaced. + new (str): The substring to replace `old` with. + + Returns: + str: The modified text with the specified replacements made within the matched section. + + """ start = match.start() end = match.end() replacement = text[start:end].replace(old, new) @@ -35,6 +51,20 @@ def replace_match(text: str, match: re.Match, old: str, new: str) -> str: def match_endofsentence(text: str) -> int: + """Finds the position of the end of a sentence in the provided text string. + + This function processes the input text by replacing periods in email + addresses and numbers with ampersands to prevent them from being + misidentified as sentence terminals. It then searches for the end of a + sentence using a specified regex pattern. + + Args: + text (str): The input text in which to find the end of the sentence. + + Returns: + int: The position of the end of the sentence if found, otherwise 0. + + """ text = text.rstrip() # Replace email dots by ampersands so we can find the end of sentence. For @@ -52,3 +82,49 @@ def match_endofsentence(text: str) -> int: match = ENDOFSENTENCE_PATTERN.search(text) return match.end() if match else 0 + + +def parse_start_end_tags( + text: str, + tags: Sequence[StartEndTags], + current_tag: Optional[StartEndTags], + current_tag_index: int, +) -> Tuple[Optional[StartEndTags], int]: + """Parses the given text to identify a pair of start/end tags. + + If a start tag was previously found (i.e. current_tags is valid), wait for + the corresponding end tag. Otherwise, wait for a start tag. + + This function will return the index in the text that we should start parsing + in the next call and the current or new tags. + + Parameters: + - text (str): The text to be parsed. + - tags (Sequence[StartEndTags]): List of tuples containing start and end tags. + - current_tags (Optional[StartEndTags]): The currently active tags, if any. + - current_tags_index (int): The current index in the text. + + Returns: + Tuple[Optional[StartEndTags], int]: A tuple containing None or the current + tag and the index of the text. + + """ + # If we are already inside a tag, check if the end tag is in the text. + if current_tag: + _, end_tag = current_tag + if end_tag in text[current_tag_index:]: + return (None, len(text)) + return (current_tag, current_tag_index) + + # Check if any start tag appears in the text + for start_tag, end_tag in tags: + start_tag_count = text[current_tag_index:].count(start_tag) + end_tag_count = text[current_tag_index:].count(end_tag) + if start_tag_count == 0 and end_tag_count == 0: + return (None, current_tag_index) + elif start_tag_count > end_tag_count: + return ((start_tag, end_tag), len(text)) + elif start_tag_count == end_tag_count: + return (None, len(text)) + + return (None, current_tag_index) diff --git a/tests/test_utils_string.py b/tests/test_utils_string.py index ee0946d69..cabd88a36 100644 --- a/tests/test_utils_string.py +++ b/tests/test_utils_string.py @@ -6,7 +6,7 @@ import unittest -from pipecat.utils.string import match_endofsentence +from pipecat.utils.string import match_endofsentence, parse_start_end_tags class TestUtilsString(unittest.IsolatedAsyncioTestCase): @@ -23,6 +23,7 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): assert match_endofsentence("My emails are foo@pipecat.ai and bar@pipecat.ai.") == 48 assert match_endofsentence("My email is foo.bar@pipecat.ai.") == 31 assert match_endofsentence("My email is spell(foo.bar@pipecat.ai).") == 38 + assert match_endofsentence("My email is foo.bar@pipecat.ai.") == 46 assert match_endofsentence("The number pi is 3.14159.") == 25 assert match_endofsentence("Valid scientific notation 1.23e4.") == 33 assert match_endofsentence("Valid scientific notation 0.e4.") == 31 @@ -60,3 +61,50 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): for i in hindi_sentences: assert match_endofsentence(i) assert not match_endofsentence("हैलो，") + + +class TestStartEndTags(unittest.IsolatedAsyncioTestCase): + async def test_empty(self): + assert parse_start_end_tags("", [], None, 0) == (None, 0) + assert parse_start_end_tags("Hello from Pipecat!", [], None, 0) == (None, 0) + + async def test_simple(self): + # (, ) + assert parse_start_end_tags("Hello from Pipecat!", [("", "")], None, 0) == ( + None, + 26, + ) + assert parse_start_end_tags("Hello from Pipecat", [("", "")], None, 0) == ( + ("", ""), + 21, + ) + assert parse_start_end_tags("Hello from Pipecat", [("", "")], None, 6) == ( + ("", ""), + 21, + ) + + # (spell(, )) + assert parse_start_end_tags("Hello from spell(Pipecat)!", [("spell(", ")")], None, 0) == ( + None, + 26, + ) + assert parse_start_end_tags("Hello from spell(Pipecat", [("spell(", ")")], None, 0) == ( + ("spell(", ")"), + 24, + ) + + async def test_multiple(self): + # (, ) + assert parse_start_end_tags( + "Hello from Pipecat! Hello World!", [("", "")], None, 0 + ) == ( + None, + 46, + ) + + assert parse_start_end_tags( + "Hello from Pipecat! Hello World", [("", "")], None, 0 + ) == ( + ("", ""), + 41, + ) From e7224473f2728e37c95aee52d48b9099a16bbdc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Mar 2025 23:04:14 -0700 Subject: [PATCH 060/132] utils(text): add new SkipTagsAggregator --- CHANGELOG.md | 4 + .../utils/text/skip_tags_aggregator.py | 94 +++++++++++++++++++ tests/test_skip_tags_aggregator.py | 54 +++++++++++ 3 files changed, 152 insertions(+) create mode 100644 src/pipecat/utils/text/skip_tags_aggregator.py create mode 100644 tests/test_skip_tags_aggregator.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c530b0986..5abdf4ef5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added new `SkipTagsAggregator` that extends `BaseTextAggregator` to aggregate + text and skips end of sentence matching if aggregated text is between + start/end tags. + - Added new `PatternPairAggregator` that extends `BaseTextAggregator` to identify content between matching pattern pairs in streamed text. This allows for detection and processing of structured content like XML-style tags that diff --git a/src/pipecat/utils/text/skip_tags_aggregator.py b/src/pipecat/utils/text/skip_tags_aggregator.py new file mode 100644 index 000000000..00129028e --- /dev/null +++ b/src/pipecat/utils/text/skip_tags_aggregator.py @@ -0,0 +1,94 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from typing import Optional, Sequence + +from pipecat.utils.string import StartEndTags, match_endofsentence, parse_start_end_tags +from pipecat.utils.text.base_text_aggregator import BaseTextAggregator + + +class SkipTagsAggregator(BaseTextAggregator): + """Aggregator that prevents end of sentence matching between start/end tags. + + This aggregator buffers text until it finds an end of sentence or a start + tag. If a start tag is found the aggregator will keep aggregating text + unconditionally until the corresponding end tag is found. It's particularly + useful for processing content with custom delimiters that should prevent + text from being considered for end of sentence matching.. + + The aggregator ensures that tags spanning multiple text chunks are correctly + identified. + + """ + + def __init__(self, tags: Sequence[StartEndTags]): + """Initialize the pattern pair aggregator. + + Creates an empty aggregator with no patterns or handlers registered. + """ + self._text = "" + self._tags = tags + self._current_tag: Optional[StartEndTags] = None + self._current_tag_index: int = 0 + + @property + def text(self) -> str: + """Get the currently buffered text. + + Returns: + The current text buffer content. + """ + return self._text + + def aggregate(self, text: str) -> Optional[str]: + """Aggregate text and process pattern pairs. + + This method adds the new text to the buffer, processes any complete pattern + pairs, and returns processed text up to sentence boundaries if possible. + If there are incomplete patterns (start without matching end), it will + continue buffering text. + + Args: + text: New text to add to the buffer. + + Returns: + Processed text up to a sentence boundary, or None if more + text is needed to form a complete sentence or pattern. + """ + # Add new text to buffer + self._text += text + + (self._current_tag, self._current_tag_index) = parse_start_end_tags( + self._text, self._tags, self._current_tag, self._current_tag_index + ) + + # Find sentence boundary if no incomplete patterns + if not self._current_tag: + eos_marker = match_endofsentence(self._text) + if eos_marker: + # Extract text up to the sentence boundary + result = self._text[:eos_marker] + self._text = self._text[eos_marker:] + return result + + # No complete sentence found yet + return None + + def handle_interruption(self): + """Handle interruptions by clearing the buffer. + + Called when an interruption occurs in the processing pipeline, + to reset the state and discard any partially aggregated text. + """ + self._text = "" + + def reset(self): + """Clear the internally aggregated text. + + Resets the aggregator to its initial state, discarding any + buffered text. + """ + self._text = "" diff --git a/tests/test_skip_tags_aggregator.py b/tests/test_skip_tags_aggregator.py new file mode 100644 index 000000000..8f36c4c05 --- /dev/null +++ b/tests/test_skip_tags_aggregator.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2024-2025 Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import unittest + +from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator + + +class TestSkipTagsAggregator(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.aggregator = SkipTagsAggregator([("", "")]) + + async def test_no_tags(self): + self.aggregator.reset() + + # No tags involved, aggregate at end of sentence. + result = self.aggregator.aggregate("Hello Pipecat!") + self.assertEqual(result, "Hello Pipecat!") + self.assertEqual(self.aggregator.text, "") + + async def test_basic_tags(self): + self.aggregator.reset() + + # Tags involved, avoid aggregation during tags. + result = self.aggregator.aggregate("My email is foo@pipecat.ai.") + self.assertEqual(result, "My email is foo@pipecat.ai.") + self.assertEqual(self.aggregator.text, "") + + async def test_streaming_tags(self): + self.aggregator.reset() + + # Tags involved, stream small chunk of texts. + result = self.aggregator.aggregate("My email is foo.") + self.assertIsNone(result) + self.assertEqual(self.aggregator.text, "My email is foo.") + + result = self.aggregator.aggregate("bar@pipecat.") + self.assertIsNone(result) + self.assertEqual(self.aggregator.text, "My email is foo.bar@pipecat.") + + result = self.aggregator.aggregate("aifoo.bar@pipecat.ai.") + self.assertEqual(result, "My email is foo.bar@pipecat.ai.") + self.assertEqual(self.aggregator.text, "") From 54620133d4473e3f8d10d26112fbf002ece6b8ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Mar 2025 23:08:34 -0700 Subject: [PATCH 061/132] services: add spelling out support to CartesiaTTSService and RimeTTSService --- CHANGELOG.md | 3 +++ src/pipecat/services/cartesia.py | 6 +++++- src/pipecat/services/rime.py | 6 +++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5abdf4ef5..3006ed5ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -142,6 +142,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `CartesiaTTSService` and `RimeTTSService` issue that would consider + text between spelling out tags end of sentence. + - Fixed a `match_endofsentence` issue that would result in floating point numbers to be considered an end of sentence. diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 8b7f57c63..5a795d750 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -7,7 +7,7 @@ import base64 import json import uuid -from typing import AsyncGenerator, List, Optional, Union +from typing import AsyncGenerator, List, Optional, Sequence, Union from loguru import logger from pydantic import BaseModel @@ -26,6 +26,8 @@ from pipecat.frames.frames import ( from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import AudioContextWordTTSService, TTSService from pipecat.transcriptions.language import Language +from pipecat.utils.text.base_text_aggregator import BaseTextAggregator +from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator # See .env.example for Cartesia configuration needed try: @@ -89,6 +91,7 @@ class CartesiaTTSService(AudioContextWordTTSService): encoding: str = "pcm_s16le", container: str = "raw", params: InputParams = InputParams(), + text_aggregators: Sequence[BaseTextAggregator] = [], **kwargs, ): # Aggregating sentences still gives cleaner-sounding results and fewer @@ -106,6 +109,7 @@ class CartesiaTTSService(AudioContextWordTTSService): push_text_frames=False, pause_frame_processing=True, sample_rate=sample_rate, + text_aggregators=text_aggregators or [SkipTagsAggregator([("", "")])], **kwargs, ) diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index b2610b06c..471f82d66 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -7,7 +7,7 @@ import base64 import json import uuid -from typing import AsyncGenerator, Optional +from typing import AsyncGenerator, Optional, Sequence import aiohttp from loguru import logger @@ -27,6 +27,8 @@ from pipecat.frames.frames import ( from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import AudioContextWordTTSService, TTSService from pipecat.transcriptions.language import Language +from pipecat.utils.text.base_text_aggregator import BaseTextAggregator +from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator try: import websockets @@ -78,6 +80,7 @@ class RimeTTSService(AudioContextWordTTSService): model: str = "mistv2", sample_rate: Optional[int] = None, params: InputParams = InputParams(), + text_aggregators: Sequence[BaseTextAggregator] = [], **kwargs, ): """Initialize Rime TTS service. @@ -97,6 +100,7 @@ class RimeTTSService(AudioContextWordTTSService): push_stop_frames=True, pause_frame_processing=True, sample_rate=sample_rate, + text_aggregators=text_aggregators or [SkipTagsAggregator([("spell(", ")")])], **kwargs, ) From fc0f404d26ddb8f7cb589c2f84c2c3ef96885f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 26 Feb 2025 17:23:58 -0800 Subject: [PATCH 062/132] examples: add new 36-user-email-gathering.py --- CHANGELOG.md | 4 + .../foundational/36-user-email-gathering.py | 141 ++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 examples/foundational/36-user-email-gathering.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3006ed5ca..64f342dc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -166,6 +166,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Other +- Added a new example `examples/foundational/36-user-email-gathering.py` to show + how to gather user emails. The example uses's Cartesia's `` + tags and Rime `spell()` function to spell out the emails for confirmation. + - Update the `34-audio-recording.py` example to include an STT processor. - Added foundational example `35-voice-switching.py` showing how to use the new diff --git a/examples/foundational/36-user-email-gathering.py b/examples/foundational/36-user-email-gathering.py new file mode 100644 index 000000000..0e76826e3 --- /dev/null +++ b/examples/foundational/36-user-email-gathering.py @@ -0,0 +1,141 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from openai.types.chat import ChatCompletionToolParam +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.openai import OpenAILLMContext, OpenAILLMService +from pipecat.services.rime import RimeHttpTTSService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def store_user_emails(function_name, tool_call_id, args, llm, context, result_callback): + print(f"User emails: {args}") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + # Cartesia offers a `` tags that we can use to ask the user + # to confirm the emails. + # (see https://docs.cartesia.ai/build-with-sonic/formatting-text-for-sonic/spelling-out-input-text) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + aiohttp_session=session, + ) + + # Rime offers a function `spell()` that we can use to ask the user + # to confirm the emails. + # (see https://docs.rime.ai/api-reference/spell) + # tts = RimeHttpTTSService( + # api_key=os.getenv("RIME_API_KEY", ""), + # voice_id="eva", + # aiohttp_session=session, + # ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + # You can aslo register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("store_user_emails", store_user_emails) + + tools = [ + ChatCompletionToolParam( + type="function", + function={ + "name": "store_user_emails", + "description": "Store user emails when confirmed", + "parameters": { + "type": "object", + "properties": { + "emails": { + "type": "array", + "description": "The list of user emails", + "items": {"type": "string"}, + }, + }, + "required": ["emails"], + }, + }, + ) + ] + messages = [ + { + "role": "system", + # Cartesia + "content": "You need to gather a valid email or emails from the user. Your output will be converted to audio so don't include special characters in your answers. If the user provides one or more email addresses confirm them with the user. Enclose all emails with tags, for example a@a.com.", + # Rime spell() + # "content": "You need to gather a valid email or emails from the user. Your output will be converted to audio so don't include special characters in your answers. If the user provides one or more email addresses confirm them with the user. Enclose all emails with spell(), for example spell(a@a.com).", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) From 336e2f1579d53fe8d6c0fa336fc91546cc74618c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 11:02:29 -0700 Subject: [PATCH 063/132] TTSServices: for now just specify a single text aggregator --- CHANGELOG.md | 4 +-- .../35-pattern-pair-voice-switching.py | 2 +- src/pipecat/services/ai_services.py | 29 ++++--------------- src/pipecat/services/cartesia.py | 6 ++-- src/pipecat/services/rime.py | 6 ++-- 5 files changed, 15 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64f342dc5..125ca20d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,8 +21,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added new `BaseTextAggregator`. Text aggregators are used by the TTS service to aggregate LLM tokens and decide when the aggregated text should be pushed to the TTS service. They also allow for the text to be manipulated while it's - being aggregated. Multiple text aggregators can be passed with - `text_aggregators` to the TTS service. + being aggregated. A text aggregator can be passed via `text_aggregator` to the + TTS service. - Added new `UltravoxSTTService`. (see https://github.com/fixie-ai/ultravox) diff --git a/examples/foundational/35-pattern-pair-voice-switching.py b/examples/foundational/35-pattern-pair-voice-switching.py index 7d0094132..bb9587706 100644 --- a/examples/foundational/35-pattern-pair-voice-switching.py +++ b/examples/foundational/35-pattern-pair-voice-switching.py @@ -119,7 +119,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id=VOICE_IDS["narrator"], - text_aggregators=[pattern_aggregator], + text_aggregator=pattern_aggregator, ) # Initialize LLM diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 904e5cf90..eae030b27 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -239,7 +239,7 @@ class TTSService(AIService): # TTS output sample rate sample_rate: Optional[int] = None, # Text aggregator to aggregate incoming tokens and decide when to push to the TTS. - text_aggregators: Sequence[BaseTextAggregator] = [], + text_aggregator: Optional[BaseTextAggregator] = None, # Text filter executed after text has been aggregated. text_filters: Sequence[BaseTextFilter] = [], text_filter: Optional[BaseTextFilter] = None, @@ -257,10 +257,7 @@ class TTSService(AIService): self._sample_rate = 0 self._voice_id: str = "" self._settings: Dict[str, Any] = {} - # Ensure there's at least one text aggregator. - self._text_aggregators: Sequence[BaseTextAggregator] = text_aggregators or [ - SimpleTextAggregator() - ] + self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() self._text_filters: Sequence[BaseTextFilter] = text_filters if text_filter: import warnings @@ -358,8 +355,8 @@ class TTSService(AIService): # pause to avoid audio overlapping. await self._maybe_pause_frame_processing() - sentence = self._text_aggregators[-1].text - self._reset_aggregators() + sentence = self._text_aggregator.text + self._text_aggregator.reset() self._processing_text = False await self._push_tts_frames(sentence) if isinstance(frame, LLMFullResponseEndFrame): @@ -405,8 +402,7 @@ class TTSService(AIService): async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): self._processing_text = False - for aggregator in self._text_aggregators: - aggregator.handle_interruption() + self._text_aggregator.handle_interruption() for filter in self._text_filters: filter.handle_interruption() @@ -418,25 +414,12 @@ class TTSService(AIService): if self._pause_frame_processing: await self.resume_processing_frames() - def _reset_aggregators(self): - for aggregator in self._text_aggregators: - aggregator.reset() - async def _process_text_frame(self, frame: TextFrame): text: Optional[str] = None if not self._aggregate_sentences: text = frame.text else: - current_text = frame.text - - # Process all aggregators except the last one. - for aggregator in self._text_aggregators[:-1]: - aggregator.aggregate(current_text) - current_text = aggregator.text - - # The last aggregator decides whether we are sending text to the - # TTS or not. - text = self._text_aggregators[-1].aggregate(current_text) + text = self._text_aggregator.aggregate(frame.text) if text: await self._push_tts_frames(text) diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 5a795d750..3b491d26b 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -7,7 +7,7 @@ import base64 import json import uuid -from typing import AsyncGenerator, List, Optional, Sequence, Union +from typing import AsyncGenerator, List, Optional, Union from loguru import logger from pydantic import BaseModel @@ -91,7 +91,7 @@ class CartesiaTTSService(AudioContextWordTTSService): encoding: str = "pcm_s16le", container: str = "raw", params: InputParams = InputParams(), - text_aggregators: Sequence[BaseTextAggregator] = [], + text_aggregator: Optional[BaseTextAggregator] = None, **kwargs, ): # Aggregating sentences still gives cleaner-sounding results and fewer @@ -109,7 +109,7 @@ class CartesiaTTSService(AudioContextWordTTSService): push_text_frames=False, pause_frame_processing=True, sample_rate=sample_rate, - text_aggregators=text_aggregators or [SkipTagsAggregator([("", "")])], + text_aggregator=text_aggregator or SkipTagsAggregator([("", "")]), **kwargs, ) diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index 471f82d66..c6fc50001 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -7,7 +7,7 @@ import base64 import json import uuid -from typing import AsyncGenerator, Optional, Sequence +from typing import AsyncGenerator, Optional import aiohttp from loguru import logger @@ -80,7 +80,7 @@ class RimeTTSService(AudioContextWordTTSService): model: str = "mistv2", sample_rate: Optional[int] = None, params: InputParams = InputParams(), - text_aggregators: Sequence[BaseTextAggregator] = [], + text_aggregator: Optional[BaseTextAggregator] = None, **kwargs, ): """Initialize Rime TTS service. @@ -100,7 +100,7 @@ class RimeTTSService(AudioContextWordTTSService): push_stop_frames=True, pause_frame_processing=True, sample_rate=sample_rate, - text_aggregators=text_aggregators or [SkipTagsAggregator([("spell(", ")")])], + text_aggregator=text_aggregator or SkipTagsAggregator([("spell(", ")")]), **kwargs, ) From 48d73a263619145159bb05c25d25ce799f960bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Mar 2025 23:53:48 -0700 Subject: [PATCH 064/132] SegmentedSTTService: allow audio to pass-through downstream --- CHANGELOG.md | 3 +++ src/pipecat/services/ai_services.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 125ca20d8..5d30759a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -142,6 +142,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue with `SegmentedSTTService` based services + (e.g. `GroqSTTService`) that was not allow audio to pass-through downstream. + - Fixed a `CartesiaTTSService` and `RimeTTSService` issue that would consider text between spelling out tags end of sentence. diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index eae030b27..ba895c53d 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -811,8 +811,6 @@ class STTService(AIService): return await self.process_generator(self.run_stt(frame.audio)) - if self._audio_passthrough: - await self.push_frame(frame, direction) async def process_frame(self, frame: Frame, direction: FrameDirection): """Processes a frame of audio data, either buffering or transcribing it.""" @@ -823,6 +821,8 @@ class STTService(AIService): # push a TextFrame. We also push audio downstream in case someone # else needs it. await self.process_audio_frame(frame, direction) + if self._audio_passthrough: + await self.push_frame(frame, direction) elif isinstance(frame, STTUpdateSettingsFrame): await self._update_settings(frame.settings) elif isinstance(frame, STTMuteFrame): From 8942c2e05359606c8083a22172cc3d5938a2c593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 15:33:33 -0700 Subject: [PATCH 065/132] GeminiMultimodalLiveLLMService: fix duplicated messages in context Fixes #1384 --- CHANGELOG.md | 3 +++ .../services/gemini_multimodal_live/gemini.py | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d30759a6..7350fe6d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -142,6 +142,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `GeminiMultimodalLiveLLMService` issue that was causing messages to be + duplicated in the context when pushing `LLMMessagesAppendFrame` frames. + - Fixed an issue with `SegmentedSTTService` based services (e.g. `GroqSTTService`) that was not allow audio to pass-through downstream. diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 5fe8a792b..9484fc27d 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -341,10 +341,8 @@ class GeminiMultimodalLiveLLMService(LLMService): async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) - # logger.debug(f"Processing frame: {frame}") - if isinstance(frame, TranscriptionFrame): - pass + await self.push_frame(frame, direction) elif isinstance(frame, OpenAILLMContextFrame): context: GeminiMultimodalLiveContext = GeminiMultimodalLiveContext.upgrade( frame.context @@ -361,31 +359,35 @@ class GeminiMultimodalLiveLLMService(LLMService): # Support just one tool call per context frame for now tool_result_message = context.messages[-1] await self._tool_result(tool_result_message) - elif isinstance(frame, InputAudioRawFrame): await self._send_user_audio(frame) + await self.push_frame(frame, direction) elif isinstance(frame, InputImageRawFrame): await self._send_user_video(frame) + await self.push_frame(frame, direction) elif isinstance(frame, StartInterruptionFrame): await self._handle_interruption() + await self.push_frame(frame, direction) elif isinstance(frame, UserStartedSpeakingFrame): await self._handle_user_started_speaking(frame) + await self.push_frame(frame, direction) elif isinstance(frame, UserStoppedSpeakingFrame): await self._handle_user_stopped_speaking(frame) + await self.push_frame(frame, direction) elif isinstance(frame, BotStartedSpeakingFrame): # Ignore this frame. Use the serverContent API message instead - pass + await self.push_frame(frame, direction) elif isinstance(frame, BotStoppedSpeakingFrame): # ignore this frame. Use the serverContent.turnComplete API message - pass + await self.push_frame(frame, direction) elif isinstance(frame, LLMMessagesAppendFrame): await self._create_single_response(frame.messages) elif isinstance(frame, LLMUpdateSettingsFrame): await self._update_settings(frame.settings) elif isinstance(frame, LLMSetToolsFrame): await self._update_settings() - - await self.push_frame(frame, direction) + else: + await self.push_frame(frame, direction) # # websocket communication From f31e77c4f644e302b7f65d70ba44bde8e4ef82af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 18:43:07 -0700 Subject: [PATCH 066/132] pyproject: added empty tavus dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 586c5c026..4176ae2a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ sentry = [ "sentry-sdk~=2.20.0" ] silero = [ "onnxruntime~=1.20.1" ] simli = [ "simli-ai~=0.1.10"] soundfile = [ "soundfile~=0.13.0" ] +tavus=[] together = [] ultravox = [ "transformers~=4.48.0", "vllm~=0.7.3" ] websocket = [ "websockets~=13.1", "fastapi~=0.115.6" ] From a3b5e4413ac7370031b6e3a6ceb17a2ca5c6da06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 16:00:57 -0700 Subject: [PATCH 067/132] WebsocketTTSService: add `on_connection_error` and `reconnect_on_error` --- CHANGELOG.md | 7 ++++ src/pipecat/pipeline/task.py | 4 ++- src/pipecat/services/ai_services.py | 36 +++++++++++++++++--- src/pipecat/services/cartesia.py | 3 +- src/pipecat/services/elevenlabs.py | 3 +- src/pipecat/services/fish.py | 3 +- src/pipecat/services/lmnt.py | 3 +- src/pipecat/services/neuphonic.py | 3 +- src/pipecat/services/playht.py | 8 +++-- src/pipecat/services/rime.py | 3 +- src/pipecat/services/websocket_service.py | 40 +++++++++++++---------- 11 files changed, 81 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d30759a6..35df891ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added a `reconnect_on_error` parameter to websocket-based TTS services as well + as a `on_connection_error` event handler. The `reconnect_on_error` indicates + whether the TTS service should reconnect on error. The `on_connection_error` + will always get called if there's any error no matter the value of + `reconnect_on_error`. This allows, for example, to fallback to a different TTS + provider if something goes wrong with the current one. + - Added new `SkipTagsAggregator` that extends `BaseTextAggregator` to aggregate text and skips end of sentence matching if aggregated text is between start/end tags. diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 12c56ef3f..e9744fd04 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -425,12 +425,14 @@ class PipelineTask(BaseTask): # Tell the task we should stop nicely. await self.queue_frame(StopFrame()) elif isinstance(frame, ErrorFrame): - logger.error(f"Error running app: {frame}") if frame.fatal: + logger.error(f"A fatal error occurred: {frame}") # Cancel all tasks downstream. await self.queue_frame(CancelFrame()) # Tell the task we should stop. await self.queue_frame(StopTaskFrame()) + else: + logger.warning(f"Something went wrong: {frame}") self._up_queue.task_done() async def _process_down_queue(self): diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index ba895c53d..d29e0e714 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -549,11 +549,25 @@ class WordTTSService(TTSService): class WebsocketTTSService(TTSService, WebsocketService): - """This is a base class for websocket-based TTS services.""" + """This is a base class for websocket-based TTS services. - def __init__(self, **kwargs): + If an error occurs with the websocket, an "on_connection_error" event will + be triggered: + + @tts.event_handler("on_connection_error") + async def on_connection_error(tts: TTSService, error: str): + ... + + """ + + def __init__(self, *, reconnect_on_error: bool = True, **kwargs): TTSService.__init__(self, **kwargs) - WebsocketService.__init__(self) + WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs) + self._register_event_handler("on_connection_error") + + async def _report_error(self, error: ErrorFrame): + await self._call_event_handler("on_connection_error", error.error) + await self.push_error(error) class InterruptibleTTSService(WebsocketTTSService): @@ -590,11 +604,23 @@ class WebsocketWordTTSService(WordTTSService, WebsocketService): """This is a base class for websocket-based TTS services that support word timestamps. + If an error occurs with the websocket a "on_connection_error" event will be + triggered: + + @tts.event_handler("on_connection_error") + async def on_connection_error(tts: TTSService, error: str): + ... + """ - def __init__(self, **kwargs): + def __init__(self, *, reconnect_on_error: bool = True, **kwargs): WordTTSService.__init__(self, **kwargs) - WebsocketService.__init__(self) + WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs) + self._register_event_handler("on_connection_error") + + async def _report_error(self, error: ErrorFrame): + await self._call_event_handler("on_connection_error", error.error) + await self.push_error(error) class InterruptibleWordTTSService(WebsocketWordTTSService): diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 3b491d26b..11796464d 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -187,7 +187,7 @@ class CartesiaTTSService(AudioContextWordTTSService): async def _connect(self): await self._connect_websocket() if not self._receive_task: - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) async def _disconnect(self): if self._receive_task: @@ -207,6 +207,7 @@ class CartesiaTTSService(AudioContextWordTTSService): except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): try: diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index c41e5de02..568b9eb64 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -309,7 +309,7 @@ class ElevenLabsTTSService(InterruptibleWordTTSService): await self._connect_websocket() if not self._receive_task: - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) if not self._keepalive_task: self._keepalive_task = self.create_task(self._keepalive_task_handler()) @@ -364,6 +364,7 @@ class ElevenLabsTTSService(InterruptibleWordTTSService): except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): try: diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py index 96968d6e9..9e6a8b91e 100644 --- a/src/pipecat/services/fish.py +++ b/src/pipecat/services/fish.py @@ -107,7 +107,7 @@ class FishAudioTTSService(InterruptibleTTSService): async def _connect(self): await self._connect_websocket() if not self._receive_task: - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) async def _disconnect(self): if self._receive_task: @@ -132,6 +132,7 @@ class FishAudioTTSService(InterruptibleTTSService): except Exception as e: logger.error(f"Fish Audio initialization error: {e}") self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): try: diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py index cfdf8e6cd..d3cc92603 100644 --- a/src/pipecat/services/lmnt.py +++ b/src/pipecat/services/lmnt.py @@ -112,7 +112,7 @@ class LmntTTSService(InterruptibleTTSService): await self._connect_websocket() if not self._receive_task: - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) async def _disconnect(self): if self._receive_task: @@ -147,6 +147,7 @@ class LmntTTSService(InterruptibleTTSService): except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): """Disconnect from LMNT websocket.""" diff --git a/src/pipecat/services/neuphonic.py b/src/pipecat/services/neuphonic.py index b935885b6..407e54a83 100644 --- a/src/pipecat/services/neuphonic.py +++ b/src/pipecat/services/neuphonic.py @@ -162,7 +162,7 @@ class NeuphonicTTSService(InterruptibleTTSService): async def _connect(self): await self._connect_websocket() - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) self._keepalive_task = self.create_task(self._keepalive_task_handler()) async def _disconnect(self): @@ -197,6 +197,7 @@ class NeuphonicTTSService(InterruptibleTTSService): except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): try: diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index 80d313765..75677876f 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -160,7 +160,7 @@ class PlayHTTTSService(InterruptibleTTSService): await self._connect_websocket() if not self._receive_task: - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) async def _disconnect(self): if self._receive_task: @@ -183,12 +183,14 @@ class PlayHTTTSService(InterruptibleTTSService): raise ValueError("WebSocket URL is not a string") self._websocket = await websockets.connect(self._websocket_url) - except ValueError as ve: - logger.error(f"{self} initialization error: {ve}") + except ValueError as e: + logger.error(f"{self} initialization error: {e}") self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): try: diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index c6fc50001..a1a455eb5 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -171,7 +171,7 @@ class RimeTTSService(AudioContextWordTTSService): await self._connect_websocket() if not self._receive_task: - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) async def _disconnect(self): """Close websocket connection and clean up tasks.""" @@ -194,6 +194,7 @@ class RimeTTSService(AudioContextWordTTSService): except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): """Close websocket connection and reset state.""" diff --git a/src/pipecat/services/websocket_service.py b/src/pipecat/services/websocket_service.py index 19fb4ae01..2e82ddaba 100644 --- a/src/pipecat/services/websocket_service.py +++ b/src/pipecat/services/websocket_service.py @@ -19,9 +19,10 @@ from pipecat.utils.network import exponential_backoff_time class WebsocketService(ABC): """Base class for websocket-based services with reconnection logic.""" - def __init__(self): + def __init__(self, *, reconnect_on_error: bool = True, **kwargs): """Initialize websocket attributes.""" self._websocket: Optional[websockets.WebSocketClientProtocol] = None + self._reconnect_on_error = reconnect_on_error async def _verify_connection(self) -> bool: """Verify websocket connection is working. @@ -72,24 +73,29 @@ class WebsocketService(ABC): self._websocket.close_rcvd_then_sent, ) except Exception as e: - retry_count += 1 - if retry_count >= MAX_RETRIES: - message = f"{self} error receiving messages: {e}" - logger.error(message) - await report_error(ErrorFrame(message, fatal=True)) + message = f"{self} error receiving messages: {e}" + logger.error(message) + + if self._reconnect_on_error: + retry_count += 1 + if retry_count >= MAX_RETRIES: + await report_error(ErrorFrame(message, fatal=True)) + break + + logger.warning(f"{self} connection error, will retry: {e}") + await report_error(ErrorFrame(message)) + + try: + if await self._reconnect_websocket(retry_count): + retry_count = 0 # Reset counter on successful reconnection + wait_time = exponential_backoff_time(retry_count) + await asyncio.sleep(wait_time) + except Exception as reconnect_error: + logger.error(f"{self} reconnection failed: {reconnect_error}") + else: + await report_error(ErrorFrame(message)) break - logger.warning(f"{self} connection error, will retry: {e}") - - try: - if await self._reconnect_websocket(retry_count): - retry_count = 0 # Reset counter on successful reconnection - wait_time = exponential_backoff_time(retry_count) - await asyncio.sleep(wait_time) - except Exception as reconnect_error: - logger.error(f"{self} reconnection failed: {reconnect_error}") - continue - @abstractmethod async def _connect(self): """Implement service-specific connection logic. This function will From 07a77e066fa515b7fdf526639b3d3c077ca45dd4 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 19 Mar 2025 23:18:30 -0400 Subject: [PATCH 068/132] Update to Claude 3.7 Sonnet latest in examples --- examples/foundational/14a-function-calling-anthropic.py | 2 +- examples/foundational/14b-function-calling-anthropic-video.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/foundational/14a-function-calling-anthropic.py b/examples/foundational/14a-function-calling-anthropic.py index 13505550b..923b1488d 100644 --- a/examples/foundational/14a-function-calling-anthropic.py +++ b/examples/foundational/14a-function-calling-anthropic.py @@ -57,7 +57,7 @@ async def main(): ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-5-sonnet-20240620" + api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-7-sonnet-latest" ) llm.register_function("get_weather", get_weather) diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index 8c49900fa..302722be2 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -67,8 +67,7 @@ async def main(): llm = AnthropicLLMService( api_key=os.getenv("ANTHROPIC_API_KEY"), - # model="claude-3-5-sonnet-20240620", - model="claude-3-5-sonnet-latest", + model="claude-3-7-sonnet-latest", enable_prompt_caching_beta=True, ) llm.register_function("get_weather", get_weather) From e18d9f6a111d27ac3009ac9609b3e83d59d9b39a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 17:26:38 -0700 Subject: [PATCH 069/132] PipelineTask: automatically cancel tasks if pipeline is idle --- CHANGELOG.md | 13 +++++ src/pipecat/pipeline/task.py | 94 +++++++++++++++++++++++++++++++++++- src/pipecat/tests/utils.py | 6 ++- tests/test_pipeline.py | 91 ++++++++++++++++++++++++++++++++-- 4 files changed, 198 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35df891ef..2b0da82a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added support for detecting idle pipelines. By default, if no activity has + been detected during 5 minutes, the `PipelineTask` will be automatically + cancelled. It is possible to override this behavior by passing + `cancel_on_idle_timeout=False`. It is also possible to change the default + timeout with `idle_timeout_secs` or the frames that prevent the pipeline from + being idle with `idle_timeout_frames`. Finally, an `on_idle_timeout` event + handler will be triggered if the idle timeout is reached (whether the pipeline + task is cancelled or not). + - Added a `reconnect_on_error` parameter to websocket-based TTS services as well as a `on_connection_error` event handler. The `reconnect_on_error` indicates whether the TTS service should reconnect on error. The `on_connection_error` @@ -111,6 +120,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- ⚠️ `PipelineTask` will now be automatically cancelled if no bot activity is + happening in the pipeline. There are a few settings to configure this + behavior, see `PipelineTask` documentation for more details. + - All event handlers are now executed in separate tasks in order to prevent blocking the pipeline. It is possible that event handlers take some time to execute in which case the pipeline would be blocked waiting for the event diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index e9744fd04..cf62be64f 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -5,6 +5,7 @@ # import asyncio +import time from typing import Any, AsyncIterable, Dict, Iterable, List, Optional, Tuple, Type from loguru import logger @@ -13,6 +14,7 @@ from pydantic import BaseModel, ConfigDict from pipecat.clocks.base_clock import BaseClock from pipecat.clocks.system_clock import SystemClock from pipecat.frames.frames import ( + BotSpeakingFrame, CancelFrame, CancelTaskFrame, EndFrame, @@ -20,6 +22,7 @@ from pipecat.frames.frames import ( ErrorFrame, Frame, HeartbeatFrame, + LLMFullResponseEndFrame, MetricsFrame, StartFrame, StopFrame, @@ -133,12 +136,27 @@ class PipelineTask(BaseTask): async def on_frame_reached_downstream(task, frame): ... + It also has an event handler that detects when the pipeline is idle. By + default, a pipeline is idle if no `BotSpeakingFrame` or + `LLMFullResponseEndFrame` are received within `idle_timeout_secs`. + + @task.event_handler("on_idle_timeout") + async def on_idle_timeout(task): + ... + Args: pipeline: The pipeline to execute. params: Configuration parameters for the pipeline. observers: List of observers for monitoring pipeline execution. clock: Clock implementation for timing operations. check_dangling_tasks: Whether to check for processors' tasks finishing properly. + idle_timeout_secs: Timeout (in seconds) to consider pipeline idle or + None. If a pipeline is idle the pipeline task will be cancelled + automatically. + idle_timeout_frames: A tuple with the frames that should trigger an idle + timeout if not received withing `idle_timeout_seconds`. + cancel_on_idle_timeout: Whether the pipeline task should be cancelled if + the idle timeout is reached. """ @@ -151,12 +169,21 @@ class PipelineTask(BaseTask): clock: BaseClock = SystemClock(), task_manager: Optional[BaseTaskManager] = None, check_dangling_tasks: bool = True, + idle_timeout_secs: Optional[float] = 300, + idle_timeout_frames: Tuple[Type[Frame], ...] = ( + BotSpeakingFrame, + LLMFullResponseEndFrame, + ), + cancel_on_idle_timeout: bool = True, ): super().__init__() self._pipeline = pipeline self._clock = clock self._params = params self._check_dangling_tasks = check_dangling_tasks + self._idle_timeout_secs = idle_timeout_secs + self._idle_timeout_frames = idle_timeout_frames + self._cancel_on_idle_timeout = cancel_on_idle_timeout if self._params.observers: import warnings @@ -178,6 +205,10 @@ class PipelineTask(BaseTask): # This is the heartbeat queue. When a heartbeat frame is received in the # down queue we add it to the heartbeat queue for processing. self._heartbeat_queue = asyncio.Queue() + # This is the idle queue. When frames are received downstream they are + # put in the queue. If no frame is received the pipeline is considered + # idle. + self._idle_queue = asyncio.Queue() # This event is used to indicate a finalize frame (e.g. EndFrame, # StopFrame) has been received in the down queue. self._pipeline_end_event = asyncio.Event() @@ -213,6 +244,7 @@ class PipelineTask(BaseTask): self._reached_downstream_types: Tuple[Type[Frame], ...] = () self._register_event_handler("on_frame_reached_upstream") self._register_event_handler("on_frame_reached_downstream") + self._register_event_handler("on_idle_timeout") @property def params(self) -> PipelineParams: @@ -328,19 +360,30 @@ class PipelineTask(BaseTask): self._heartbeat_monitor_handler(), f"{self}::_heartbeat_monitor_handler" ) + def _maybe_start_idle_task(self): + if self._idle_timeout_secs: + self._idle_monitor_task = self._task_manager.create_task( + self._idle_monitor_handler(), f"{self}::_idle_monitor_handler" + ) + async def _cancel_tasks(self): - await self._maybe_cancel_heartbeat_tasks() + await self._observer.stop() await self._task_manager.cancel_task(self._process_up_task) await self._task_manager.cancel_task(self._process_down_task) - await self._observer.stop() + await self._maybe_cancel_heartbeat_tasks() + await self._maybe_cancel_idle_task() async def _maybe_cancel_heartbeat_tasks(self): if self._params.enable_heartbeats: await self._task_manager.cancel_task(self._heartbeat_push_task) await self._task_manager.cancel_task(self._heartbeat_monitor_task) + async def _maybe_cancel_idle_task(self): + if self._idle_timeout_secs: + await self._task_manager.cancel_task(self._idle_monitor_task) + def _initial_metrics_frame(self) -> MetricsFrame: processors = self._pipeline.processors_with_metrics() data = [] @@ -372,6 +415,7 @@ class PipelineTask(BaseTask): self._clock.start() self._maybe_start_heartbeat_tasks() + self._maybe_start_idle_task() start_frame = StartFrame( clock=self._clock, @@ -445,6 +489,10 @@ class PipelineTask(BaseTask): while True: frame = await self._down_queue.get() + # Queue received frame to the idle queue so we can monitor idle + # pipelines. + await self._idle_queue.put(frame) + if isinstance(frame, self._reached_downstream_types): await self._call_event_handler("on_frame_reached_downstream", frame) @@ -482,6 +530,48 @@ class PipelineTask(BaseTask): f"{self}: heartbeat frame not received for more than {wait_time} seconds" ) + async def _idle_monitor_handler(self): + """This tasks monitors activity in the pipeline. If no frames are + received (heartbeats don't count) the pipeline is considered idle. + + """ + running = True + last_frame_time = 0 + while running: + try: + frame = await asyncio.wait_for( + self._idle_queue.get(), timeout=self._idle_timeout_secs + ) + + if isinstance(frame, StartFrame) or isinstance(frame, self._idle_timeout_frames): + # If we find a StartFrame or one of the frames that prevents a + # time out we update the time. + last_frame_time = time.time() + else: + # If we find any other frame we check if the pipeline is + # idle by checking the last time we received one of the + # valid frames. + diff_time = time.time() - last_frame_time + if diff_time >= self._idle_timeout_secs: + running = await self._idle_timeout_detected() + + self._idle_queue.task_done() + except asyncio.TimeoutError: + running = await self._idle_timeout_detected() + + async def _idle_timeout_detected(self) -> bool: + """Logic for when the pipeline is idle. + + Returns: + bool: Whther the pipeline task is being cancelled or not. + """ + await self._call_event_handler("on_idle_timeout") + if self._cancel_on_idle_timeout: + logger.warning(f"Idle pipeline detected, cancelling pipeline task...") + await self.cancel() + return False + return True + def _print_dangling_tasks(self): tasks = [t.get_name() for t in self._task_manager.current_tasks()] if tasks: diff --git a/src/pipecat/tests/utils.py b/src/pipecat/tests/utils.py index 7fcecc3ff..d68c87d88 100644 --- a/src/pipecat/tests/utils.py +++ b/src/pipecat/tests/utils.py @@ -101,7 +101,11 @@ async def run_test( pipeline = Pipeline([source, processor, sink]) - task = PipelineTask(pipeline, params=PipelineParams(start_metadata=start_metadata)) + task = PipelineTask( + pipeline, + params=PipelineParams(start_metadata=start_metadata), + cancel_on_idle_timeout=False, + ) async def push_frames(): # Just give a little head start to the runner. diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index c3811a672..46114b5b2 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -5,6 +5,7 @@ # import asyncio +import time import unittest from pipecat.frames.frames import EndFrame, HeartbeatFrame, StartFrame, TextFrame @@ -100,7 +101,7 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): identity = IdentityFilter() pipeline = Pipeline([identity]) - task = PipelineTask(pipeline) + task = PipelineTask(pipeline, cancel_on_idle_timeout=False) task.set_event_loop(asyncio.get_event_loop()) task.set_reached_upstream_filter((TextFrame,)) task.set_reached_downstream_filter((TextFrame,)) @@ -123,7 +124,7 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): await task.queue_frame(TextFrame(text="Hello Downstream!")) try: - await asyncio.wait_for(task.run(), timeout=1.0) + await asyncio.wait_for(asyncio.shield(task.run()), timeout=1.0) except asyncio.TimeoutError: pass @@ -149,6 +150,7 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): heartbeats_period_secs=0.2, ), observers=[heartbeats_observer], + cancel_on_idle_timeout=False, ) task.set_event_loop(asyncio.get_event_loop()) @@ -156,7 +158,90 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): await task.queue_frame(TextFrame(text="Hello!")) try: - await asyncio.wait_for(task.run(), timeout=1.0) + await asyncio.wait_for(asyncio.shield(task.run()), timeout=1.0) except asyncio.TimeoutError: pass assert heartbeats_counter == expected_heartbeats + + async def test_idle_task(self): + identity = IdentityFilter() + pipeline = Pipeline([identity]) + task = PipelineTask(pipeline, idle_timeout_secs=0.2) + task.set_event_loop(asyncio.get_event_loop()) + await task.run() + assert True + + async def test_no_idle_task(self): + identity = IdentityFilter() + pipeline = Pipeline([identity]) + task = PipelineTask(pipeline, idle_timeout_secs=0.2, cancel_on_idle_timeout=False) + task.set_event_loop(asyncio.get_event_loop()) + try: + await asyncio.wait_for(asyncio.shield(task.run()), timeout=0.3) + except asyncio.TimeoutError: + assert True + else: + assert False + + async def test_idle_task_heartbeats(self): + identity = IdentityFilter() + pipeline = Pipeline([identity]) + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_heartbeats=True, + heartbeats_period_secs=0.1, + ), + idle_timeout_secs=0.3, + ) + task.set_event_loop(asyncio.get_event_loop()) + await task.run() + assert True + + async def test_idle_task_event_handler(self): + identity = IdentityFilter() + pipeline = Pipeline([identity]) + task = PipelineTask(pipeline, idle_timeout_secs=0.2, cancel_on_idle_timeout=False) + task.set_event_loop(asyncio.get_event_loop()) + + idle_timeout = False + + @task.event_handler("on_idle_timeout") + async def on_idle_timeout(task: PipelineTask): + nonlocal idle_timeout + idle_timeout = True + await task.cancel() + + await task.run() + assert True + + async def test_idle_task_frames(self): + idle_timeout_secs = 0.2 + sleep_time_secs = idle_timeout_secs / 2 + + identity = IdentityFilter() + pipeline = Pipeline([identity]) + task = PipelineTask( + pipeline, + idle_timeout_secs=idle_timeout_secs, + idle_timeout_frames=(TextFrame,), + ) + task.set_event_loop(asyncio.get_event_loop()) + + async def delayed_frames(): + await asyncio.sleep(sleep_time_secs) + await task.queue_frame(TextFrame("Hello Pipecat!")) + await asyncio.sleep(sleep_time_secs) + await task.queue_frame(TextFrame("Hello Pipecat!")) + await asyncio.sleep(sleep_time_secs) + await task.queue_frame(TextFrame("Hello Pipecat!")) + + start_time = time.time() + + tasks = {asyncio.create_task(task.run()), asyncio.create_task(delayed_frames())} + + await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + + diff_time = time.time() - start_time + + self.assertGreater(diff_time, sleep_time_secs * 3) From b6be25ab84d2d1bdcf49cb4493efe0b90c4f7e1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 23:56:40 -0700 Subject: [PATCH 070/132] SegmentedSTTService: use VAD events to detect valid audio --- CHANGELOG.md | 4 ++ src/pipecat/services/ai_services.py | 106 ++++++++++++---------------- 2 files changed, 50 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb4ff6185..1cdc10c9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -149,6 +149,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `SegmentedSTTService` issue that was causing audio to be sent + prematurely to the STT service. Instead of analyzing the volume in this + service we rely on VAD events which use both VAD and volume. + - Fixed a `GeminiMultimodalLiveLLMService` issue that was causing messages to be duplicated in the context when pushing `LLMMessagesAppendFrame` frames. diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index d29e0e714..e00970433 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -14,7 +14,6 @@ from loguru import logger from pipecat.adapters.base_llm_adapter import BaseLLMAdapter from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter -from pipecat.audio.utils import calculate_audio_volume, exp_smoothing from pipecat.frames.frames import ( AudioRawFrame, BotStartedSpeakingFrame, @@ -38,6 +37,8 @@ from pipecat.frames.frames import ( TTSTextFrame, TTSUpdateSettingsFrame, UserImageRequestFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, VisionImageRawFrame, ) from pipecat.metrics.metrics import MetricsData @@ -859,79 +860,64 @@ class STTService(AIService): class SegmentedSTTService(STTService): - """SegmentedSTTService is an STTService that will detect speech and will run - speech-to-text on speech segments only, instead of a continous stream. + """SegmentedSTTService is an STTService that uses VAD events to detect + speech and will run speech-to-text on speech segments only, instead of a + continous stream. Since it uses VAD it means that VAD needs to be enabled in + the pipeline. + + This service always keeps a small audio buffer to take into account that VAD + events are delayed from when the user speech really starts. """ - def __init__( - self, - *, - min_volume: float = 0.6, - max_silence_secs: float = 0.3, - max_buffer_secs: float = 1.5, - sample_rate: Optional[int] = None, - **kwargs, - ): + def __init__(self, *, sample_rate: Optional[int] = None, **kwargs): super().__init__(sample_rate=sample_rate, **kwargs) - self._min_volume = min_volume - self._max_silence_secs = max_silence_secs - self._max_buffer_secs = max_buffer_secs self._content = None self._wave = None - self._silence_num_frames = 0 - # Volume exponential smoothing - self._smoothing_factor = 0.2 - self._prev_volume = 0 - - async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection): - # Try to filter out empty background noise - volume = self._get_smoothed_volume(frame) - if volume >= self._min_volume: - # If volume is high enough, write new data to wave file - self._wave.writeframes(frame.audio) - self._silence_num_frames = 0 - else: - self._silence_num_frames += frame.num_frames - self._prev_volume = volume - - # If buffer is not empty and we have enough data or there's been a long - # silence, transcribe the audio gathered so far. - silence_secs = self._silence_num_frames / self.sample_rate - buffer_secs = self._wave.getnframes() / self.sample_rate - if self._content.tell() > 0 and ( - buffer_secs > self._max_buffer_secs or silence_secs > self._max_silence_secs - ): - self._silence_num_frames = 0 - self._wave.close() - self._content.seek(0) - await self.process_generator(self.run_stt(self._content.read())) - (self._content, self._wave) = self._new_wave() + self._audio_buffer = bytearray() + self._audio_buffer_size_1s = 0 + self._user_speaking = False async def start(self, frame: StartFrame): await super().start(frame) - if not self._wave: - (self._content, self._wave) = self._new_wave() + self._audio_buffer_size_1s = self.sample_rate * 2 - async def stop(self, frame: EndFrame): - await super().stop(frame) - self._wave.close() + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) - async def cancel(self, frame: CancelFrame): - await super().cancel(frame) - self._wave.close() + if isinstance(frame, UserStartedSpeakingFrame): + await self._handle_user_started_speaking(frame) + elif isinstance(frame, UserStoppedSpeakingFrame): + await self._handle_user_stopped_speaking(frame) + + async def _handle_user_started_speaking(self, frame: UserStartedSpeakingFrame): + self._user_speaking = True + + async def _handle_user_stopped_speaking(self, frame: UserStoppedSpeakingFrame): + self._user_speaking = False - def _new_wave(self): content = io.BytesIO() - ww = wave.open(content, "wb") - ww.setsampwidth(2) - ww.setnchannels(1) - ww.setframerate(self.sample_rate) - return (content, ww) + wav = wave.open(content, "wb") + wav.setsampwidth(2) + wav.setnchannels(1) + wav.setframerate(self.sample_rate) + wav.writeframes(self._audio_buffer) + wav.close() + content.seek(0) - def _get_smoothed_volume(self, frame: AudioRawFrame) -> float: - volume = calculate_audio_volume(frame.audio, frame.sample_rate) - return exp_smoothing(volume, self._prev_volume, self._smoothing_factor) + await self.process_generator(self.run_stt(content.read())) + + # Start clean. + self._audio_buffer.clear() + + async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection): + # If the user is speaking the audio buffer will keep growin. + self._audio_buffer += frame.audio + + # If the user is not speaking we keep just a little bit of audio. + if not self._user_speaking and len(self._audio_buffer) > self._audio_buffer_size_1s: + discarded = len(self._audio_buffer) - self._audio_buffer_size_1s + self._audio_buffer = self._audio_buffer[discarded:] class ImageGenService(AIService): From 7cdcd1c3d1ac4fbdfa2b3e2bddaf1d3513b7a512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 00:36:03 -0700 Subject: [PATCH 071/132] OpenAITTSService: allow specifying any model name --- src/pipecat/services/openai.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 5a3a993aa..89bf4a04b 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -472,22 +472,16 @@ class OpenAITTSService(TTSService): """OpenAI Text-to-Speech service that generates audio from text. This service uses the OpenAI TTS API to generate PCM-encoded audio at 24kHz. - When using with DailyTransport, configure the sample rate in DailyParams - as shown below: - - DailyParams( - audio_out_enabled=True, - audio_out_sample_rate=24_000, - ) Args: api_key: OpenAI API key. Defaults to None. voice: Voice ID to use. Defaults to "alloy". - model: TTS model to use ("tts-1" or "tts-1-hd"). Defaults to "tts-1". - sample_rate: Output audio sample rate in Hz. Defaults to 24000. + model: TTS model to use. Defaults to "tts-1". + sample_rate: Output audio sample rate in Hz. Defaults to None. **kwargs: Additional keyword arguments passed to TTSService. The service returns PCM-encoded audio at the specified sample rate. + """ OPENAI_SAMPLE_RATE = 24000 # OpenAI TTS always outputs at 24kHz @@ -497,7 +491,7 @@ class OpenAITTSService(TTSService): *, api_key: Optional[str] = None, voice: str = "alloy", - model: Literal["tts-1", "tts-1-hd"] = "tts-1", + model: str = "tts-1", sample_rate: Optional[int] = None, **kwargs, ): From 2417ec4f926eb27053dfeded8725706d3e4d3853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 00:31:54 -0700 Subject: [PATCH 072/132] LLMUserContextAggregator: increase bot_interruption_timeout to 5 seconds --- src/pipecat/processors/aggregators/llm_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index d8582e32f..aed12db9e 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -275,7 +275,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): self, context: OpenAILLMContext, aggregation_timeout: float = 1.0, - bot_interruption_timeout: float = 2.0, + bot_interruption_timeout: float = 5.0, **kwargs, ): super().__init__(context=context, role="user", **kwargs) From 0e14cec1392fbaec8d1fab4c64e728501bea945a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 01:22:33 -0700 Subject: [PATCH 073/132] pyproject: update multiple libraries --- pyproject.toml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4176ae2a0..87a34f929 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "pyloudnorm~=0.1.1", "resampy~=0.4.3", "soxr~=0.5.0", - "openai~=1.59.6" + "openai~=1.67.0" ] [project.urls] @@ -39,42 +39,43 @@ Source = "https://github.com/pipecat-ai/pipecat" Website = "https://pipecat.ai" [project.optional-dependencies] -anthropic = [ "anthropic~=0.47.2" ] -assemblyai = [ "assemblyai~=0.36.0" ] -aws = [ "boto3~=1.35.99" ] -azure = [ "azure-cognitiveservices-speech~=1.42.0"] +anthropic = [ "anthropic~=0.49.0" ] +assemblyai = [ "assemblyai~=0.37.0" ] +aws = [ "boto3~=1.37.16" ] +azure = [ "azure-cognitiveservices-speech~=1.43.0"] canonical = [ "aiofiles~=24.1.0" ] -cartesia = [ "cartesia~=1.3.1", "websockets~=13.1" ] +cartesia = [ "cartesia~=1.4.0", "websockets~=13.1" ] neuphonic = [ "pyneuphonic~=1.5.13", "websockets~=13.1" ] cerebras = [] deepseek = [] daily = [ "daily-python~=0.15.0" ] -deepgram = [ "deepgram-sdk~=3.8.0" ] +deepgram = [ "deepgram-sdk~=3.10.1" ] elevenlabs = [ "websockets~=13.1" ] -fal = [ "fal-client~=0.5.6" ] +fal = [ "fal-client~=0.5.9" ] fish = [ "ormsgpack~=1.7.0", "websockets~=13.1" ] gladia = [ "websockets~=13.1" ] -google = [ "google-cloud-speech~=2.31.0", "google-cloud-texttospeech~=2.25.0", "google-genai~=1.3.0", "google-generativeai~=0.8.4" ] +google = [ "google-cloud-speech~=2.31.1", "google-cloud-texttospeech~=2.25.1", "google-genai~=1.7.0", "google-generativeai~=0.8.4" ] grok = [] groq = [] gstreamer = [ "pygobject~=3.50.0" ] fireworks = [] krisp = [ "pipecat-ai-krisp~=0.3.0" ] koala = [ "pvkoala~=2.0.3" ] -langchain = [ "langchain~=0.3.14", "langchain-community~=0.3.14", "langchain-openai~=0.3.0" ] -livekit = [ "livekit~=0.19.1", "livekit-api~=0.8.1", "tenacity~=9.0.0" ] +langchain = [ "langchain~=0.3.20", "langchain-community~=0.3.20", "langchain-openai~=0.3.9" ] +livekit = [ "livekit~=0.22.0", "livekit-api~=0.8.2", "tenacity~=9.0.0" ] lmnt = [ "websockets~=13.1" ] local = [ "pyaudio~=0.2.14" ] moondream = [ "einops~=0.8.0", "timm~=1.0.13", "transformers~=4.48.0" ] nim = [] noisereduce = [ "noisereduce~=3.0.3" ] openai = [ "websockets~=13.1" ] -openpipe = [ "openpipe~=4.45.0" ] +openpipe = [ "openpipe~=4.48.0" ] +openrouter = [] perplexity = [] playht = [ "pyht~=0.1.12", "websockets~=13.1" ] rime = [ "websockets~=13.1" ] -riva = [ "nvidia-riva-client~=2.18.0" ] -sentry = [ "sentry-sdk~=2.20.0" ] +riva = [ "nvidia-riva-client~=2.19.0" ] +sentry = [ "sentry-sdk~=2.23.1" ] silero = [ "onnxruntime~=1.20.1" ] simli = [ "simli-ai~=0.1.10"] soundfile = [ "soundfile~=0.13.0" ] @@ -83,7 +84,6 @@ together = [] ultravox = [ "transformers~=4.48.0", "vllm~=0.7.3" ] websocket = [ "websockets~=13.1", "fastapi~=0.115.6" ] whisper = [ "faster-whisper~=1.1.1" ] -openrouter = [] [tool.setuptools.packages.find] # All the following settings are optional: From a98000fd1dcf3932f4c85a54e747d3e1d1445775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Mar 2025 14:57:27 -0700 Subject: [PATCH 074/132] function calling now run in tasks --- CHANGELOG.md | 19 +- src/pipecat/frames/frames.py | 21 ++ src/pipecat/pipeline/task.py | 2 +- .../processors/aggregators/llm_response.py | 232 +++++++++++------- .../aggregators/openai_llm_context.py | 65 +---- src/pipecat/services/ai_services.py | 200 ++++++++++++--- src/pipecat/services/anthropic.py | 164 ++++--------- .../services/gemini_multimodal_live/gemini.py | 9 +- src/pipecat/services/google/google.py | 143 +++++------ src/pipecat/services/grok.py | 85 +------ src/pipecat/services/openai.py | 161 ++++-------- tests/test_context_aggregators.py | 3 +- 12 files changed, 537 insertions(+), 567 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b931c083f..4762186cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- When registering a function call it is now possible to indicate if you want + the function call to be cancelled if there's a user interruption via + `cancel_on_interruption` (defaults to False). This is now possible because + function calls are executed concurrently. + - Added support for detecting idle pipelines. By default, if no activity has been detected during 5 minutes, the `PipelineTask` will be automatically cancelled. It is possible to override this behavior by passing @@ -120,6 +125,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Function calls are now executed in tasks. This means that the pipeline will + not be blocked while the function call is being executed. + - ⚠️ `PipelineTask` will now be automatically cancelled if no bot activity is happening in the pipeline. There are a few settings to configure this behavior, see `PipelineTask` documentation for more details. @@ -140,6 +148,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Deprecated +- Passing a `start_callback` to `LLMService.register_function()` is now + deprecated, simply move the code from the start callback to the function call. + - `TTSService` parameter `text_filter` is now deprecated, use `text_filters` instead which is now a list. This allows passing multiple filters that will be executed in order. @@ -162,6 +173,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an assistant aggregator issue that could cause assistant text to be + split into multiple chunks during function calls. + +- Fixed an assistant aggregator issue that was causing assistant text to not be + added to the context during function calls. This could lead to duplications. + - Fixed a `SegmentedSTTService` issue that was causing audio to be sent prematurely to the STT service. Instead of analyzing the volume in this service we rely on VAD events which use both VAD and volume. @@ -1978,7 +1995,7 @@ async def on_connected(processor): completed. If a task is never ran `has_finished()` will return False. - `PipelineRunner` now supports SIGTERM. If received, the runner will be - canceled. + cancelled. ### Fixed diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 74dd2accb..e589e9480 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -634,6 +634,15 @@ class FunctionCallInProgressFrame(SystemFrame): function_name: str tool_call_id: str arguments: str + cancel_on_interruption: bool + + +@dataclass +class FunctionCallCancelFrame(SystemFrame): + """A frame to signal a function call has been cancelled.""" + + function_name: str + tool_call_id: str @dataclass @@ -706,6 +715,18 @@ class VisionImageRawFrame(InputImageRawFrame): return f"{self.name}(pts: {pts}, text: [{self.text}], size: {self.size}, format: {self.format})" +@dataclass +class UserImageMessageFrame(SystemFrame): + """An image associated to a user.""" + + user_image_raw_frame: UserImageRawFrame + text: Optional[str] = None + + def __str__(self): + pts = format_pts(self.pts) + return f"{self.name}(pts: {pts}, image: {self.user_image_raw_frame}, text: {self.text})" + + # # Control frames # diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index cf62be64f..8279373cb 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -409,7 +409,7 @@ class PipelineTask(BaseTask): async def _process_push_queue(self): """This is the task that runs the pipeline for the first time by sending a StartFrame and by pushing any other frames queued by the user. It runs - until the tasks is canceled or stopped (e.g. with an EndFrame). + until the tasks is cancelled or stopped (e.g. with an EndFrame). """ self._clock.start() diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index aed12db9e..494d1de38 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -7,14 +7,20 @@ import asyncio import time from abc import abstractmethod -from typing import List +from typing import Dict, List + +from loguru import logger from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, CancelFrame, EmulateUserStartedSpeakingFrame, EmulateUserStoppedSpeakingFrame, EndFrame, Frame, + FunctionCallCancelFrame, + FunctionCallInProgressFrame, + FunctionCallResultFrame, InterimTranscriptionFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, @@ -23,10 +29,12 @@ from pipecat.frames.frames import ( LLMMessagesUpdateFrame, LLMSetToolsFrame, LLMTextFrame, + OpenAILLMContextAssistantTimestampFrame, StartFrame, StartInterruptionFrame, TextFrame, TranscriptionFrame, + UserImageMessageFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -35,6 +43,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContextFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.utils.time import time_now_iso8601 class LLMFullResponseAggregator(FrameProcessor): @@ -139,68 +148,20 @@ class BaseLLMResponseAggregator(FrameProcessor): pass @abstractmethod - async def push_aggregation(self): + async def handle_aggregation(self, aggregation: str): + """Adds the given aggregation to the aggregator. The aggregator can use + a simple list of message or a context. It doesn't not push any frames. + + """ pass - -class LLMResponseAggregator(BaseLLMResponseAggregator): - """This is a base LLM aggregator that uses a simple list of messages to - store the conversation. It pushes `LLMMessagesFrame` as an aggregation - frame. - - """ - - def __init__( - self, - *, - messages: List[dict], - role: str = "user", - **kwargs, - ): - super().__init__(**kwargs) - - self._messages = messages - self._role = role - - self._aggregation = "" - - self.reset() - - @property - def messages(self) -> List[dict]: - return self._messages - - @property - def role(self) -> str: - return self._role - - def add_messages(self, messages): - self._messages.extend(messages) - - def set_messages(self, messages): - self.reset() - self._messages.clear() - self._messages.extend(messages) - - def set_tools(self, tools): - pass - - def reset(self): - self._aggregation = "" - + @abstractmethod async def push_aggregation(self): - if len(self._aggregation) > 0: - self._messages.append({"role": self._role, "content": self._aggregation}) + """Pushes the current aggregation. For example, iN the case of context + aggregation this might push a new context frame. - # Reset the aggregation. Reset it before pushing it down, otherwise - # if the tasks gets cancelled we won't be able to clear things up. - self._aggregation = "" - - frame = LLMMessagesFrame(self._messages) - await self.push_frame(frame) - - # Reset our accumulator state. - self.reset() + """ + pass class LLMContextResponseAggregator(BaseLLMResponseAggregator): @@ -247,20 +208,6 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator): def reset(self): self._aggregation = "" - async def push_aggregation(self): - if len(self._aggregation) > 0: - self._context.add_message({"role": self.role, "content": self._aggregation}) - - # Reset the aggregation. Reset it before pushing it down, otherwise - # if the tasks gets cancelled we won't be able to clear things up. - self._aggregation = "" - - frame = OpenAILLMContextFrame(self._context) - await self.push_frame(frame) - - # Reset our accumulator state. - self.reset() - class LLMUserContextAggregator(LLMContextResponseAggregator): """This is a user LLM aggregator that uses an LLM context to store the @@ -290,12 +237,13 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): self._aggregation_event = asyncio.Event() self._aggregation_task = None - self.reset() - def reset(self): super().reset() self._seen_interim_results = False + async def handle_aggregation(self, aggregation: str): + self._context.add_message({"role": self.role, "content": self._aggregation}) + async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -331,6 +279,20 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): else: await self.push_frame(frame, direction) + async def push_aggregation(self): + if len(self._aggregation) > 0: + await self.handle_aggregation(self._aggregation) + + # Reset the aggregation. Reset it before pushing it down, otherwise + # if the tasks gets cancelled we won't be able to clear things up. + self._aggregation = "" + + frame = OpenAILLMContextFrame(self._context) + await self.push_frame(frame) + + # Reset our accumulator state. + self.reset() + async def _start(self, frame: StartFrame): self._create_aggregation_task() @@ -424,17 +386,29 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): super().__init__(context=context, role="assistant", **kwargs) self._expect_stripped_words = expect_stripped_words - self._started = False + self._started = 0 + self._function_calls_in_progress: Dict[str, FunctionCallInProgressFrame] = {} - self.reset() + async def handle_aggregation(self, aggregation: str): + self._context.add_message({"role": "assistant", "content": aggregation}) + + async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): + pass + + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + pass + + async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): + pass + + async def handle_image_frame_message(self, frame: UserImageMessageFrame): + pass async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) if isinstance(frame, StartInterruptionFrame): - await self.push_aggregation() - # Reset anyways - self.reset() + await self._handle_interruptions(frame) await self.push_frame(frame, direction) elif isinstance(frame, LLMFullResponseStartFrame): await self._handle_llm_start(frame) @@ -448,14 +422,104 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): self.set_messages(frame.messages) elif isinstance(frame, LLMSetToolsFrame): self.set_tools(frame.tools) + elif isinstance(frame, FunctionCallInProgressFrame): + await self._handle_function_call_in_progress(frame) + elif isinstance(frame, FunctionCallResultFrame): + await self._handle_function_call_result(frame) + elif isinstance(frame, FunctionCallCancelFrame): + await self._handle_function_call_cancel(frame) + elif isinstance(frame, UserImageMessageFrame): + await self._handle_image_frame_message(frame) + elif isinstance(frame, BotStoppedSpeakingFrame): + await self.push_aggregation() else: await self.push_frame(frame, direction) + async def push_aggregation(self): + if not self._aggregation: + return + + aggregation = self._aggregation.strip() + self.reset() + + if aggregation: + await self.handle_aggregation(aggregation) + + # Push context frame + await self.push_context_frame() + + # Push timestamp frame with current time + timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) + await self.push_frame(timestamp_frame) + + async def _handle_interruptions(self, frame: StartInterruptionFrame): + await self.push_aggregation() + self._started = 0 + self.reset() + + async def _handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): + logger.debug( + f"{self} FunctionCallInProgressFrame: [{frame.function_name}:{frame.tool_call_id}]" + ) + await self.handle_function_call_in_progress(frame) + self._function_calls_in_progress[frame.tool_call_id] = frame + + async def _handle_function_call_result(self, frame: FunctionCallResultFrame): + logger.debug( + f"{self} FunctionCallResultFrame: [{frame.function_name}:{frame.tool_call_id}]" + ) + if frame.tool_call_id not in self._function_calls_in_progress: + logger.warning( + f"FunctionCallResultFrame tool_call_id [{frame.tool_call_id}] is not running" + ) + return + + del self._function_calls_in_progress[frame.tool_call_id] + + properties = frame.properties + + await self.handle_function_call_result(frame) + + # Run inference if the function call result requires it. + if frame.result: + run_llm = False + + if properties and properties.run_llm is not None: + # If the tool call result has a run_llm property, use it + run_llm = properties.run_llm + else: + # Default behavior is to run the LLM if there are no function calls in progress + run_llm = not bool(self._function_calls_in_progress) + + if run_llm: + await self.push_context_frame(FrameDirection.UPSTREAM) + + # Emit the on_context_updated callback once the function call + # result is added to the context + if properties and properties.on_context_updated: + await properties.on_context_updated() + + async def _handle_function_call_cancel(self, frame: FunctionCallCancelFrame): + logger.debug( + f"{self} FunctionCallCancelFrame: [{frame.function_name}:{frame.tool_call_id}]" + ) + if frame.tool_call_id not in self._function_calls_in_progress: + return + + if self._function_calls_in_progress[frame.tool_call_id].cancel_on_interruption: + await self.handle_function_call_cancel(frame) + del self._function_calls_in_progress[frame.tool_call_id] + + async def _handle_image_frame_message(self, frame: UserImageMessageFrame): + await self.handle_image_frame_message(frame) + await self.push_aggregation() + await self.push_context_frame(FrameDirection.UPSTREAM) + async def _handle_llm_start(self, _: LLMFullResponseStartFrame): - self._started = True + self._started += 1 async def _handle_llm_end(self, _: LLMFullResponseEndFrame): - self._started = False + self._started -= 1 await self.push_aggregation() async def _handle_text(self, frame: TextFrame): @@ -474,7 +538,7 @@ class LLMUserResponseAggregator(LLMUserContextAggregator): async def push_aggregation(self): if len(self._aggregation) > 0: - self._context.add_message({"role": self.role, "content": self._aggregation}) + await self.handle_aggregation(self._aggregation) # Reset the aggregation. Reset it before pushing it down, otherwise # if the tasks gets cancelled we won't be able to clear things up. @@ -493,7 +557,7 @@ class LLMAssistantResponseAggregator(LLMAssistantContextAggregator): async def push_aggregation(self): if len(self._aggregation) > 0: - self._context.add_message({"role": self.role, "content": self._aggregation}) + await self.handle_aggregation(self._aggregation) # Reset the aggregation. Reset it before pushing it down, otherwise # if the tasks gets cancelled we won't be able to clear things up. diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index e8391d62b..93c2875be 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -9,9 +9,8 @@ import copy import io import json from dataclasses import dataclass -from typing import Any, Awaitable, Callable, List, Optional +from typing import Any, List, Optional -from loguru import logger from openai._types import NOT_GIVEN, NotGiven from openai.types.chat import ( ChatCompletionMessageParam, @@ -22,12 +21,7 @@ from PIL import Image from pipecat.adapters.base_llm_adapter import BaseLLMAdapter from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.frames.frames import ( - AudioRawFrame, - Frame, - FunctionCallInProgressFrame, - FunctionCallResultFrame, -) +from pipecat.frames.frames import AudioRawFrame, Frame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor # JSON custom encoder to handle bytes arrays so that we can log contexts @@ -187,61 +181,6 @@ class OpenAILLMContext: # todo: implement for OpenAI models and others pass - async def call_function( - self, - f: Callable[ - [str, str, Any, FrameProcessor, "OpenAILLMContext", Callable[[Any], Awaitable[None]]], - Awaitable[None], - ], - *, - function_name: str, - tool_call_id: str, - arguments: str, - llm: FrameProcessor, - run_llm: bool = True, - ) -> None: - logger.info(f"Calling function {function_name} with arguments {arguments}") - # Push a SystemFrame downstream. This frame will let our assistant context aggregator - # know that we are in the middle of a function call. Some contexts/aggregators may - # not need this. But some definitely do (Anthropic, for example). - # Also push a SystemFrame upstream for use by other processors, like STTMuteFilter. - progress_frame_downstream = FunctionCallInProgressFrame( - function_name=function_name, - tool_call_id=tool_call_id, - arguments=arguments, - ) - progress_frame_upstream = FunctionCallInProgressFrame( - function_name=function_name, - tool_call_id=tool_call_id, - arguments=arguments, - ) - - # Push frame both downstream and upstream - await llm.push_frame(progress_frame_downstream, FrameDirection.DOWNSTREAM) - await llm.push_frame(progress_frame_upstream, FrameDirection.UPSTREAM) - - # Define a callback function that pushes a FunctionCallResultFrame upstream & downstream. - async def function_call_result_callback(result, *, properties=None): - result_frame_downstream = FunctionCallResultFrame( - function_name=function_name, - tool_call_id=tool_call_id, - arguments=arguments, - result=result, - properties=properties, - ) - result_frame_upstream = FunctionCallResultFrame( - function_name=function_name, - tool_call_id=tool_call_id, - arguments=arguments, - result=result, - properties=properties, - ) - - await llm.push_frame(result_frame_downstream, FrameDirection.DOWNSTREAM) - await llm.push_frame(result_frame_upstream, FrameDirection.UPSTREAM) - - await f(function_name, tool_call_id, arguments, llm, self, function_call_result_callback) - def create_wav_header(self, sample_rate, num_channels, bits_per_sample, data_size): # RIFF chunk descriptor header = bytearray() diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index e00970433..2f30f505f 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -8,7 +8,8 @@ import asyncio import io import wave from abc import abstractmethod -from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Sequence, Tuple, Type +from dataclasses import dataclass +from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Sequence, Set, Tuple, Type from loguru import logger @@ -22,6 +23,9 @@ from pipecat.frames.frames import ( EndFrame, ErrorFrame, Frame, + FunctionCallCancelFrame, + FunctionCallInProgressFrame, + FunctionCallResultFrame, InterimTranscriptionFrame, LLMFullResponseEndFrame, StartFrame, @@ -138,6 +142,13 @@ class AIService(FrameProcessor): await self.push_frame(f) +@dataclass +class FunctionEntry: + function_name: Optional[str] + callback: Any # TODO(aleix): add proper typing. + cancel_on_interruption: bool + + class LLMService(AIService): """This class is a no-op but serves as a base class for LLM services.""" @@ -147,38 +158,74 @@ class LLMService(AIService): def __init__(self, **kwargs): super().__init__(**kwargs) - self._callbacks = {} + self._functions = {} self._start_callbacks = {} self._adapter = self.adapter_class() + self._function_call_tasks: Set[Tuple[asyncio.Task, str, str]] = set() + + self._register_event_handler("on_completion_timeout") def get_llm_adapter(self) -> BaseLLMAdapter: return self._adapter def create_context_aggregator( - self, context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True + self, + context: OpenAILLMContext, + *, + user_kwargs: Mapping[str, Any] = {}, + assistant_kwargs: Mapping[str, Any] = {}, ) -> Any: pass - self._register_event_handler("on_completion_timeout") + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) - # TODO-CB: callback function type - def register_function(self, function_name: Optional[str], callback, start_callback=None): + if isinstance(frame, StartInterruptionFrame): + await self._handle_interruptions(frame) + + async def _handle_interruptions(self, frame: StartInterruptionFrame): + for function_name, entry in self._functions.items(): + if entry.cancel_on_interruption: + await self._cancel_function_call(function_name) + + def register_function( + self, + function_name: Optional[str], + callback: Any, + start_callback=None, + *, + cancel_on_interruption: bool = False, + ): # Registering a function with the function_name set to None will run that callback # for all functions - self._callbacks[function_name] = callback - # QUESTION FOR CB: maybe this isn't needed anymore? + self._functions[function_name] = FunctionEntry( + function_name=function_name, + callback=callback, + cancel_on_interruption=cancel_on_interruption, + ) + + # Start callbacks are now deprecated. if start_callback: + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Parameter 'start_callback' is deprecated, just put your code on top of the actual function call instead.", + DeprecationWarning, + ) + self._start_callbacks[function_name] = start_callback def unregister_function(self, function_name: Optional[str]): - del self._callbacks[function_name] + del self._functions[function_name] if self._start_callbacks[function_name]: del self._start_callbacks[function_name] def has_function(self, function_name: str): - if None in self._callbacks.keys(): + if None in self._functions.keys(): return True - return function_name in self._callbacks.keys() + return function_name in self._functions.keys() async def call_function( self, @@ -188,25 +235,18 @@ class LLMService(AIService): function_name: str, arguments: str, run_llm: bool = True, - ) -> None: - f = None - if function_name in self._callbacks.keys(): - f = self._callbacks[function_name] - elif None in self._callbacks.keys(): - f = self._callbacks[None] - else: - return None - await self.call_start_function(context, function_name) - await context.call_function( - f, - function_name=function_name, - tool_call_id=tool_call_id, - arguments=arguments, - llm=self, - run_llm=run_llm, + ): + if not function_name in self._functions.keys() and not None in self._functions.keys(): + return + + task = self.create_task( + self._run_function_call(context, tool_call_id, function_name, arguments, run_llm) ) - # QUESTION FOR CB: maybe this isn't needed anymore? + self._function_call_tasks.add((task, tool_call_id, function_name)) + + task.add_done_callback(self._function_call_task_finished) + async def call_start_function(self, context: OpenAILLMContext, function_name: str): if function_name in self._start_callbacks.keys(): await self._start_callbacks[function_name](function_name, self, context) @@ -218,6 +258,106 @@ class LLMService(AIService): UserImageRequestFrame(user_id=user_id, context=text_content), FrameDirection.UPSTREAM ) + async def _run_function_call( + self, + context: OpenAILLMContext, + tool_call_id: str, + function_name: str, + arguments: str, + run_llm: bool = True, + ): + if function_name in self._functions.keys(): + entry = self._functions[function_name] + elif None in self._functions.keys(): + entry = self._functions[None] + else: + return + + logger.info(f"Calling function {function_name} with arguments {arguments}") + + # NOTE(aleix): This needs to be removed after we remove the deprecation. + await self.call_start_function(context, function_name) + + # Push a SystemFrame downstream. This frame will let our assistant context aggregator + # know that we are in the middle of a function call. Some contexts/aggregators may + # not need this. But some definitely do (Anthropic, for example). + # Also push a SystemFrame upstream for use by other processors, like STTMuteFilter. + progress_frame_downstream = FunctionCallInProgressFrame( + function_name=function_name, + tool_call_id=tool_call_id, + arguments=arguments, + cancel_on_interruption=entry.cancel_on_interruption, + ) + progress_frame_upstream = FunctionCallInProgressFrame( + function_name=function_name, + tool_call_id=tool_call_id, + arguments=arguments, + cancel_on_interruption=entry.cancel_on_interruption, + ) + + # Push frame both downstream and upstream + await self.push_frame(progress_frame_downstream, FrameDirection.DOWNSTREAM) + await self.push_frame(progress_frame_upstream, FrameDirection.UPSTREAM) + + # Define a callback function that pushes a FunctionCallResultFrame upstream & downstream. + async def function_call_result_callback(result, *, properties=None): + result_frame_downstream = FunctionCallResultFrame( + function_name=function_name, + tool_call_id=tool_call_id, + arguments=arguments, + result=result, + properties=properties, + ) + result_frame_upstream = FunctionCallResultFrame( + function_name=function_name, + tool_call_id=tool_call_id, + arguments=arguments, + result=result, + properties=properties, + ) + + await self.push_frame(result_frame_downstream, FrameDirection.DOWNSTREAM) + await self.push_frame(result_frame_upstream, FrameDirection.UPSTREAM) + + await entry.callback( + function_name, tool_call_id, arguments, self, context, function_call_result_callback + ) + + async def _cancel_function_call(self, function_name: str): + cancelled_tasks = set() + for task, tool_call_id, name in self._function_call_tasks: + if name == function_name: + # We remove the callback because we are going to cancel the task + # now, otherwise we will be removing it from the set while we + # are iterating. + task.remove_done_callback(self._function_call_task_finished) + + logger.debug(f"{self} Cancelling function call [{name}:{tool_call_id}]...") + + await self.cancel_task(task) + + frame = FunctionCallCancelFrame( + function_name=function_name, tool_call_id=tool_call_id + ) + await self.push_frame(frame) + + logger.debug(f"{self} Function call [{name}:{tool_call_id}] has been cancelled") + + cancelled_tasks.add(task) + + # Remove all cancelled tasks from our set. + for task in cancelled_tasks: + self._function_call_task_finished(task) + + def _function_call_task_finished(self, task: asyncio.Task): + tuple_to_remove = next((t for t in self._function_call_tasks if t[0] == task), None) + if tuple_to_remove: + self._function_call_tasks.discard(tuple_to_remove) + # The task is finished so this should exit immediately. We need to + # do this because otherwise the task manager would have a dangling + # task if we don't remove it. + asyncio.run_coroutine_threadsafe(self.wait_for_task(task), self.get_event_loop()) + class TTSService(AIService): def __init__( @@ -366,12 +506,14 @@ class TTSService(AIService): else: await self.push_frame(frame, direction) elif isinstance(frame, TTSSpeakFrame): + # Store if we were processing text or not so we can set it back. + processing_text = self._processing_text await self._push_tts_frames(frame.text) # We pause processing incoming frames because we are sending data to # the TTS. We pause to avoid audio overlapping. await self._maybe_pause_frame_processing() await self.flush_audio() - self._processing_text = False + self._processing_text = processing_text elif isinstance(frame, TTSUpdateSettingsFrame): await self._update_settings(frame.settings) elif isinstance(frame, BotStoppedSpeakingFrame): diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index 10a2ab7b7..8e06f4558 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -21,17 +21,16 @@ from pydantic import BaseModel, Field from pipecat.adapters.services.anthropic_adapter import AnthropicLLMAdapter from pipecat.frames.frames import ( Frame, + FunctionCallCancelFrame, FunctionCallInProgressFrame, FunctionCallResultFrame, - FunctionCallResultProperties, LLMEnablePromptCachingFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMessagesFrame, LLMTextFrame, LLMUpdateSettingsFrame, - OpenAILLMContextAssistantTimestampFrame, - StartInterruptionFrame, + UserImageMessageFrame, UserImageRawFrame, UserImageRequestFrame, VisionImageRawFrame, @@ -47,7 +46,6 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import LLMService -from pipecat.utils.time import time_now_iso8601 try: from anthropic import NOT_GIVEN, AsyncAnthropic, NotGiven @@ -60,13 +58,6 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -# internal use only -- todo: refactor -@dataclass -class AnthropicImageMessageFrame(Frame): - user_image_raw_frame: UserImageRawFrame - text: Optional[str] = None - - @dataclass class AnthropicContextAggregatorPair: _user: "AnthropicUserContextAggregator" @@ -715,7 +706,7 @@ class AnthropicUserContextAggregator(LLMUserContextAggregator): text = self._context._user_image_request_context.get(frame.user_id) or "" if text: del self._context._user_image_request_context[frame.user_id] - frame = AnthropicImageMessageFrame(user_image_raw_frame=frame, text=text) + frame = UserImageMessageFrame(user_image_raw_frame=frame, text=text) await self.push_frame(frame) except Exception as e: logger.error(f"Error processing frame: {e}") @@ -734,110 +725,61 @@ class AnthropicUserContextAggregator(LLMUserContextAggregator): class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): def __init__(self, context: OpenAILLMContext | AnthropicLLMContext, **kwargs): super().__init__(context=context, **kwargs) - self._function_call_in_progress = None - self._function_call_result = None - self._pending_image_frame_message = None - async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - # See note above about not calling push_frame() here. - if isinstance(frame, StartInterruptionFrame): - self._function_call_in_progress = None - self._function_call_finished = None - elif isinstance(frame, FunctionCallInProgressFrame): - self._function_call_in_progress = frame - elif isinstance(frame, FunctionCallResultFrame): - if ( - self._function_call_in_progress - and self._function_call_in_progress.tool_call_id == frame.tool_call_id - ): - self._function_call_in_progress = None - self._function_call_result = frame - await self.push_aggregation() - else: - logger.warning( - "FunctionCallResultFrame tool_call_id != InProgressFrame tool_call_id" - ) - self._function_call_in_progress = None - self._function_call_result = None - elif isinstance(frame, AnthropicImageMessageFrame): - self._pending_image_frame_message = frame - await self.push_aggregation() + async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): + assistant_message = {"role": "assistant", "content": []} + assistant_message["content"].append( + { + "type": "tool_use", + "id": frame.tool_call_id, + "name": frame.function_name, + "input": frame.arguments, + } + ) + self._context.add_message(assistant_message) + self._context.add_message( + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": frame.tool_call_id, + "content": "IN_PROGRESS", + } + ], + } + ) - async def push_aggregation(self): - if not ( - self._aggregation or self._function_call_result or self._pending_image_frame_message - ): + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + if not frame.result: return - run_llm = False - properties: Optional[FunctionCallResultProperties] = None + result = json.dumps(frame.result) - aggregation = self._aggregation.strip() - self.reset() + await self._update_function_call_result(frame.function_name, frame.tool_call_id, result) - try: - if aggregation: - self._context.add_message({"role": "assistant", "content": aggregation}) + async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): + await self._update_function_call_result( + frame.function_name, frame.tool_call_id, "CANCELLED" + ) - if self._function_call_result: - frame = self._function_call_result - properties = frame.properties - self._function_call_result = None - if frame.result: - assistant_message = {"role": "assistant", "content": []} - assistant_message["content"].append( - { - "type": "tool_use", - "id": frame.tool_call_id, - "name": frame.function_name, - "input": frame.arguments, - } - ) - self._context.add_message(assistant_message) - self._context.add_message( - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": frame.tool_call_id, - "content": json.dumps(frame.result), - } - ], - } - ) - if properties and properties.run_llm is not None: - # If the tool call result has a run_llm property, use it - run_llm = properties.run_llm - else: - # Default behavior - run_llm = True + async def _update_function_call_result( + self, function_name: str, tool_call_id: str, result: str + ): + for message in self._context.messages: + if message["role"] == "user": + for content in message["content"]: + if ( + isinstance(content, dict) + and content["type"] == "tool_result" + and content["tool_use_id"] == tool_call_id + ): + content["content"] = result - if self._pending_image_frame_message: - frame = self._pending_image_frame_message - self._pending_image_frame_message = None - self._context.add_image_frame_message( - format=frame.user_image_raw_frame.format, - size=frame.user_image_raw_frame.size, - image=frame.user_image_raw_frame.image, - text=frame.text, - ) - run_llm = True - - if run_llm: - await self.push_context_frame(FrameDirection.UPSTREAM) - - # Emit the on_context_updated callback once the function call result is added to the context - if properties and properties.on_context_updated is not None: - await properties.on_context_updated() - - # Push context frame - await self.push_context_frame() - - # Push timestamp frame with current time - timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) - await self.push_frame(timestamp_frame) - - except Exception as e: - logger.error(f"Error processing frame: {e}") + async def handle_image_frame_message(self, frame: UserImageMessageFrame): + self._context.add_image_frame_message( + format=frame.user_image_raw_frame.format, + size=frame.user_image_raw_frame.size, + image=frame.user_image_raw_frame.image, + text=frame.text, + ) diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 9484fc27d..da1697da6 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -39,6 +39,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, TTSTextFrame, + UserImageMessageFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -118,10 +119,10 @@ class GeminiMultimodalLiveUserContextAggregator(OpenAIUserContextAggregator): class GeminiMultimodalLiveAssistantContextAggregator(OpenAIAssistantContextAggregator): - async def push_aggregation(self): - # We don't want to store any images in the context. Revisit this later when the API evolves. - self._pending_image_frame_message = None - await super().push_aggregation() + async def handle_image_frame_message(self, frame: UserImageMessageFrame): + # We don't want to store any images in the context. Revisit this later + # when the API evolves. + pass @dataclass diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index c10e67531..c07707899 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -10,6 +10,7 @@ import io import json import os import time +import uuid from google.api_core.exceptions import DeadlineExceeded from openai import AsyncStream @@ -33,20 +34,22 @@ from pipecat.frames.frames import ( EndFrame, ErrorFrame, Frame, - FunctionCallResultProperties, + FunctionCallCancelFrame, + FunctionCallInProgressFrame, + FunctionCallResultFrame, InterimTranscriptionFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMessagesFrame, LLMTextFrame, LLMUpdateSettingsFrame, - OpenAILLMContextAssistantTimestampFrame, StartFrame, TranscriptionFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, URLImageRawFrame, + UserImageMessageFrame, VisionImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage @@ -565,91 +568,69 @@ class GoogleUserContextAggregator(OpenAIUserContextAggregator): class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): - async def push_aggregation(self): - if not ( - self._aggregation or self._function_call_result or self._pending_image_frame_message - ): + async def handle_aggregation(self, aggregation: str): + self._context.add_message(glm.Content(role="model", parts=[glm.Part(text=aggregation)])) + + async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): + self._context.add_message( + glm.Content( + role="model", + parts=[ + glm.Part( + function_call=glm.FunctionCall( + id=frame.tool_call_id, name=frame.function_name, args=frame.arguments + ) + ) + ], + ) + ) + self._context.add_message( + glm.Content( + role="user", + parts=[ + glm.Part( + function_response=glm.FunctionResponse( + id=frame.tool_call_id, + name=frame.function_name, + response={"response": "IN_PROGRESS"}, + ) + ) + ], + ) + ) + + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + if not frame.result: return - run_llm = False - properties: Optional[FunctionCallResultProperties] = None + if not isinstance(frame.result, str): + return - aggregation = self._aggregation.strip() - self.reset() + response = {"response": frame.result} - try: - if aggregation: - self._context.add_message( - glm.Content(role="model", parts=[glm.Part(text=aggregation)]) - ) + await self._update_function_call_result(frame.function_name, frame.tool_call_id, response) - if self._function_call_result: - frame = self._function_call_result - properties = frame.properties - self._function_call_result = None - if frame.result: - logger.debug(f"FunctionCallResultFrame result: {frame.arguments}") - self._context.add_message( - glm.Content( - role="model", - parts=[ - glm.Part( - function_call=glm.FunctionCall( - name=frame.function_name, args=frame.arguments - ) - ) - ], - ) - ) - response = frame.result - if isinstance(response, str): - response = {"response": response} - self._context.add_message( - glm.Content( - role="user", - parts=[ - glm.Part( - function_response=glm.FunctionResponse( - name=frame.function_name, response=response - ) - ) - ], - ) - ) - if properties and properties.run_llm is not None: - # If the tool call result has a run_llm property, use it - run_llm = properties.run_llm - else: - # Default behavior is to run the LLM if there are no function calls in progress - run_llm = not bool(self._function_calls_in_progress) + async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): + await self._update_function_call_result( + frame.function_name, frame.tool_call_id, "CANCELLED" + ) - if self._pending_image_frame_message: - frame = self._pending_image_frame_message - self._pending_image_frame_message = None - self._context.add_image_frame_message( - format=frame.user_image_raw_frame.format, - size=frame.user_image_raw_frame.size, - image=frame.user_image_raw_frame.image, - text=frame.text, - ) - run_llm = True + async def _update_function_call_result( + self, function_name: str, tool_call_id: str, result: Any + ): + for message in self._context.messages: + if message.role == "user": + for part in message.parts: + if part.function_response and part.function_response.id == tool_call_id: + part.function_response.response = result - if run_llm: - await self.push_context_frame(FrameDirection.UPSTREAM) - - # Emit the on_context_updated callback once the function call result is added to the context - if properties and properties.on_context_updated is not None: - await properties.on_context_updated() - - # Push context frame - await self.push_context_frame() - - # Push timestamp frame with current time - timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) - await self.push_frame(timestamp_frame) - - except Exception as e: - logger.exception(f"Error processing frame: {e}") + async def handle_image_frame_message(self, frame: UserImageMessageFrame): + self._context.add_image_frame_message( + format=frame.user_image_raw_frame.format, + size=frame.user_image_raw_frame.size, + image=frame.user_image_raw_frame.image, + text=frame.text, + ) @dataclass @@ -1071,7 +1052,7 @@ class GoogleLLMService(LLMService): args = type(c.function_call).to_dict(c.function_call).get("args", {}) await self.call_function( context=context, - tool_call_id="what_should_this_be", + tool_call_id=str(uuid.uuid4()), function_name=c.function_call.name, arguments=args, ) diff --git a/src/pipecat/services/grok.py b/src/pipecat/services/grok.py index cf7d74f59..faed13050 100644 --- a/src/pipecat/services/grok.py +++ b/src/pipecat/services/grok.py @@ -25,94 +25,15 @@ from pipecat.services.openai import ( ) -class GrokAssistantContextAggregator(OpenAIAssistantContextAggregator): - """Custom assistant context aggregator for Grok that handles empty content requirement.""" - - async def push_aggregation(self): - if not ( - self._aggregation or self._function_call_result or self._pending_image_frame_message - ): - return - - run_llm = False - properties: Optional[FunctionCallResultProperties] = None - - aggregation = self._aggregation.strip() - self.reset() - - try: - if aggregation: - self._context.add_message({"role": "assistant", "content": aggregation}) - - if self._function_call_result: - frame = self._function_call_result - properties = frame.properties - self._function_call_result = None - if frame.result: - # Grok requires an empty content field for function calls - self._context.add_message( - { - "role": "assistant", - "content": "", # Required by Grok - "tool_calls": [ - { - "id": frame.tool_call_id, - "function": { - "name": frame.function_name, - "arguments": json.dumps(frame.arguments), - }, - "type": "function", - } - ], - } - ) - self._context.add_message( - { - "role": "tool", - "content": json.dumps(frame.result), - "tool_call_id": frame.tool_call_id, - } - ) - if properties and properties.run_llm is not None: - # If the tool call result has a run_llm property, use it - run_llm = properties.run_llm - else: - # Default behavior is to run the LLM if there are no function calls in progress - run_llm = not bool(self._function_calls_in_progress) - - if self._pending_image_frame_message: - frame = self._pending_image_frame_message - self._pending_image_frame_message = None - self._context.add_image_frame_message( - format=frame.user_image_raw_frame.format, - size=frame.user_image_raw_frame.size, - image=frame.user_image_raw_frame.image, - text=frame.text, - ) - run_llm = True - - if run_llm: - await self.push_context_frame(FrameDirection.UPSTREAM) - - # Emit the on_context_updated callback once the function call result is added to the context - if properties and properties.on_context_updated is not None: - await properties.on_context_updated() - - await self.push_context_frame() - - except Exception as e: - logger.error(f"Error processing frame: {e}") - - @dataclass class GrokContextAggregatorPair: _user: "OpenAIUserContextAggregator" - _assistant: "GrokAssistantContextAggregator" + _assistant: "OpenAIAssistantContextAggregator" def user(self) -> "OpenAIUserContextAggregator": return self._user - def assistant(self) -> "GrokAssistantContextAggregator": + def assistant(self) -> "OpenAIAssistantContextAggregator": return self._assistant @@ -235,5 +156,5 @@ class GrokLLMService(OpenAILLMService): context.set_llm_adapter(self.get_llm_adapter()) user = OpenAIUserContextAggregator(context, **user_kwargs) - assistant = GrokAssistantContextAggregator(context, **assistant_kwargs) + assistant = OpenAIAssistantContextAggregator(context, **assistant_kwargs) return GrokContextAggregatorPair(_user=user, _assistant=assistant) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 89bf4a04b..700f1c0b5 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -27,21 +27,20 @@ from pydantic import BaseModel, Field from pipecat.frames.frames import ( ErrorFrame, Frame, + FunctionCallCancelFrame, FunctionCallInProgressFrame, FunctionCallResultFrame, - FunctionCallResultProperties, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMessagesFrame, LLMTextFrame, LLMUpdateSettingsFrame, - OpenAILLMContextAssistantTimestampFrame, StartFrame, - StartInterruptionFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, URLImageRawFrame, + UserImageMessageFrame, UserImageRawFrame, UserImageRequestFrame, VisionImageRawFrame, @@ -63,7 +62,6 @@ from pipecat.services.ai_services import ( ) from pipecat.services.base_whisper import BaseWhisperSTTService, Transcription from pipecat.transcriptions.language import Language -from pipecat.utils.time import time_now_iso8601 ValidVoice = Literal["alloy", "echo", "fable", "onyx", "nova", "shimmer"] @@ -558,13 +556,6 @@ class OpenAITTSService(TTSService): logger.exception(f"{self} error generating TTS: {e}") -# internal use only -- todo: refactor -@dataclass -class OpenAIImageMessageFrame(Frame): - user_image_raw_frame: UserImageRawFrame - text: Optional[str] = None - - class OpenAIUserContextAggregator(LLMUserContextAggregator): def __init__(self, context: OpenAILLMContext, **kwargs): super().__init__(context=context, **kwargs) @@ -596,7 +587,7 @@ class OpenAIUserContextAggregator(LLMUserContextAggregator): text = self._context._user_image_request_context.get(frame.user_id) or "" if text: del self._context._user_image_request_context[frame.user_id] - frame = OpenAIImageMessageFrame(user_image_raw_frame=frame, text=text) + frame = UserImageMessageFrame(user_image_raw_frame=frame, text=text) await self.push_frame(frame) except Exception as e: logger.error(f"Error processing frame: {e}") @@ -605,109 +596,59 @@ class OpenAIUserContextAggregator(LLMUserContextAggregator): class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): def __init__(self, context: OpenAILLMContext, **kwargs): super().__init__(context=context, **kwargs) - self._function_calls_in_progress = {} - self._function_call_result = None - self._pending_image_frame_message = None - async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - # See note above about not calling push_frame() here. - if isinstance(frame, StartInterruptionFrame): - self._function_calls_in_progress.clear() - self._function_call_finished = None - elif isinstance(frame, FunctionCallInProgressFrame): - logger.debug(f"FunctionCallInProgressFrame: {frame}") - self._function_calls_in_progress[frame.tool_call_id] = frame - elif isinstance(frame, FunctionCallResultFrame): - logger.debug(f"FunctionCallResultFrame: {frame}") - if frame.tool_call_id in self._function_calls_in_progress: - del self._function_calls_in_progress[frame.tool_call_id] - self._function_call_result = frame - # TODO-CB: Kwin wants us to refactor this out of here but I REFUSE - await self.push_aggregation() - else: - logger.warning( - "FunctionCallResultFrame tool_call_id does not match any function call in progress" - ) - self._function_call_result = None - elif isinstance(frame, OpenAIImageMessageFrame): - self._pending_image_frame_message = frame - await self.push_aggregation() + async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): + self._context.add_message( + { + "role": "assistant", + "tool_calls": [ + { + "id": frame.tool_call_id, + "function": { + "name": frame.function_name, + "arguments": json.dumps(frame.arguments), + }, + "type": "function", + } + ], + } + ) + self._context.add_message( + { + "role": "tool", + "content": "IN_PROGRESS", + "tool_call_id": frame.tool_call_id, + } + ) - async def push_aggregation(self): - if not ( - self._aggregation or self._function_call_result or self._pending_image_frame_message - ): + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + if not frame.result: return - run_llm = False - properties: Optional[FunctionCallResultProperties] = None + result = json.dumps(frame.result) - aggregation = self._aggregation.strip() - self.reset() + await self._update_function_call_result(frame.function_name, frame.tool_call_id, result) - try: - if aggregation: - self._context.add_message({"role": "assistant", "content": aggregation}) + async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): + await self._update_function_call_result( + frame.function_name, frame.tool_call_id, "CANCELLED" + ) - if self._function_call_result: - frame = self._function_call_result - properties = frame.properties - self._function_call_result = None - if frame.result: - self._context.add_message( - { - "role": "assistant", - "tool_calls": [ - { - "id": frame.tool_call_id, - "function": { - "name": frame.function_name, - "arguments": json.dumps(frame.arguments), - }, - "type": "function", - } - ], - } - ) - self._context.add_message( - { - "role": "tool", - "content": json.dumps(frame.result), - "tool_call_id": frame.tool_call_id, - } - ) - if properties and properties.run_llm is not None: - # If the tool call result has a run_llm property, use it - run_llm = properties.run_llm - else: - # Default behavior is to run the LLM if there are no function calls in progress - run_llm = not bool(self._function_calls_in_progress) + async def _update_function_call_result( + self, function_name: str, tool_call_id: str, result: str + ): + for message in self._context.messages: + if ( + message["role"] == "tool" + and message["tool_call_id"] + and message["tool_call_id"] == tool_call_id + ): + message["content"] = result - if self._pending_image_frame_message: - frame = self._pending_image_frame_message - self._pending_image_frame_message = None - self._context.add_image_frame_message( - format=frame.user_image_raw_frame.format, - size=frame.user_image_raw_frame.size, - image=frame.user_image_raw_frame.image, - text=frame.text, - ) - run_llm = True - - if run_llm: - await self.push_context_frame(FrameDirection.UPSTREAM) - - # Emit the on_context_updated callback once the function call result is added to the context - if properties and properties.on_context_updated is not None: - await properties.on_context_updated() - - # Push context frame - await self.push_context_frame() - - # Push timestamp frame with current time - timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) - await self.push_frame(timestamp_frame) - - except Exception as e: - logger.error(f"Error processing frame: {e}") + async def handle_image_frame_message(self, frame: UserImageMessageFrame): + self._context.add_image_frame_message( + format=frame.user_image_raw_frame.format, + size=frame.user_image_raw_frame.size, + image=frame.user_image_raw_frame.image, + text=frame.text, + ) diff --git a/tests/test_context_aggregators.py b/tests/test_context_aggregators.py index d4b8c35ce..0c9b6d5e4 100644 --- a/tests/test_context_aggregators.py +++ b/tests/test_context_aggregators.py @@ -418,7 +418,7 @@ class BaseTestUserContextAggregator: class BaseTestAssistantContextAggreagator: CONTEXT_CLASS = None # To be set in subclasses AGGREGATOR_CLASS = None # To be set in subclasses - EXPECTED_CONTEXT_FRAMES = [OpenAILLMContextFrame] + EXPECTED_CONTEXT_FRAMES = None # To be set in subclasses def check_message_content(self, context: OpenAILLMContext, index: int, content: str): assert context.messages[index]["content"] == content @@ -577,6 +577,7 @@ class TestLLMAssistantContextAggregator( ): CONTEXT_CLASS = OpenAILLMContext AGGREGATOR_CLASS = LLMAssistantContextAggregator + EXPECTED_CONTEXT_FRAMES = [OpenAILLMContextFrame, OpenAILLMContextAssistantTimestampFrame] # From c15286b14836e35fbbf94eb5e2f39ce38761f79a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Mar 2025 14:57:49 -0700 Subject: [PATCH 075/132] examples: deprecate start_callback from LLMService.register_function() --- examples/foundational/14-function-calling.py | 12 ++---- .../14c-function-calling-together.py | 11 ++--- .../14e-function-calling-gemini.py | 9 +---- .../foundational/14f-function-calling-groq.py | 11 ++--- .../foundational/14g-function-calling-grok.py | 11 +---- .../14h-function-calling-azure.py | 11 ++--- .../14i-function-calling-fireworks.py | 13 ++---- .../foundational/14j-function-calling-nim.py | 11 ++--- .../14k-function-calling-cerebras.py | 11 ++--- .../14l-function-calling-deepseek.py | 11 ++--- .../14m-function-calling-openrouter.py | 11 ++--- ...o-function-calling-gemini-openai-format.py | 13 ++---- .../14p-function-calling-gemini-vertex-ai.py | 13 ++---- .../22b-natural-conversation-proposal.py | 11 ++--- .../22c-natural-conversation-mixed-llms.py | 9 +---- examples/foundational/24-stt-mute-filter.py | 6 +-- examples/patient-intake/bot.py | 40 ++++++++++--------- 17 files changed, 67 insertions(+), 147 deletions(-) diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index 1d4e37344..dddfed1d2 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -30,13 +30,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -62,9 +57,10 @@ async def main(): ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - # Register a function_name of None to get all functions + + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index dd3eb714f..94d587799 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -66,9 +61,9 @@ async def main(): api_key=os.getenv("TOGETHER_API_KEY"), model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", ) - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index bf022a65a..f0003fbab 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -33,13 +33,8 @@ logger.add(sys.stderr, level="DEBUG") video_participant_id = None -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) location = arguments["location"] await result_callback(f"The weather in {location} is currently 72 degrees and sunny.") @@ -72,7 +67,7 @@ async def main(): ) llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") - llm.register_function("get_weather", get_weather, start_fetch_weather) + llm.register_function("get_weather", get_weather) llm.register_function("get_image", get_image) weather_function = FunctionSchema( diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index c402ac91a..9818d185f 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -65,9 +60,9 @@ async def main(): ) llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile") - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index e6b822e39..e5bb2f9b3 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -16,7 +16,6 @@ from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -31,12 +30,6 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): await result_callback({"conditions": "nice", "temperature": "75"}) @@ -63,9 +56,9 @@ async def main(): ) llm = GrokLLMService(api_key=os.getenv("GROK_API_KEY")) - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index aefa1a474..6c8465832 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -67,9 +62,9 @@ async def main(): endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"), ) - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index 50291615b..887e734dd 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -64,11 +59,11 @@ async def main(): llm = FireworksLLMService( api_key=os.getenv("FIREWORKS_API_KEY"), - model="accounts/fireworks/models/firefunction-v2", + model="accounts/fireworks/models/llama-v3p1-405b-instruct", ) - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index ea8e25cf6..c628bedf0 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -66,9 +61,9 @@ async def main(): llm = NimLLMService( api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.3-70b-instruct" ) - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py index 9b3641307..541e6f250 100644 --- a/examples/foundational/14k-function-calling-cerebras.py +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -63,9 +58,9 @@ async def main(): ) llm = CerebrasLLMService(api_key=os.getenv("CEREBRAS_API_KEY"), model="llama-3.3-70b") - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14l-function-calling-deepseek.py b/examples/foundational/14l-function-calling-deepseek.py index 0360f0b84..812a6716e 100644 --- a/examples/foundational/14l-function-calling-deepseek.py +++ b/examples/foundational/14l-function-calling-deepseek.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -63,9 +58,9 @@ async def main(): ) llm = DeepSeekLLMService(api_key=os.getenv("DEEPSEEK_API_KEY"), model="deepseek-chat") - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14m-function-calling-openrouter.py b/examples/foundational/14m-function-calling-openrouter.py index e816f6322..a675c8ac4 100644 --- a/examples/foundational/14m-function-calling-openrouter.py +++ b/examples/foundational/14m-function-calling-openrouter.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -67,9 +62,9 @@ async def main(): llm = OpenRouterLLMService( api_key=os.getenv("OPENROUTER_API_KEY"), model="openai/gpt-4o-2024-11-20" ) - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14o-function-calling-gemini-openai-format.py b/examples/foundational/14o-function-calling-gemini-openai-format.py index a1bb53b6b..78100f0d3 100644 --- a/examples/foundational/14o-function-calling-gemini-openai-format.py +++ b/examples/foundational/14o-function-calling-gemini-openai-format.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -63,11 +58,9 @@ async def main(): ) llm = GoogleLLMOpenAIBetaService(api_key=os.getenv("GEMINI_API_KEY")) - # Register a function_name of None to get all functions + # You can aslo register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function( - "get_current_weather", fetch_weather_from_api, start_callback=start_fetch_weather - ) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14p-function-calling-gemini-vertex-ai.py b/examples/foundational/14p-function-calling-gemini-vertex-ai.py index d64bae89d..8888a2922 100644 --- a/examples/foundational/14p-function-calling-gemini-vertex-ai.py +++ b/examples/foundational/14p-function-calling-gemini-vertex-ai.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -68,11 +63,9 @@ async def main(): project_id="", ) ) - # Register a function_name of None to get all functions + # You can aslo register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function( - "get_current_weather", fetch_weather_from_api, start_callback=start_fetch_weather - ) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index c97289217..d7f7eb597 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -199,13 +199,8 @@ class OutputGate(FrameProcessor): break -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -239,9 +234,9 @@ async def main(): # This is the regular LLM. llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) tools = [ ChatCompletionToolParam( diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index d342a2765..7fa1bb5fa 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -403,13 +403,8 @@ class OutputGate(FrameProcessor): break -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -451,7 +446,7 @@ async def main(): ) # Register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) tools = [ ChatCompletionToolParam( diff --git a/examples/foundational/24-stt-mute-filter.py b/examples/foundational/24-stt-mute-filter.py index 7ae6c73fa..d2b7174d1 100644 --- a/examples/foundational/24-stt-mute-filter.py +++ b/examples/foundational/24-stt-mute-filter.py @@ -30,10 +30,6 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): # Add a delay to test interruption during function calls logger.info("Weather API call starting...") @@ -72,7 +68,7 @@ async def main(): tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en") llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) tools = [ ChatCompletionToolParam( diff --git a/examples/patient-intake/bot.py b/examples/patient-intake/bot.py index b6f7fb941..de6ad2051 100644 --- a/examples/patient-intake/bot.py +++ b/examples/patient-intake/bot.py @@ -142,7 +142,9 @@ class IntakeProcessor: ] ) - async def start_prescriptions(self, function_name, llm, context): + async def list_prescriptions( + self, function_name, tool_call_id, args, llm, context, result_callback + ): print(f"!!! doing start prescriptions") # Move on to allergies context.set_tools( @@ -182,9 +184,12 @@ class IntakeProcessor: print(f"!!! about to await llm process frame in start prescrpitions") await llm.queue_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM) print(f"!!! past await process frame in start prescriptions") + await self.save_data(args, result_callback) - async def start_allergies(self, function_name, llm, context): - print("!!! doing start allergies") + async def list_allergies( + self, function_name, tool_call_id, args, llm, context, result_callback + ): + print("!!! doing list allergies") # Move on to conditions context.set_tools( [ @@ -221,8 +226,11 @@ class IntakeProcessor: } ) await llm.queue_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM) + await self.save_data(args, result_callback) - async def start_conditions(self, function_name, llm, context): + async def list_conditions( + self, function_name, tool_call_id, args, llm, context, result_callback + ): print("!!! doing start conditions") # Move on to visit reasons context.set_tools( @@ -260,8 +268,11 @@ class IntakeProcessor: } ) await llm.queue_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM) + await self.save_data(args, result_callback) - async def start_visit_reasons(self, function_name, llm, context): + async def list_visit_reasons( + self, function_name, tool_call_id, args, llm, context, result_callback + ): print("!!! doing start visit reasons") # move to finish call context.set_tools([]) @@ -269,8 +280,9 @@ class IntakeProcessor: {"role": "system", "content": "Now, thank the user and end the conversation."} ) await llm.queue_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM) + await self.save_data(args, result_callback) - async def save_data(self, function_name, tool_call_id, args, llm, context, result_callback): + async def save_data(self, args, result_callback): logger.info(f"!!! Saving data: {args}") # Since this is supposed to be "async", returning None from the callback # will prevent adding anything to context or re-prompting @@ -319,18 +331,10 @@ async def main(): intake = IntakeProcessor(context) llm.register_function("verify_birthday", intake.verify_birthday) - llm.register_function( - "list_prescriptions", intake.save_data, start_callback=intake.start_prescriptions - ) - llm.register_function( - "list_allergies", intake.save_data, start_callback=intake.start_allergies - ) - llm.register_function( - "list_conditions", intake.save_data, start_callback=intake.start_conditions - ) - llm.register_function( - "list_visit_reasons", intake.save_data, start_callback=intake.start_visit_reasons - ) + llm.register_function("list_prescriptions", intake.list_prescriptions) + llm.register_function("list_allergies", intake.list_allergies) + llm.register_function("list_conditions", intake.list_conditions) + llm.register_function("list_visit_reasons", intake.list_visit_reasons) fl = FrameLogger("LLM Output") From d1550d5a854ca1617068a67d368e28bfedb00b5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Mar 2025 15:52:37 -0700 Subject: [PATCH 076/132] tests: remove TestFrameProcessor, reimplement with run_test() --- src/pipecat/tests/utils.py | 42 +++--- src/pipecat/utils/test_frame_processor.py | 48 ------- tests/integration/integration_azure_llm.py | 33 ----- tests/integration/integration_ollama_llm.py | 28 ---- tests/integration/integration_openai_llm.py | 128 ------------------ ...st_integration_unified_function_calling.py | 27 ++-- 6 files changed, 37 insertions(+), 269 deletions(-) delete mode 100644 src/pipecat/utils/test_frame_processor.py delete mode 100644 tests/integration/integration_azure_llm.py delete mode 100644 tests/integration/integration_ollama_llm.py delete mode 100644 tests/integration/integration_openai_llm.py diff --git a/src/pipecat/tests/utils.py b/src/pipecat/tests/utils.py index d68c87d88..e2368ba09 100644 --- a/src/pipecat/tests/utils.py +++ b/src/pipecat/tests/utils.py @@ -6,7 +6,7 @@ import asyncio from dataclasses import dataclass -from typing import Any, Awaitable, Callable, Dict, Sequence, Tuple +from typing import Any, Awaitable, Callable, Dict, Optional, Sequence, Tuple from pipecat.frames.frames import ( EndFrame, @@ -80,8 +80,8 @@ async def run_test( processor: FrameProcessor, *, frames_to_send: Sequence[Frame], - expected_down_frames: Sequence[type], - expected_up_frames: Sequence[type] = [], + expected_down_frames: Optional[Sequence[type]] = None, + expected_up_frames: Optional[Sequence[type]] = None, ignore_start: bool = True, start_metadata: Dict[str, Any] = {}, send_end_frame: bool = True, @@ -126,33 +126,35 @@ async def run_test( # Down frames # received_down_frames: Sequence[Frame] = [] - while not received_down.empty(): - frame = await received_down.get() - if not isinstance(frame, EndFrame) or not send_end_frame: - received_down_frames.append(frame) + if expected_down_frames is not None: + while not received_down.empty(): + frame = await received_down.get() + if not isinstance(frame, EndFrame) or not send_end_frame: + received_down_frames.append(frame) - print("received DOWN frames =", received_down_frames) - print("expected DOWN frames =", expected_down_frames) + print("received DOWN frames =", received_down_frames) + print("expected DOWN frames =", expected_down_frames) - assert len(received_down_frames) == len(expected_down_frames) + assert len(received_down_frames) == len(expected_down_frames) - for real, expected in zip(received_down_frames, expected_down_frames): - assert isinstance(real, expected) + for real, expected in zip(received_down_frames, expected_down_frames): + assert isinstance(real, expected) # # Up frames # received_up_frames: Sequence[Frame] = [] - while not received_up.empty(): - frame = await received_up.get() - received_up_frames.append(frame) + if expected_up_frames is not None: + while not received_up.empty(): + frame = await received_up.get() + received_up_frames.append(frame) - print("received UP frames =", received_up_frames) - print("expected UP frames =", expected_up_frames) + print("received UP frames =", received_up_frames) + print("expected UP frames =", expected_up_frames) - assert len(received_up_frames) == len(expected_up_frames) + assert len(received_up_frames) == len(expected_up_frames) - for real, expected in zip(received_up_frames, expected_up_frames): - assert isinstance(real, expected) + for real, expected in zip(received_up_frames, expected_up_frames): + assert isinstance(real, expected) return (received_down_frames, received_up_frames) diff --git a/src/pipecat/utils/test_frame_processor.py b/src/pipecat/utils/test_frame_processor.py deleted file mode 100644 index b35864497..000000000 --- a/src/pipecat/utils/test_frame_processor.py +++ /dev/null @@ -1,48 +0,0 @@ -from typing import List - -from pipecat.processors.frame_processor import FrameProcessor - - -class TestException(Exception): - pass - - -class TestFrameProcessor(FrameProcessor): - __test__ = False # Prevents pytest from collecting this class as a test - - def __init__(self, test_frames): - self.test_frames = test_frames - self._list_counter = 0 - super().__init__() - - async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - - if not self.test_frames[ - 0 - ]: # then we've run out of required frames but the generator is still going? - raise TestException(f"Oops, got an extra frame, {frame}") - if isinstance(self.test_frames[0], List): - # We need to consume frames until we see the next frame type after this - next_frame = self.test_frames[1] - if isinstance(frame, next_frame): - # we're done iterating the list I guess - print(f"TestFrameProcessor got expected list exit frame: {frame}") - # pop twice to get rid of the list, as well as the next frame - self.test_frames.pop(0) - self.test_frames.pop(0) - self.list_counter = 0 - else: - fl = self.test_frames[0] - fl_el = fl[self._list_counter % len(fl)] - if isinstance(frame, fl_el): - print(f"TestFrameProcessor got expected list frame: {frame}") - self._list_counter += 1 - else: - raise TestException(f"Inside a list, expected {fl_el} but got {frame}") - - else: - if not isinstance(frame, self.test_frames[0]): - raise TestException(f"Expected {self.test_frames[0]}, but got {frame}") - print(f"TestFrameProcessor got expected frame: {frame}") - self.test_frames.pop(0) diff --git a/tests/integration/integration_azure_llm.py b/tests/integration/integration_azure_llm.py deleted file mode 100644 index 8e49e9d04..000000000 --- a/tests/integration/integration_azure_llm.py +++ /dev/null @@ -1,33 +0,0 @@ -import asyncio -import os -import unittest - -from openai.types.chat import ( - ChatCompletionSystemMessageParam, -) - -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) -from pipecat.services.azure import AzureLLMService - -if __name__ == "__main__": - - @unittest.skip("Skip azure integration test") - async def test_chat(): - llm = AzureLLMService( - api_key=os.getenv("AZURE_CHATGPT_API_KEY"), - endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), - model=os.getenv("AZURE_CHATGPT_MODEL"), - ) - context = OpenAILLMContext() - message: ChatCompletionSystemMessageParam = ChatCompletionSystemMessageParam( - content="Please tell the world hello.", name="system", role="system" - ) - context.add_message(message) - frame = OpenAILLMContextFrame(context) - async for s in llm.process_frame(frame): - print(s) - - asyncio.run(test_chat()) diff --git a/tests/integration/integration_ollama_llm.py b/tests/integration/integration_ollama_llm.py deleted file mode 100644 index 085500cb8..000000000 --- a/tests/integration/integration_ollama_llm.py +++ /dev/null @@ -1,28 +0,0 @@ -import asyncio -import unittest - -from openai.types.chat import ( - ChatCompletionSystemMessageParam, -) - -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) -from pipecat.services.ollama import OLLamaLLMService - -if __name__ == "__main__": - - @unittest.skip("Skip azure integration test") - async def test_chat(): - llm = OLLamaLLMService() - context = OpenAILLMContext() - message: ChatCompletionSystemMessageParam = ChatCompletionSystemMessageParam( - content="Please tell the world hello.", name="system", role="system" - ) - context.add_message(message) - frame = OpenAILLMContextFrame(context) - async for s in llm.process_frame(frame): - print(s) - - asyncio.run(test_chat()) diff --git a/tests/integration/integration_openai_llm.py b/tests/integration/integration_openai_llm.py deleted file mode 100644 index 2c84c8882..000000000 --- a/tests/integration/integration_openai_llm.py +++ /dev/null @@ -1,128 +0,0 @@ -import asyncio -import json -import os -from typing import List - -from openai.types.chat import ( - ChatCompletionSystemMessageParam, - ChatCompletionToolParam, - ChatCompletionUserMessageParam, -) - -from pipecat.frames.frames import LLMFullResponseEndFrame, LLMFullResponseStartFrame, TextFrame -from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame, OpenAILLMService -from pipecat.utils.test_frame_processor import TestFrameProcessor - -tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, - }, - ) -] - -if __name__ == "__main__": - - async def test_simple_functions(): - async def get_weather_from_api(llm, args): - return json.dumps({"conditions": "nice", "temperature": "75"}) - - api_key = os.getenv("OPENAI_API_KEY") - - llm = OpenAILLMService( - api_key=api_key or "", - model="gpt-4-1106-preview", - ) - - llm.register_function("get_current_weather", get_weather_from_api) - t = TestFrameProcessor([LLMFullResponseStartFrame, TextFrame, LLMFullResponseEndFrame]) - llm.link(t) - - context = OpenAILLMContext(tools=tools) - system_message: ChatCompletionSystemMessageParam = ChatCompletionSystemMessageParam( - content="Ask the user to ask for a weather report", name="system", role="system" - ) - user_message: ChatCompletionUserMessageParam = ChatCompletionUserMessageParam( - content="Could you tell me the weather for Boulder, Colorado", - name="user", - role="user", - ) - context.add_message(system_message) - context.add_message(user_message) - frame = OpenAILLMContextFrame(context) - await llm.process_frame(frame, FrameDirection.DOWNSTREAM) - - async def test_advanced_functions(): - async def get_weather_from_api(llm, args): - return [ - { - "role": "system", - "content": "The user has asked for live weather. Respond by telling them we don't currently support live weather for that area, but it's coming soon.", - } - ] - - api_key = os.getenv("OPENAI_API_KEY") - - llm = OpenAILLMService( - api_key=api_key or "", - model="gpt-4-1106-preview", - ) - - llm.register_function("get_current_weather", get_weather_from_api) - t = TestFrameProcessor([LLMFullResponseStartFrame, TextFrame, LLMFullResponseEndFrame]) - llm.link(t) - - context = OpenAILLMContext(tools=tools) - system_message: ChatCompletionSystemMessageParam = ChatCompletionSystemMessageParam( - content="Ask the user to ask for a weather report", name="system", role="system" - ) - user_message: ChatCompletionUserMessageParam = ChatCompletionUserMessageParam( - content="Could you tell me the weather for Boulder, Colorado", - name="user", - role="user", - ) - context.add_message(system_message) - context.add_message(user_message) - frame = OpenAILLMContextFrame(context) - await llm.process_frame(frame, FrameDirection.DOWNSTREAM) - - async def test_chat(): - api_key = os.getenv("OPENAI_API_KEY") - t = TestFrameProcessor([LLMFullResponseStartFrame, TextFrame, LLMFullResponseEndFrame]) - llm = OpenAILLMService( - api_key=api_key or "", - model="gpt-4o", - ) - llm.link(t) - context = OpenAILLMContext() - message: ChatCompletionSystemMessageParam = ChatCompletionSystemMessageParam( - content="Please tell the world hello.", name="system", role="system" - ) - context.add_message(message) - frame = OpenAILLMContextFrame(context) - await llm.process_frame(frame, FrameDirection.DOWNSTREAM) - - async def run_tests(): - await test_simple_functions() - await test_advanced_functions() - await test_chat() - - asyncio.run(run_tests()) diff --git a/tests/integration/test_integration_unified_function_calling.py b/tests/integration/test_integration_unified_function_calling.py index 88407d703..0696c531a 100644 --- a/tests/integration/test_integration_unified_function_calling.py +++ b/tests/integration/test_integration_unified_function_calling.py @@ -1,3 +1,9 @@ +# +# Copyright (c) 2024-2025 Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + import os from unittest.mock import AsyncMock @@ -6,17 +12,12 @@ from dotenv import load_dotenv from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.frames.frames import ( - LLMFullResponseEndFrame, - LLMFullResponseStartFrame, - LLMTextFrame, -) -from pipecat.processors.frame_processor import FrameDirection +from pipecat.pipeline.pipeline import Pipeline from pipecat.services.ai_services import LLMService from pipecat.services.anthropic import AnthropicLLMService from pipecat.services.google import GoogleLLMService from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame, OpenAILLMService -from pipecat.utils.test_frame_processor import TestFrameProcessor +from pipecat.tests.utils import run_test load_dotenv(override=True) @@ -47,8 +48,6 @@ async def _test_llm_function_calling(llm: LLMService): mock_fetch_weather = AsyncMock() llm.register_function(None, mock_fetch_weather) - t = TestFrameProcessor([LLMFullResponseStartFrame, LLMTextFrame, LLMFullResponseEndFrame]) - llm.link(t) messages = [ { @@ -61,10 +60,14 @@ async def _test_llm_function_calling(llm: LLMService): # This is done by default inside the create_context_aggregator context.set_llm_adapter(llm.get_llm_adapter()) - frame = OpenAILLMContextFrame(context) + pipeline = Pipeline([llm]) - # This will fail if an exception is raised - await llm.process_frame(frame, FrameDirection.DOWNSTREAM) + frames_to_send = [OpenAILLMContextFrame(context)] + await run_test( + pipeline, + frames_to_send=frames_to_send, + expected_down_frames=None, + ) # Assert that the mock function was called mock_fetch_weather.assert_called_once() From d455fd070eebedcbab3fbf932b09d724dd39df55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Mar 2025 16:43:43 -0700 Subject: [PATCH 077/132] update CHANGELOG --- CHANGELOG.md | 3 +++ src/pipecat/services/google/google.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4762186cd..247c3280d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 being aggregated. A text aggregator can be passed via `text_aggregator` to the TTS service. +- Added new `sample_rate` constructor parameter to `TavusVideoService` to allow + changing the output sample rate. + - Added new `UltravoxSTTService`. (see https://github.com/fixie-ai/ultravox) diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index c07707899..3db2044db 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -1316,9 +1316,12 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService): class GoogleVertexLLMService(OpenAILLMService): - """Implements inference with Google's AI models via Vertex AI while maintaining OpenAI API compatibility. + """Implements inference with Google's AI models via Vertex AI while + maintaining OpenAI API compatibility. + Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library + """ class InputParams(OpenAILLMService.InputParams): From 3e9678db8430101a7964d9f3e246a6db48355c9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 13:12:42 -0700 Subject: [PATCH 078/132] user image requests can now be related to function calls --- .../14b-function-calling-anthropic-video.py | 7 ++- .../14d-function-calling-video.py | 9 ++- .../14e-function-calling-gemini.py | 7 ++- .../20d-persistent-context-gemini.py | 7 ++- examples/moondream-chatbot/bot.py | 1 - src/pipecat/frames/frames.py | 25 ++++---- .../processors/aggregators/llm_response.py | 12 ++-- .../aggregators/openai_llm_context.py | 1 - src/pipecat/services/ai_services.py | 17 +++++- src/pipecat/services/anthropic.py | 57 ++++--------------- .../services/gemini_multimodal_live/gemini.py | 4 +- src/pipecat/services/google/google.py | 15 +++-- src/pipecat/services/openai.py | 55 ++++-------------- src/pipecat/transports/services/daily.py | 21 ++++--- 14 files changed, 101 insertions(+), 137 deletions(-) diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index 302722be2..cd9777b72 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -39,7 +39,12 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): question = arguments["question"] - await llm.request_image_frame(user_id=video_participant_id, text_content=question) + await llm.request_image_frame( + user_id=video_participant_id, + function_name=function_name, + tool_call_id=tool_call_id, + text_content=question, + ) async def main(): diff --git a/examples/foundational/14d-function-calling-video.py b/examples/foundational/14d-function-calling-video.py index 6e290d55f..a7e997f9b 100644 --- a/examples/foundational/14d-function-calling-video.py +++ b/examples/foundational/14d-function-calling-video.py @@ -39,7 +39,12 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): logger.debug(f"!!! IN get_image {video_participant_id}, {arguments}") question = arguments["question"] - await llm.request_image_frame(user_id=video_participant_id, text_content=question) + await llm.request_image_frame( + user_id=video_participant_id, + function_name=function_name, + tool_call_id=tool_call_id, + text_content=question, + ) async def main(): @@ -141,7 +146,7 @@ indicate you should use the get_image tool are: await transport.capture_participant_transcription(participant["id"]) await transport.capture_participant_video(video_participant_id, framerate=0) # Kick off the conversation. - await tts.say("Hi! Ask me about the weather in San Francisco.") + await task.queue_frames([context_aggregator.user().get_context_frame()]) runner = PipelineRunner() diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index f0003fbab..8448643b4 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -42,7 +42,12 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): logger.debug(f"!!! IN get_image {video_participant_id}, {arguments}") question = arguments["question"] - await llm.request_image_frame(user_id=video_participant_id, text_content=question) + await llm.request_image_frame( + user_id=video_participant_id, + function_name=function_name, + tool_call_id=tool_call_id, + text_content=question, + ) async def main(): diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index 740bdc68f..2381ea131 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -54,7 +54,12 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): question = arguments["question"] - await llm.request_image_frame(user_id=video_participant_id, text_content=question) + await llm.request_image_frame( + user_id=video_participant_id, + function_name=function_name, + tool_call_id=tool_call_id, + text_content=question, + ) async def get_saved_conversation_filenames( diff --git a/examples/moondream-chatbot/bot.py b/examples/moondream-chatbot/bot.py index 5dd88f4d1..6639c08c1 100644 --- a/examples/moondream-chatbot/bot.py +++ b/examples/moondream-chatbot/bot.py @@ -23,7 +23,6 @@ from pipecat.frames.frames import ( OutputImageRawFrame, SpriteFrame, TextFrame, - UserImageRawFrame, UserImageRequestFrame, ) from pipecat.pipeline.parallel_pipeline import ParallelPipeline diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index e589e9480..c2a79461f 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -662,13 +662,19 @@ class TransportMessageUrgentFrame(SystemFrame): @dataclass class UserImageRequestFrame(SystemFrame): - """A frame user to request an image from the given user.""" + """A frame to request an image from the given user. The frame might be + generated by a function call in which case the corresponding fields will be + properly set. + + """ user_id: str context: Optional[Any] = None + function_name: Optional[str] = None + tool_call_id: Optional[str] = None def __str__(self): - return f"{self.name}, user: {self.user_id}" + return f"{self.name}(user: {self.user_id}, function: {self.function_name}, request: {self.tool_call_id})" @dataclass @@ -698,10 +704,11 @@ class UserImageRawFrame(InputImageRawFrame): """An image associated to a user.""" user_id: str + request: Optional[UserImageRequestFrame] = None def __str__(self): pts = format_pts(self.pts) - return f"{self.name}(pts: {pts}, user: {self.user_id}, size: {self.size}, format: {self.format})" + return f"{self.name}(pts: {pts}, user: {self.user_id}, size: {self.size}, format: {self.format}, request: {self.request})" @dataclass @@ -715,18 +722,6 @@ class VisionImageRawFrame(InputImageRawFrame): return f"{self.name}(pts: {pts}, text: [{self.text}], size: {self.size}, format: {self.format})" -@dataclass -class UserImageMessageFrame(SystemFrame): - """An image associated to a user.""" - - user_image_raw_frame: UserImageRawFrame - text: Optional[str] = None - - def __str__(self): - pts = format_pts(self.pts) - return f"{self.name}(pts: {pts}, image: {self.user_image_raw_frame}, text: {self.text})" - - # # Control frames # diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 494d1de38..ff1b6e1be 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -34,7 +34,7 @@ from pipecat.frames.frames import ( StartInterruptionFrame, TextFrame, TranscriptionFrame, - UserImageMessageFrame, + UserImageRawFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -401,7 +401,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): pass - async def handle_image_frame_message(self, frame: UserImageMessageFrame): + async def handle_user_image_frame(self, frame: UserImageRawFrame): pass async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -428,8 +428,8 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): await self._handle_function_call_result(frame) elif isinstance(frame, FunctionCallCancelFrame): await self._handle_function_call_cancel(frame) - elif isinstance(frame, UserImageMessageFrame): - await self._handle_image_frame_message(frame) + elif isinstance(frame, UserImageRawFrame) and frame.request and frame.request.tool_call_id: + await self._handle_user_image_frame(frame) elif isinstance(frame, BotStoppedSpeakingFrame): await self.push_aggregation() else: @@ -510,8 +510,8 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): await self.handle_function_call_cancel(frame) del self._function_calls_in_progress[frame.tool_call_id] - async def _handle_image_frame_message(self, frame: UserImageMessageFrame): - await self.handle_image_frame_message(frame) + async def _handle_user_image_frame(self, frame: UserImageRawFrame): + await self.handle_user_image_frame(frame) await self.push_aggregation() await self.push_context_frame(FrameDirection.UPSTREAM) diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index 93c2875be..2e5ade0a0 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -46,7 +46,6 @@ class OpenAILLMContext: self._messages: List[ChatCompletionMessageParam] = messages if messages else [] self._tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = tool_choice self._tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = tools - self._user_image_request_context = {} self._llm_adapter: Optional[BaseLLMAdapter] = None def get_llm_adapter(self) -> Optional[BaseLLMAdapter]: diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 2f30f505f..678132cb6 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -253,9 +253,22 @@ class LLMService(AIService): elif None in self._start_callbacks.keys(): return await self._start_callbacks[None](function_name, self, context) - async def request_image_frame(self, user_id: str, *, text_content: Optional[str] = None): + async def request_image_frame( + self, + user_id: str, + *, + function_name: Optional[str] = None, + tool_call_id: Optional[str] = None, + text_content: Optional[str] = None, + ): await self.push_frame( - UserImageRequestFrame(user_id=user_id, context=text_content), FrameDirection.UPSTREAM + UserImageRequestFrame( + user_id=user_id, + function_name=function_name, + tool_call_id=tool_call_id, + context=text_content, + ), + FrameDirection.UPSTREAM, ) async def _run_function_call( diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index 8e06f4558..a80e2154c 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -30,9 +30,7 @@ from pipecat.frames.frames import ( LLMMessagesFrame, LLMTextFrame, LLMUpdateSettingsFrame, - UserImageMessageFrame, UserImageRawFrame, - UserImageRequestFrame, VisionImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage @@ -674,42 +672,7 @@ class AnthropicLLMContext(OpenAILLMContext): class AnthropicUserContextAggregator(LLMUserContextAggregator): - def __init__(self, context: OpenAILLMContext | AnthropicLLMContext, **kwargs): - super().__init__(context=context, **kwargs) - - async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - # Our parent method has already called push_frame(). So we can't interrupt the - # flow here and we don't need to call push_frame() ourselves. Possibly something - # to talk through (tagging @aleix). At some point we might need to refactor these - # context aggregators. - try: - if isinstance(frame, UserImageRequestFrame): - # The LLM sends a UserImageRequestFrame upstream. Cache any context provided with - # that frame so we can use it when we assemble the image message in the assistant - # context aggregator. - if frame.context: - if isinstance(frame.context, str): - self._context._user_image_request_context[frame.user_id] = frame.context - else: - logger.error( - f"Unexpected UserImageRequestFrame context type: {type(frame.context)}" - ) - del self._context._user_image_request_context[frame.user_id] - else: - if frame.user_id in self._context._user_image_request_context: - del self._context._user_image_request_context[frame.user_id] - elif isinstance(frame, UserImageRawFrame): - # Push a new AnthropicImageMessageFrame with the text context we cached - # downstream to be handled by our assistant context aggregator. This is - # necessary so that we add the message to the context in the right order. - text = self._context._user_image_request_context.get(frame.user_id) or "" - if text: - del self._context._user_image_request_context[frame.user_id] - frame = UserImageMessageFrame(user_image_raw_frame=frame, text=text) - await self.push_frame(frame) - except Exception as e: - logger.error(f"Error processing frame: {e}") + pass # @@ -723,9 +686,6 @@ class AnthropicUserContextAggregator(LLMUserContextAggregator): class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): - def __init__(self, context: OpenAILLMContext | AnthropicLLMContext, **kwargs): - super().__init__(context=context, **kwargs) - async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): assistant_message = {"role": "assistant", "content": []} assistant_message["content"].append( @@ -776,10 +736,13 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): ): content["content"] = result - async def handle_image_frame_message(self, frame: UserImageMessageFrame): - self._context.add_image_frame_message( - format=frame.user_image_raw_frame.format, - size=frame.user_image_raw_frame.size, - image=frame.user_image_raw_frame.image, - text=frame.text, + async def handle_user_image_frame(self, frame: UserImageRawFrame): + await self._update_function_call_result( + frame.request.function_name, frame.request.tool_call_id, "COMPLETED" + ) + self._context.add_image_frame_message( + format=frame.format, + size=frame.size, + image=frame.image, + text=frame.request.context, ) diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index da1697da6..801af46b4 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -39,7 +39,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, TTSTextFrame, - UserImageMessageFrame, + UserImageRawFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -119,7 +119,7 @@ class GeminiMultimodalLiveUserContextAggregator(OpenAIUserContextAggregator): class GeminiMultimodalLiveAssistantContextAggregator(OpenAIAssistantContextAggregator): - async def handle_image_frame_message(self, frame: UserImageMessageFrame): + async def handle_user_image_frame(self, frame: UserImageRawFrame): # We don't want to store any images in the context. Revisit this later # when the API evolves. pass diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 3db2044db..128eb43b6 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -49,7 +49,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, URLImageRawFrame, - UserImageMessageFrame, + UserImageRawFrame, VisionImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage @@ -624,12 +624,15 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): if part.function_response and part.function_response.id == tool_call_id: part.function_response.response = result - async def handle_image_frame_message(self, frame: UserImageMessageFrame): + async def handle_user_image_frame(self, frame: UserImageRawFrame): + await self._update_function_call_result( + frame.request.function_name, frame.request.tool_call_id, "COMPLETED" + ) self._context.add_image_frame_message( - format=frame.user_image_raw_frame.format, - size=frame.user_image_raw_frame.size, - image=frame.user_image_raw_frame.image, - text=frame.text, + format=frame.format, + size=frame.size, + image=frame.image, + text=frame.request.context, ) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 700f1c0b5..8318739de 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -40,9 +40,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, URLImageRawFrame, - UserImageMessageFrame, UserImageRawFrame, - UserImageRequestFrame, VisionImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage @@ -557,46 +555,10 @@ class OpenAITTSService(TTSService): class OpenAIUserContextAggregator(LLMUserContextAggregator): - def __init__(self, context: OpenAILLMContext, **kwargs): - super().__init__(context=context, **kwargs) - - async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - # Our parent method has already called push_frame(). So we can't interrupt the - # flow here and we don't need to call push_frame() ourselves. - try: - if isinstance(frame, UserImageRequestFrame): - # The LLM sends a UserImageRequestFrame upstream. Cache any context provided with - # that frame so we can use it when we assemble the image message in the assistant - # context aggregator. - if frame.context: - if isinstance(frame.context, str): - self._context._user_image_request_context[frame.user_id] = frame.context - else: - logger.error( - f"Unexpected UserImageRequestFrame context type: {type(frame.context)}" - ) - del self._context._user_image_request_context[frame.user_id] - else: - if frame.user_id in self._context._user_image_request_context: - del self._context._user_image_request_context[frame.user_id] - elif isinstance(frame, UserImageRawFrame): - # Push a new OpenAIImageMessageFrame with the text context we cached - # downstream to be handled by our assistant context aggregator. This is - # necessary so that we add the message to the context in the right order. - text = self._context._user_image_request_context.get(frame.user_id) or "" - if text: - del self._context._user_image_request_context[frame.user_id] - frame = UserImageMessageFrame(user_image_raw_frame=frame, text=text) - await self.push_frame(frame) - except Exception as e: - logger.error(f"Error processing frame: {e}") + pass class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): - def __init__(self, context: OpenAILLMContext, **kwargs): - super().__init__(context=context, **kwargs) - async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): self._context.add_message( { @@ -645,10 +607,13 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): ): message["content"] = result - async def handle_image_frame_message(self, frame: UserImageMessageFrame): - self._context.add_image_frame_message( - format=frame.user_image_raw_frame.format, - size=frame.user_image_raw_frame.size, - image=frame.user_image_raw_frame.image, - text=frame.text, + async def handle_user_image_frame(self, frame: UserImageRawFrame): + await self._update_function_call_result( + frame.request.function_name, frame.request.tool_call_id, "COMPLETED" + ) + self._context.add_image_frame_message( + format=frame.format, + size=frame.size, + image=frame.image, + text=frame.request.context, ) diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index f4b83dfa7..af7d2308c 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -898,7 +898,7 @@ class DailyInputTransport(BaseInputTransport): await super().process_frame(frame, direction) if isinstance(frame, UserImageRequestFrame): - await self.request_participant_image(frame.user_id) + await self.request_participant_image(frame) # # Frames @@ -935,16 +935,16 @@ class DailyInputTransport(BaseInputTransport): self._video_renderers[participant_id] = { "framerate": framerate, "timestamp": 0, - "render_next_frame": False, + "render_next_frame": [], } await self._client.capture_participant_video( participant_id, self._on_participant_video_frame, framerate, video_source, color_format ) - async def request_participant_image(self, participant_id: str): - if participant_id in self._video_renderers: - self._video_renderers[participant_id]["render_next_frame"] = True + async def request_participant_image(self, frame: UserImageRequestFrame): + if frame.user_id in self._video_renderers: + self._video_renderers[frame.user_id]["render_next_frame"].append(frame) async def _on_participant_video_frame(self, participant_id: str, buffer, size, format): render_frame = False @@ -953,17 +953,24 @@ class DailyInputTransport(BaseInputTransport): prev_time = self._video_renderers[participant_id]["timestamp"] framerate = self._video_renderers[participant_id]["framerate"] + # Some times we render frames because of a request. + request_frame = None + if framerate > 0: next_time = prev_time + 1 / framerate render_frame = (next_time - curr_time) < 0.1 elif self._video_renderers[participant_id]["render_next_frame"]: - self._video_renderers[participant_id]["render_next_frame"] = False + request_frame = self._video_renderers[participant_id]["render_next_frame"].pop(0) render_frame = True if render_frame: frame = UserImageRawFrame( - user_id=participant_id, image=buffer, size=size, format=format + user_id=participant_id, + request=request_frame, + image=buffer, + size=size, + format=format, ) await self.push_frame(frame) self._video_renderers[participant_id]["timestamp"] = curr_time From 1f6ed01ba6b910a65cf8237bd69aff18800a43d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 14:06:22 -0700 Subject: [PATCH 079/132] LLMAssistantContextAggregator: remove tool call id with image requests --- src/pipecat/processors/aggregators/llm_response.py | 12 ++++++++++++ src/pipecat/services/ai_services.py | 4 +++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index ff1b6e1be..9267af364 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -511,6 +511,18 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): del self._function_calls_in_progress[frame.tool_call_id] async def _handle_user_image_frame(self, frame: UserImageRawFrame): + logger.debug( + f"{self} UserImageRawFrame: [{frame.request.function_name}:{frame.request.tool_call_id}]" + ) + + if frame.request.tool_call_id not in self._function_calls_in_progress: + logger.warning( + f"UserImageRawFrame tool_call_id [{frame.request.tool_call_id}] is not running" + ) + return + + del self._function_calls_in_progress[frame.request.tool_call_id] + await self.handle_user_image_frame(frame) await self.push_aggregation() await self.push_context_frame(FrameDirection.UPSTREAM) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 678132cb6..9f9804e65 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -286,7 +286,9 @@ class LLMService(AIService): else: return - logger.info(f"Calling function {function_name} with arguments {arguments}") + logger.debug( + f"{self} Calling function [{function_name}:{tool_call_id}] with arguments {arguments}" + ) # NOTE(aleix): This needs to be removed after we remove the deprecation. await self.call_start_function(context, function_name) From b1d506c137cec966f9aca0fbb835e20ce5822183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 14:09:13 -0700 Subject: [PATCH 080/132] GoogleAssistantContextAggregator: properly update function response --- src/pipecat/services/google/google.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 128eb43b6..7aa9b7b7b 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -622,7 +622,7 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): if message.role == "user": for part in message.parts: if part.function_response and part.function_response.id == tool_call_id: - part.function_response.response = result + part.function_response.response = {"response": result} async def handle_user_image_frame(self, frame: UserImageRawFrame): await self._update_function_call_result( From e0c3f6ad832b8e583b188c34546dbec0edc6a021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 14:34:31 -0700 Subject: [PATCH 081/132] services: mark function calls as completed even the result is None --- src/pipecat/services/anthropic.py | 13 +++++++------ src/pipecat/services/google/google.py | 18 +++++++++++------- src/pipecat/services/openai.py | 13 +++++++------ 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index a80e2154c..6a95d04e2 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -711,12 +711,13 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): ) async def handle_function_call_result(self, frame: FunctionCallResultFrame): - if not frame.result: - return - - result = json.dumps(frame.result) - - await self._update_function_call_result(frame.function_name, frame.tool_call_id, result) + if frame.result: + result = json.dumps(frame.result) + await self._update_function_call_result(frame.function_name, frame.tool_call_id, result) + else: + await self._update_function_call_result( + frame.function_name, frame.tool_call_id, "COMPLETED" + ) async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): await self._update_function_call_result( diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 7aa9b7b7b..bfddce46d 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -600,15 +600,19 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): ) async def handle_function_call_result(self, frame: FunctionCallResultFrame): - if not frame.result: - return + if frame.result: + if not isinstance(frame.result, str): + return - if not isinstance(frame.result, str): - return + response = {"response": frame.result} - response = {"response": frame.result} - - await self._update_function_call_result(frame.function_name, frame.tool_call_id, response) + await self._update_function_call_result( + frame.function_name, frame.tool_call_id, response + ) + else: + await self._update_function_call_result( + frame.function_name, frame.tool_call_id, "COMPLETED" + ) async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): await self._update_function_call_result( diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 8318739de..b5f5d9d54 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -584,12 +584,13 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): ) async def handle_function_call_result(self, frame: FunctionCallResultFrame): - if not frame.result: - return - - result = json.dumps(frame.result) - - await self._update_function_call_result(frame.function_name, frame.tool_call_id, result) + if frame.result: + result = json.dumps(frame.result) + await self._update_function_call_result(frame.function_name, frame.tool_call_id, result) + else: + await self._update_function_call_result( + frame.function_name, frame.tool_call_id, "COMPLETED" + ) async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): await self._update_function_call_result( From f298febacf1e0639c4716710b0e404e4e85e1af3 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 20 Mar 2025 08:17:55 -0400 Subject: [PATCH 082/132] Add FalSTTService --- CHANGELOG.md | 4 + README.md | 2 +- .../foundational/07w-interruptible-fal.py | 110 +++++++++ src/pipecat/services/fal.py | 227 +++++++++++++++++- src/pipecat/transcriptions/language.py | 61 ++++- 5 files changed, 400 insertions(+), 4 deletions(-) create mode 100644 examples/foundational/07w-interruptible-fal.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 247c3280d..6d55b97c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 handler will be triggered if the idle timeout is reached (whether the pipeline task is cancelled or not). +- Added `FalSTTService`, which provides STT for Fal's Wizper API. + - Added a `reconnect_on_error` parameter to websocket-based TTS services as well as a `on_connection_error` event handler. The `reconnect_on_error` indicates whether the TTS service should reconnect on error. The `on_connection_error` @@ -216,6 +218,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Other +- Add foundational example `07w-interruptible-fal.py`, showing `FalSTTService`. + - Added a new example `examples/foundational/36-user-email-gathering.py` to show how to gather user emails. The example uses's Cartesia's `` tags and Rime `spell()` function to spell out the emails for confirmation. diff --git a/README.md b/README.md index 9522ce175..f28005618 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ pip install "pipecat-ai[option,...]" | Category | Services | Install Command Example | | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Parakeet (NVIDIA)](https://docs.pipecat.ai/server/services/stt/parakeet), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Parakeet (NVIDIA)](https://docs.pipecat.ai/server/services/stt/parakeet), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"` | | Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [FastPitch (NVIDIA)](https://docs.pipecat.ai/server/services/tts/fastpitch), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | | Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[google]"` | diff --git a/examples/foundational/07w-interruptible-fal.py b/examples/foundational/07w-interruptible-fal.py new file mode 100644 index 000000000..526602166 --- /dev/null +++ b/examples/foundational/07w-interruptible-fal.py @@ -0,0 +1,110 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.fal import FalSTTService +from pipecat.services.gladia import GladiaSTTService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + stt = FalSTTService( + api_key=os.getenv("FAL_KEY"), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + # Register an event handler to exit the application when the user leaves. + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + await task.cancel() + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/pipecat/services/fal.py b/src/pipecat/services/fal.py index 7173861ab..cb39da75f 100644 --- a/src/pipecat/services/fal.py +++ b/src/pipecat/services/fal.py @@ -7,6 +7,7 @@ import asyncio import io import os +import wave from typing import AsyncGenerator, Dict, Optional, Union import aiohttp @@ -14,8 +15,10 @@ from loguru import logger from PIL import Image from pydantic import BaseModel -from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame -from pipecat.services.ai_services import ImageGenService +from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame, URLImageRawFrame +from pipecat.services.ai_services import ImageGenService, SegmentedSTTService +from pipecat.transcriptions.language import Language +from pipecat.utils.time import time_now_iso8601 try: import fal_client @@ -27,6 +30,120 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +def language_to_fal_language(language: Language) -> Optional[str]: + """Language support for Fal's Wizper API.""" + BASE_LANGUAGES = { + Language.AF: "af", + Language.AM: "am", + Language.AR: "ar", + Language.AS: "as", + Language.AZ: "az", + Language.BA: "ba", + Language.BE: "be", + Language.BG: "bg", + Language.BN: "bn", + Language.BO: "bo", + Language.BR: "br", + Language.BS: "bs", + Language.CA: "ca", + Language.CS: "cs", + Language.CY: "cy", + Language.DA: "da", + Language.DE: "de", + Language.EL: "el", + Language.EN: "en", + Language.ES: "es", + Language.ET: "et", + Language.EU: "eu", + Language.FA: "fa", + Language.FI: "fi", + Language.FO: "fo", + Language.FR: "fr", + Language.GL: "gl", + Language.GU: "gu", + Language.HA: "ha", + Language.HE: "he", + Language.HI: "hi", + Language.HR: "hr", + Language.HT: "ht", + Language.HU: "hu", + Language.HY: "hy", + Language.ID: "id", + Language.IS: "is", + Language.IT: "it", + Language.JA: "ja", + Language.JW: "jw", + Language.KA: "ka", + Language.KK: "kk", + Language.KM: "km", + Language.KN: "kn", + Language.KO: "ko", + Language.LA: "la", + Language.LB: "lb", + Language.LN: "ln", + Language.LO: "lo", + Language.LT: "lt", + Language.LV: "lv", + Language.MG: "mg", + Language.MI: "mi", + Language.MK: "mk", + Language.ML: "ml", + Language.MN: "mn", + Language.MR: "mr", + Language.MS: "ms", + Language.MT: "mt", + Language.MY: "my", + Language.NE: "ne", + Language.NL: "nl", + Language.NN: "nn", + Language.NO: "no", + Language.OC: "oc", + Language.PA: "pa", + Language.PL: "pl", + Language.PS: "ps", + Language.PT: "pt", + Language.RO: "ro", + Language.RU: "ru", + Language.SA: "sa", + Language.SD: "sd", + Language.SI: "si", + Language.SK: "sk", + Language.SL: "sl", + Language.SN: "sn", + Language.SO: "so", + Language.SQ: "sq", + Language.SR: "sr", + Language.SU: "su", + Language.SV: "sv", + Language.SW: "sw", + Language.TA: "ta", + Language.TE: "te", + Language.TG: "tg", + Language.TH: "th", + Language.TK: "tk", + Language.TL: "tl", + Language.TR: "tr", + Language.TT: "tt", + Language.UK: "uk", + Language.UR: "ur", + Language.UZ: "uz", + Language.VI: "vi", + Language.YI: "yi", + Language.YO: "yo", + Language.ZH: "zh", + } + + result = BASE_LANGUAGES.get(language) + + # If not found in base languages, try to find the base language from a variant + if not result: + lang_str = str(language.value) + base_code = lang_str.split("-")[0].lower() + result = base_code if base_code in BASE_LANGUAGES.values() else None + + return result + + class FalImageGenService(ImageGenService): class InputParams(BaseModel): seed: Optional[int] = None @@ -84,3 +201,109 @@ class FalImageGenService(ImageGenService): frame = URLImageRawFrame(url=image_url, image=image_bytes, size=size, format=format) yield frame + + +class FalSTTService(SegmentedSTTService): + """Speech-to-text service using Fal's Wizper API. + + This service uses Fal's Wizper API to perform speech-to-text transcription on audio + segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection. + + Args: + api_key: Fal API key. If not provided, will check FAL_KEY environment variable. + sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate. + params: Configuration parameters for the Wizper API. + **kwargs: Additional arguments passed to SegmentedSTTService. + """ + + class InputParams(BaseModel): + """Configuration parameters for Fal's Wizper API. + + Attributes: + language: Language of the audio input. Defaults to English. + task: Task to perform ('transcribe' or 'translate'). Defaults to 'transcribe'. + chunk_level: Level of chunking ('segment'). Defaults to 'segment'. + version: Version of Wizper model to use. Defaults to '3'. + """ + + language: Optional[Language] = Language.EN + task: str = "transcribe" + chunk_level: str = "segment" + version: str = "3" + + def __init__( + self, + *, + api_key: Optional[str] = None, + sample_rate: Optional[int] = None, + params: InputParams = InputParams(), + **kwargs, + ): + super().__init__( + sample_rate=sample_rate, + **kwargs, + ) + + if api_key: + os.environ["FAL_KEY"] = api_key + elif "FAL_KEY" not in os.environ: + raise ValueError( + "FAL_KEY must be provided either through api_key parameter or environment variable" + ) + + self._fal_client = fal_client.AsyncClient(key=api_key or os.getenv("FAL_KEY")) + self._settings = { + "task": params.task, + "language": self.language_to_service_language(params.language) + if params.language + else "en", + "chunk_level": params.chunk_level, + "version": params.version, + } + + def can_generate_metrics(self) -> bool: + return True + + def language_to_service_language(self, language: Language) -> Optional[str]: + return language_to_fal_language(language) + + async def set_language(self, language: Language): + logger.info(f"Switching STT language to: [{language}]") + self._settings["language"] = self.language_to_service_language(language) + + async def set_model(self, model: str): + await super().set_model(model) + logger.info(f"Switching STT model to: [{model}]") + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Transcribes an audio segment using Fal's Wizper API. + + Args: + audio: Raw audio bytes in WAV format (already converted by base class). + + Yields: + Frame: TranscriptionFrame containing the transcribed text. + + Note: + The audio is already in WAV format from the SegmentedSTTService. + Only non-empty transcriptions are yielded. + """ + try: + # Send to Fal directly (audio is already in WAV format from base class) + data_uri = fal_client.encode(audio, "audio/x-wav") + response = await self._fal_client.run( + "fal-ai/wizper", + arguments={"audio_url": data_uri, **self._settings}, + ) + + if response and "text" in response: + text = response["text"].strip() + if text: # Only yield non-empty text + logger.debug(f"Transcription: [{text}]") + yield TranscriptionFrame( + text, "", time_now_iso8601(), Language(self._settings["language"]) + ) + + except Exception as e: + logger.error(f"Fal Wizper error: {e}") + yield ErrorFrame(f"Fal Wizper error: {str(e)}") diff --git a/src/pipecat/transcriptions/language.py b/src/pipecat/transcriptions/language.py index b8b9fafe9..75f714a72 100644 --- a/src/pipecat/transcriptions/language.py +++ b/src/pipecat/transcriptions/language.py @@ -54,6 +54,9 @@ class Language(StrEnum): AZ = "az" AZ_AZ = "az-AZ" + # Bashkir + BA = "ba" + # Belarusian BE = "be" @@ -66,6 +69,12 @@ class Language(StrEnum): BN_BD = "bn-BD" BN_IN = "bn-IN" + # Tibetan + BO = "bo" + + # Breton + BR = "br" + # Bosnian BS = "bs" BS_BA = "bs-BA" @@ -159,6 +168,9 @@ class Language(StrEnum): FIL = "fil" FIL_PH = "fil-PH" + # Faroese + FO = "fo" + # French FR = "fr" FR_BE = "fr-BE" @@ -178,6 +190,9 @@ class Language(StrEnum): GU = "gu" GU_IN = "gu-IN" + # Hausa + HA = "ha" + # Hebrew HE = "he" HE_IL = "he-IL" @@ -190,6 +205,9 @@ class Language(StrEnum): HR = "hr" HR_HR = "hr-HR" + # Haitian Creole + HT = "ht" + # Hungarian HU = "hu" HU_HU = "hu-HU" @@ -224,6 +242,7 @@ class Language(StrEnum): # Javanese JV = "jv" JV_ID = "jv-ID" + JW = "jw" # Fal requires for Javanese # Georgian KA = "ka" @@ -245,6 +264,15 @@ class Language(StrEnum): KO = "ko" KO_KR = "ko-KR" + # Latin + LA = "la" + + # Luxembourgish + LB = "lb" + + # Lingala + LN = "ln" + # Lao LO = "lo" LO_LA = "lo-LA" @@ -257,6 +285,9 @@ class Language(StrEnum): LV = "lv" LV_LV = "lv-LV" + # Malagasy + MG = "mg" + # Macedonian MK = "mk" MK_MK = "mk-MK" @@ -289,9 +320,10 @@ class Language(StrEnum): MY_MM = "my-MM" # Norwegian - NB = "nb" + NB = "nb" # Norwegian Bokmål NB_NO = "nb-NO" NO = "no" + NN = "nn" # Norwegian Nynorsk # Nepali NE = "ne" @@ -302,6 +334,9 @@ class Language(StrEnum): NL_BE = "nl-BE" NL_NL = "nl-NL" + # Occitan + OC = "oc" + # Odia OR = "or" OR_IN = "or-IN" @@ -331,6 +366,12 @@ class Language(StrEnum): RU = "ru" RU_RU = "ru-RU" + # Sanskrit + SA = "sa" + + # Sindhi + SD = "sd" + # Sinhala SI = "si" SI_LK = "si-LK" @@ -343,6 +384,9 @@ class Language(StrEnum): SL = "sl" SL_SI = "sl-SI" + # Shona + SN = "sn" + # Somali SO = "so" SO_SO = "so-SO" @@ -384,14 +428,23 @@ class Language(StrEnum): TE = "te" TE_IN = "te-IN" + # Tajik + TG = "tg" + # Thai TH = "th" TH_TH = "th-TH" + # Turkmen + TK = "tk" + # Turkish TR = "tr" TR_TR = "tr-TR" + # Tatar + TT = "tt" + # Ukrainian UK = "uk" UK_UA = "uk-UA" @@ -413,6 +466,12 @@ class Language(StrEnum): WUU = "wuu" WUU_CN = "wuu-CN" + # Yiddish + YI = "yi" + + # Yoruba + YO = "yo" + # Yue Chinese YUE = "yue" YUE_CN = "yue-CN" From e4bb4aacb4c2438d13f5422cf54860fa264ca6cb Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 20 Mar 2025 12:46:00 -0400 Subject: [PATCH 083/132] Example: Rename 07 ultravox example --- ...7u-interruptible-ultravox.py => 07v-interruptible-ultravox.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/foundational/{07u-interruptible-ultravox.py => 07v-interruptible-ultravox.py} (100%) diff --git a/examples/foundational/07u-interruptible-ultravox.py b/examples/foundational/07v-interruptible-ultravox.py similarity index 100% rename from examples/foundational/07u-interruptible-ultravox.py rename to examples/foundational/07v-interruptible-ultravox.py From 2b4debec1116883d520f038b611329b3fb80b9c7 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 16 Mar 2025 18:23:30 -0700 Subject: [PATCH 084/132] add support for conversation.item.input_audio_transcription.delta --- src/pipecat/services/openai_realtime_beta/events.py | 8 ++++++++ src/pipecat/services/openai_realtime_beta/openai.py | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 17ce0a6d4..caa7e964c 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -193,6 +193,13 @@ class ConversationItemCreated(ServerEvent): item: ConversationItem +class ConversationItemInputAudioTranscriptionDelta(ServerEvent): + type: Literal["conversation.item.input_audio_transcription.delta"] + item_id: str + content_index: int + delta: str + + class ConversationItemInputAudioTranscriptionCompleted(ServerEvent): type: Literal["conversation.item.input_audio_transcription.completed"] item_id: str @@ -400,6 +407,7 @@ _server_event_types = { "input_audio_buffer.speech_started": InputAudioBufferSpeechStarted, "input_audio_buffer.speech_stopped": InputAudioBufferSpeechStopped, "conversation.item.created": ConversationItemCreated, + "conversation.item.input_audio_transcription.delta": ConversationItemInputAudioTranscriptionDelta, "conversation.item.input_audio_transcription.completed": ConversationItemInputAudioTranscriptionCompleted, "conversation.item.input_audio_transcription.failed": ConversationItemInputAudioTranscriptionFailed, "conversation.item.truncated": ConversationItemTruncated, diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 321c66826..6f0edd67c 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -29,6 +29,7 @@ from pipecat.frames.frames import ( EndFrame, ErrorFrame, Frame, + InterimTranscriptionFrame, InputAudioRawFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, @@ -354,6 +355,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): await self._handle_evt_audio_done(evt) elif evt.type == "conversation.item.created": await self._handle_evt_conversation_item_created(evt) + elif evt.type == "conversation.item.input_audio_transcription.delta": + await self._handle_evt_input_audio_transcription_delta(evt) elif evt.type == "conversation.item.input_audio_transcription.completed": await self.handle_evt_input_audio_transcription_completed(evt) elif evt.type == "response.done": @@ -425,6 +428,13 @@ class OpenAIRealtimeBetaLLMService(LLMService): self._current_assistant_response = evt.item await self.push_frame(LLMFullResponseStartFrame()) + async def _handle_evt_input_audio_transcription_delta(self, evt): + if self._send_transcription_frames: + await self.push_frame( + # no way to get a language code? + InterimTranscriptionFrame(evt.delta, "", time_now_iso8601()) + ) + async def handle_evt_input_audio_transcription_completed(self, evt): if self._send_transcription_frames: await self.push_frame( From bfdf52bd69428faea7cb8eb6077b69dd5b8b7be0 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 16 Mar 2025 18:27:17 -0700 Subject: [PATCH 085/132] change examples/foundational/19-openai-realtime-beta.py to use the new preview model --- examples/foundational/19-openai-realtime-beta.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 8796e0141..8027b0669 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -114,6 +114,7 @@ Remember, your responses should be short. Just one or two sentences, usually.""" llm = OpenAIRealtimeBetaLLMService( api_key=os.getenv("OPENAI_API_KEY"), + model="gpt-4o-realtime-preview-latest", session_properties=session_properties, start_audio_paused=False, ) From 4449e9a25bcf79b7659afec00937aebfcd5a06dd Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 16 Mar 2025 18:40:19 -0700 Subject: [PATCH 086/132] add response.done status=failed error --- examples/foundational/19-openai-realtime-beta.py | 1 - src/pipecat/services/openai_realtime_beta/openai.py | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 8027b0669..8796e0141 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -114,7 +114,6 @@ Remember, your responses should be short. Just one or two sentences, usually.""" llm = OpenAIRealtimeBetaLLMService( api_key=os.getenv("OPENAI_API_KEY"), - model="gpt-4o-realtime-preview-latest", session_properties=session_properties, start_audio_paused=False, ) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 6f0edd67c..f4c99a73f 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -87,7 +87,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): self, *, api_key: str, - model: str = "gpt-4o-realtime-preview-2024-12-17", + # model: str = "gpt-4o-realtime-preview-2024-12-17", + model: str = "gpt-4o-realtime-preview-latest", base_url: str = "wss://api.openai.com/v1/realtime", session_properties: events.SessionProperties = events.SessionProperties(), start_audio_paused: bool = False, @@ -465,6 +466,10 @@ class OpenAIRealtimeBetaLLMService(LLMService): await self.stop_processing_metrics() await self.push_frame(LLMFullResponseEndFrame()) self._current_assistant_response = None + # error handling + if evt.response.status == "failed": + await self.push_error(ErrorFrame(error=evt.response.status_details["error"]["message"], fatal=True)) + return # response content pair = self._user_and_response_message_tuple if pair: From 16accafa6debc58db7cbd7c5fff40360eda2ca10 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 10:47:00 -0400 Subject: [PATCH 087/132] formatting fix --- src/pipecat/services/openai_realtime_beta/openai.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index f4c99a73f..879d96e6f 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -468,7 +468,9 @@ class OpenAIRealtimeBetaLLMService(LLMService): self._current_assistant_response = None # error handling if evt.response.status == "failed": - await self.push_error(ErrorFrame(error=evt.response.status_details["error"]["message"], fatal=True)) + await self.push_error( + ErrorFrame(error=evt.response.status_details["error"]["message"], fatal=True) + ) return # response content pair = self._user_and_response_message_tuple From 214c8f79eb2b5f372df0b118c1f389bf5d7f433a Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 10:50:04 -0400 Subject: [PATCH 088/132] linter fix --- src/pipecat/services/openai_realtime_beta/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 879d96e6f..68dde4d32 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -29,8 +29,8 @@ from pipecat.frames.frames import ( EndFrame, ErrorFrame, Frame, - InterimTranscriptionFrame, InputAudioRawFrame, + InterimTranscriptionFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMessagesAppendFrame, From e80bfe22de734d9449b53fd6d0224159fd8cc6ab Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 12:15:14 -0400 Subject: [PATCH 089/132] Add new GPT-4o transcription option to OpenAIRealtimeBetaLLMService --- examples/foundational/19-openai-realtime-beta.py | 4 ++-- examples/foundational/19a-azure-realtime-beta.py | 4 ++-- .../20b-persistent-context-openai-realtime.py | 4 ++-- .../services/openai_realtime_beta/__init__.py | 2 +- src/pipecat/services/openai_realtime_beta/events.py | 13 +++++++++++-- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 8796e0141..c39c0f570 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -21,7 +21,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai_realtime_beta import ( - InputAudioTranscription, + InputAudioTranscriptionModels, OpenAIRealtimeBetaLLMService, SessionProperties, TurnDetection, @@ -89,7 +89,7 @@ async def main(): ) session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscription(), + input_audio_transcription=InputAudioTranscriptionModels.Whisper1(), # Set openai TurnDetection parameters. Not setting this at all will turn it # on by default turn_detection=TurnDetection(silence_duration_ms=1000), diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py index 14d034836..12847ee33 100644 --- a/examples/foundational/19a-azure-realtime-beta.py +++ b/examples/foundational/19a-azure-realtime-beta.py @@ -23,7 +23,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai_realtime_beta import ( AzureRealtimeBetaLLMService, - InputAudioTranscription, + InputAudioTranscriptionModels, SessionProperties, TurnDetection, ) @@ -90,7 +90,7 @@ async def main(): ) session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscription(), + input_audio_transcription=InputAudioTranscriptionModels.Whisper1(), # Set openai TurnDetection parameters. Not setting this at all will turn it # on by default # turn_detection=TurnDetection(silence_duration_ms=1000), diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index ef82bd567..26a6448e9 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -25,7 +25,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, ) from pipecat.services.openai_realtime_beta import ( - InputAudioTranscription, + InputAudioTranscriptionModels, OpenAIRealtimeBetaLLMService, SessionProperties, TurnDetection, @@ -186,7 +186,7 @@ async def main(): ) session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscription(), + input_audio_transcription=InputAudioTranscriptionModels.Whisper1(), # Set openai TurnDetection parameters. Not setting this at all will turn it # on by default turn_detection=TurnDetection(silence_duration_ms=1000), diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py index 52b00f6c8..fdda648c4 100644 --- a/src/pipecat/services/openai_realtime_beta/__init__.py +++ b/src/pipecat/services/openai_realtime_beta/__init__.py @@ -1,3 +1,3 @@ from .azure import AzureRealtimeBetaLLMService -from .events import InputAudioTranscription, SessionProperties, TurnDetection +from .events import InputAudioTranscriptionModels, SessionProperties, TurnDetection from .openai import OpenAIRealtimeBetaLLMService diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index caa7e964c..9cc946807 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -16,8 +16,17 @@ from pydantic import BaseModel, Field # -class InputAudioTranscription(BaseModel): - model: Optional[str] = "whisper-1" +class InputAudioTranscriptionModels: + class Whisper1(BaseModel): + model: Optional[Literal["whisper-1"]] = "whisper-1" + + class GPT4o(BaseModel): + model: Optional[Literal["gpt-4o-transcribe-latest"]] = "gpt-4o-transcribe-latest" + language: Optional[str] = None + prompt: Optional[str] = None + + +InputAudioTranscription = Union[InputAudioTranscriptionModels.Whisper1, InputAudioTranscriptionModels.GPT4o] class TurnDetection(BaseModel): From be2cf6d556aacf8b2233f7e83907657b9414d1ae Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 12:28:11 -0400 Subject: [PATCH 090/132] formatting fix --- src/pipecat/services/openai_realtime_beta/events.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 9cc946807..adfc34133 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -26,7 +26,9 @@ class InputAudioTranscriptionModels: prompt: Optional[str] = None -InputAudioTranscription = Union[InputAudioTranscriptionModels.Whisper1, InputAudioTranscriptionModels.GPT4o] +InputAudioTranscription = Union[ + InputAudioTranscriptionModels.Whisper1, InputAudioTranscriptionModels.GPT4o +] class TurnDetection(BaseModel): From fe5fc302114586b6eeeaed55fd59c2711319345a Mon Sep 17 00:00:00 2001 From: kompfner Date: Mon, 17 Mar 2025 12:36:31 -0400 Subject: [PATCH 091/132] Revert "Add new GPT-4o transcription option to OpenAIRealtimeBetaLLMService" --- examples/foundational/19-openai-realtime-beta.py | 4 ++-- examples/foundational/19a-azure-realtime-beta.py | 4 ++-- .../20b-persistent-context-openai-realtime.py | 4 ++-- .../services/openai_realtime_beta/__init__.py | 2 +- .../services/openai_realtime_beta/events.py | 15 ++------------- 5 files changed, 9 insertions(+), 20 deletions(-) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index c39c0f570..8796e0141 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -21,7 +21,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai_realtime_beta import ( - InputAudioTranscriptionModels, + InputAudioTranscription, OpenAIRealtimeBetaLLMService, SessionProperties, TurnDetection, @@ -89,7 +89,7 @@ async def main(): ) session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscriptionModels.Whisper1(), + input_audio_transcription=InputAudioTranscription(), # Set openai TurnDetection parameters. Not setting this at all will turn it # on by default turn_detection=TurnDetection(silence_duration_ms=1000), diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py index 12847ee33..14d034836 100644 --- a/examples/foundational/19a-azure-realtime-beta.py +++ b/examples/foundational/19a-azure-realtime-beta.py @@ -23,7 +23,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai_realtime_beta import ( AzureRealtimeBetaLLMService, - InputAudioTranscriptionModels, + InputAudioTranscription, SessionProperties, TurnDetection, ) @@ -90,7 +90,7 @@ async def main(): ) session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscriptionModels.Whisper1(), + input_audio_transcription=InputAudioTranscription(), # Set openai TurnDetection parameters. Not setting this at all will turn it # on by default # turn_detection=TurnDetection(silence_duration_ms=1000), diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index 26a6448e9..ef82bd567 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -25,7 +25,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, ) from pipecat.services.openai_realtime_beta import ( - InputAudioTranscriptionModels, + InputAudioTranscription, OpenAIRealtimeBetaLLMService, SessionProperties, TurnDetection, @@ -186,7 +186,7 @@ async def main(): ) session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscriptionModels.Whisper1(), + input_audio_transcription=InputAudioTranscription(), # Set openai TurnDetection parameters. Not setting this at all will turn it # on by default turn_detection=TurnDetection(silence_duration_ms=1000), diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py index fdda648c4..52b00f6c8 100644 --- a/src/pipecat/services/openai_realtime_beta/__init__.py +++ b/src/pipecat/services/openai_realtime_beta/__init__.py @@ -1,3 +1,3 @@ from .azure import AzureRealtimeBetaLLMService -from .events import InputAudioTranscriptionModels, SessionProperties, TurnDetection +from .events import InputAudioTranscription, SessionProperties, TurnDetection from .openai import OpenAIRealtimeBetaLLMService diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index adfc34133..caa7e964c 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -16,19 +16,8 @@ from pydantic import BaseModel, Field # -class InputAudioTranscriptionModels: - class Whisper1(BaseModel): - model: Optional[Literal["whisper-1"]] = "whisper-1" - - class GPT4o(BaseModel): - model: Optional[Literal["gpt-4o-transcribe-latest"]] = "gpt-4o-transcribe-latest" - language: Optional[str] = None - prompt: Optional[str] = None - - -InputAudioTranscription = Union[ - InputAudioTranscriptionModels.Whisper1, InputAudioTranscriptionModels.GPT4o -] +class InputAudioTranscription(BaseModel): + model: Optional[str] = "whisper-1" class TurnDetection(BaseModel): From d009b804383a5dfc70bc3c6fcd655c6d5cde42cd Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 12:54:32 -0400 Subject: [PATCH 092/132] Add new GPT-4o transcription option to OpenAIRealtimeBetaLLMService --- .../services/openai_realtime_beta/events.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index caa7e964c..fcc330a6e 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -14,10 +14,25 @@ from pydantic import BaseModel, Field # # session properties # +InputAudioTranscriptionModelArg = Optional[Literal["whisper-1", "gpt-4o-transcribe-latest"]] class InputAudioTranscription(BaseModel): - model: Optional[str] = "whisper-1" + model: InputAudioTranscriptionModelArg + language: Optional[str] + prompt: Optional[str] + + def __init__( + self, + model: InputAudioTranscriptionModelArg = "whisper-1", + language: Optional[str] = None, + prompt: Optional[str] = None, + ): + super().__init__(model=model, language=language, prompt=prompt) + if self.model != "gpt-4o-transcribe-latest" and (self.language or self.prompt): + raise ValueError( + "Fields 'language' and 'prompt' are only supported when model is 'gpt-4o-transcribe-latest'" + ) class TurnDetection(BaseModel): From 1a20d9bed7c57d993c6a08d0b3ec747839190584 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 13:02:55 -0400 Subject: [PATCH 093/132] Add new input_audio_noise_reduction option to OpenAIRealtimeBetaLLMService --- src/pipecat/services/openai_realtime_beta/__init__.py | 7 ++++++- src/pipecat/services/openai_realtime_beta/events.py | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py index 52b00f6c8..a4c102609 100644 --- a/src/pipecat/services/openai_realtime_beta/__init__.py +++ b/src/pipecat/services/openai_realtime_beta/__init__.py @@ -1,3 +1,8 @@ from .azure import AzureRealtimeBetaLLMService -from .events import InputAudioTranscription, SessionProperties, TurnDetection +from .events import ( + InputAudioTranscription, + InputAudioNoiseReduction, + SessionProperties, + TurnDetection, +) from .openai import OpenAIRealtimeBetaLLMService diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index fcc330a6e..26fbc0578 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -42,6 +42,10 @@ class TurnDetection(BaseModel): silence_duration_ms: Optional[int] = 800 +class InputAudioNoiseReduction(BaseModel): + type: Optional[Literal["near_field", "far_field"]] + + class SessionProperties(BaseModel): modalities: Optional[List[Literal["text", "audio"]]] = None instructions: Optional[str] = None @@ -49,6 +53,7 @@ class SessionProperties(BaseModel): input_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None output_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None input_audio_transcription: Optional[InputAudioTranscription] = None + input_audio_noise_reduction: Optional[InputAudioNoiseReduction] = None # set turn_detection to False to disable turn detection turn_detection: Optional[Union[TurnDetection, bool]] = Field(default=None) tools: Optional[List[Dict]] = None From e91610c69eb1439679cdeb369b565f01276633c5 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 13:18:52 -0400 Subject: [PATCH 094/132] linter fix --- src/pipecat/services/openai_realtime_beta/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py index a4c102609..d54640919 100644 --- a/src/pipecat/services/openai_realtime_beta/__init__.py +++ b/src/pipecat/services/openai_realtime_beta/__init__.py @@ -1,7 +1,7 @@ from .azure import AzureRealtimeBetaLLMService from .events import ( - InputAudioTranscription, InputAudioNoiseReduction, + InputAudioTranscription, SessionProperties, TurnDetection, ) From 1075c25055873344f63d7b9f89669aa3c069b1ab Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 15:28:31 -0400 Subject: [PATCH 095/132] Add new semantic turn detection option to OpenAIRealtimeBetaLLMService --- src/pipecat/services/openai_realtime_beta/__init__.py | 1 + src/pipecat/services/openai_realtime_beta/events.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py index d54640919..595105d7f 100644 --- a/src/pipecat/services/openai_realtime_beta/__init__.py +++ b/src/pipecat/services/openai_realtime_beta/__init__.py @@ -2,6 +2,7 @@ from .azure import AzureRealtimeBetaLLMService from .events import ( InputAudioNoiseReduction, InputAudioTranscription, + SemanticTurnDetection, SessionProperties, TurnDetection, ) diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 26fbc0578..c8a7c383f 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -42,6 +42,13 @@ class TurnDetection(BaseModel): silence_duration_ms: Optional[int] = 800 +class SemanticTurnDetection(BaseModel): + type: Optional[Literal["semantic_vad"]] = "semantic_vad" + eagerness: Optional[Literal["low", "medium", "high", "auto"]] = None + create_response: Optional[bool] = None + interrupt_response: Optional[bool] = None + + class InputAudioNoiseReduction(BaseModel): type: Optional[Literal["near_field", "far_field"]] @@ -55,7 +62,9 @@ class SessionProperties(BaseModel): input_audio_transcription: Optional[InputAudioTranscription] = None input_audio_noise_reduction: Optional[InputAudioNoiseReduction] = None # set turn_detection to False to disable turn detection - turn_detection: Optional[Union[TurnDetection, bool]] = Field(default=None) + turn_detection: Optional[Union[TurnDetection, SemanticTurnDetection, bool]] = Field( + default=None + ) tools: Optional[List[Dict]] = None tool_choice: Optional[Literal["auto", "none", "required"]] = None temperature: Optional[float] = None From 9840abd85b828043bed7dfb4dab2c23dee19b546 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 14:51:19 -0400 Subject: [PATCH 096/132] Make it so you specifying `model=None` when creating a `InputAudioTranscription` results in a validation error --- src/pipecat/services/openai_realtime_beta/events.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index c8a7c383f..0f42a30b0 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -14,17 +14,17 @@ from pydantic import BaseModel, Field # # session properties # -InputAudioTranscriptionModelArg = Optional[Literal["whisper-1", "gpt-4o-transcribe-latest"]] +InputAudioTranscriptionModel = Literal["whisper-1", "gpt-4o-transcribe-latest"] class InputAudioTranscription(BaseModel): - model: InputAudioTranscriptionModelArg + model: InputAudioTranscriptionModel language: Optional[str] prompt: Optional[str] def __init__( self, - model: InputAudioTranscriptionModelArg = "whisper-1", + model: Optional[InputAudioTranscriptionModel] = "whisper-1", language: Optional[str] = None, prompt: Optional[str] = None, ): From 39ca607bbbe72a481c664c6d90055c23454ec9fc Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 16:52:53 -0400 Subject: [PATCH 097/132] Add `on_conversation_item_created` and `on_conversation_item_updated` events to OpenAIRealtimeBetaLLMService. The hope is that this will expose to the user conversation item ids at relevant times for them to use with the new `conversation.item.retrieve` introspection message. --- src/pipecat/services/openai_realtime_beta/openai.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 68dde4d32..4a3a14cf8 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -117,6 +117,9 @@ class OpenAIRealtimeBetaLLMService(LLMService): self._messages_added_manually = {} self._user_and_response_message_tuple = None + self._register_event_handler("on_conversation_item_created") + self._register_event_handler("on_conversation_item_updated") + def can_generate_metrics(self) -> bool: return True @@ -413,6 +416,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): # receive a BotStoppedSpeakingFrame from the output transport. async def _handle_evt_conversation_item_created(self, evt): + await self._call_event_handler("on_conversation_item_created", evt.item.id, evt.item) + # This will get sent from the server every time a new "message" is added # to the server's conversation state, whether we create it via the API # or the server creates it from LLM output. @@ -437,6 +442,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): ) async def handle_evt_input_audio_transcription_completed(self, evt): + await self._call_event_handler("on_conversation_item_updated", evt.item_id, None) + if self._send_transcription_frames: await self.push_frame( # no way to get a language code? @@ -473,6 +480,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): ) return # response content + for item in evt.response.output: + await self._call_event_handler("on_conversation_item_updated", item.id, item) pair = self._user_and_response_message_tuple if pair: user, assistant = pair From f693a3c70f442ac410d15bed39a0fa9be0f3578b Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 21:54:56 -0400 Subject: [PATCH 098/132] Add `retrieve_conversation_item()` method to OpenAIRealtimeBetaLLMService, using the new `conversation.item.retrieve` introspection message. --- .../services/openai_realtime_beta/events.py | 11 +++++++++++ .../services/openai_realtime_beta/openai.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 0f42a30b0..88638b5fd 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -179,6 +179,11 @@ class ConversationItemDeleteEvent(ClientEvent): item_id: str +class ConversationItemRetrieveEvent(ClientEvent): + type: Literal["conversation.item.retrieve"] = "conversation.item.retrieve" + item_id: str + + class ResponseCreateEvent(ClientEvent): type: Literal["response.create"] = "response.create" response: Optional[ResponseProperties] = None @@ -255,6 +260,11 @@ class ConversationItemDeleted(ServerEvent): item_id: str +class ConversationItemRetrieved(ServerEvent): + type: Literal["conversation.item.retrieved"] + item: ConversationItem + + class ResponseCreated(ServerEvent): type: Literal["response.created"] response: "Response" @@ -441,6 +451,7 @@ _server_event_types = { "conversation.item.input_audio_transcription.failed": ConversationItemInputAudioTranscriptionFailed, "conversation.item.truncated": ConversationItemTruncated, "conversation.item.deleted": ConversationItemDeleted, + "conversation.item.retrieved": ConversationItemRetrieved, "response.created": ResponseCreated, "response.done": ResponseDone, "response.output_item.added": ResponseOutputItemAdded, diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 4a3a14cf8..922f6f2ff 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -119,6 +119,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): self._register_event_handler("on_conversation_item_created") self._register_event_handler("on_conversation_item_updated") + self._retrieve_conversation_item_futures = {} def can_generate_metrics(self) -> bool: return True @@ -126,6 +127,12 @@ class OpenAIRealtimeBetaLLMService(LLMService): def set_audio_input_paused(self, paused: bool): self._audio_input_paused = paused + async def retrieve_conversation_item(self, item_id: str): + future = self.get_event_loop().create_future() + self._retrieve_conversation_item_futures[item_id] = future + await self.send_client_event(events.ConversationItemRetrieveEvent(item_id=item_id)) + return await future + # # standard AIService frame handling # @@ -363,6 +370,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): await self._handle_evt_input_audio_transcription_delta(evt) elif evt.type == "conversation.item.input_audio_transcription.completed": await self.handle_evt_input_audio_transcription_completed(evt) + elif evt.type == "conversation.item.retrieved": + await self._handle_conversation_item_retrieved(evt) elif evt.type == "response.done": await self._handle_evt_response_done(evt) elif evt.type == "input_audio_buffer.speech_started": @@ -461,6 +470,14 @@ class OpenAIRealtimeBetaLLMService(LLMService): # User message without preceding conversation.item.created. Bug? logger.warning(f"Transcript for unknown user message: {evt}") + async def _handle_conversation_item_retrieved(self, evt: events.ConversationItemRetrieved): + future = self._retrieve_conversation_item_futures.get(evt.item.id) + if future: + # print(f"[pk] setting result: {evt.item}") + future.set_result(evt.item) + # TODO: handle error + # TODO: what happens if we try to receive bogus item id? + async def _handle_evt_response_done(self, evt): # todo: figure out whether there's anything we need to do for "cancelled" events # usage metrics From 31317ce77d51abd060556b67a02a44c1fa5870ae Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 18 Mar 2025 09:42:20 -0400 Subject: [PATCH 099/132] Add error handling to the `retrieve_conversation_item()` method of the OpenAIRealtimeBetaLLMService --- .../services/openai_realtime_beta/openai.py | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 922f6f2ff..52a03ba94 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -23,6 +23,8 @@ except ModuleNotFoundError as e: ) raise Exception(f"Missing module: {e}") +import re + from pipecat.frames.frames import ( BotStoppedSpeakingFrame, CancelFrame, @@ -381,9 +383,10 @@ class OpenAIRealtimeBetaLLMService(LLMService): elif evt.type == "response.audio_transcript.delta": await self._handle_evt_audio_transcript_delta(evt) elif evt.type == "error": - await self._handle_evt_error(evt) - # errors are fatal, so exit the receive loop - return + if not await self._maybe_handle_evt_retrieve_conversation_item_error(evt): + await self._handle_evt_error(evt) + # errors are fatal, so exit the receive loop + return async def _handle_evt_session_created(self, evt): # session.created is received right after connecting. Send a message @@ -471,12 +474,10 @@ class OpenAIRealtimeBetaLLMService(LLMService): logger.warning(f"Transcript for unknown user message: {evt}") async def _handle_conversation_item_retrieved(self, evt: events.ConversationItemRetrieved): - future = self._retrieve_conversation_item_futures.get(evt.item.id) + future = self._retrieve_conversation_item_futures.pop(evt.item.id, None) if future: # print(f"[pk] setting result: {evt.item}") future.set_result(evt.item) - # TODO: handle error - # TODO: what happens if we try to receive bogus item id? async def _handle_evt_response_done(self, evt): # todo: figure out whether there's anything we need to do for "cancelled" events @@ -530,6 +531,24 @@ class OpenAIRealtimeBetaLLMService(LLMService): await self.push_frame(StopInterruptionFrame()) await self.push_frame(UserStoppedSpeakingFrame()) + async def _maybe_handle_evt_retrieve_conversation_item_error(self, evt: events.ErrorEvent): + """If the given error event is an error retrieving a conversation item: + - set an exception on the future that retrieve_conversation_item() is waiting on + - return true + Otherwise: + - return false + """ + match = re.match( + r"^Error retrieving item: the item with id '(.*)' does not exist\.$", evt.error.message + ) + if match: + item_id = match.group(1) + future = self._retrieve_conversation_item_futures.pop(item_id, None) + if future: + future.set_exception(Exception(evt.error.message)) + return True + return False + async def _handle_evt_error(self, evt): # Errors are fatal to this connection. Send an ErrorFrame. await self.push_error(ErrorFrame(error=f"Error: {evt}", fatal=True)) From 7b594093ddb7b586c0c3c521f082bfee2a362a64 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 18 Mar 2025 10:05:41 -0400 Subject: [PATCH 100/132] Handle the possibility of multiple concurrent calls to `retrieve_conversation_item()` in the OpenAIRealtimeBetaLLMService --- .../services/openai_realtime_beta/openai.py | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 52a03ba94..4b993c28a 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -131,8 +131,14 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def retrieve_conversation_item(self, item_id: str): future = self.get_event_loop().create_future() - self._retrieve_conversation_item_futures[item_id] = future - await self.send_client_event(events.ConversationItemRetrieveEvent(item_id=item_id)) + retrieval_in_progress = False + if not self._retrieve_conversation_item_futures.get(item_id): + self._retrieve_conversation_item_futures[item_id] = [] + else: + retrieval_in_progress = True + self._retrieve_conversation_item_futures[item_id].append(future) + if not retrieval_in_progress: + await self.send_client_event(events.ConversationItemRetrieveEvent(item_id=item_id)) return await future # @@ -474,10 +480,10 @@ class OpenAIRealtimeBetaLLMService(LLMService): logger.warning(f"Transcript for unknown user message: {evt}") async def _handle_conversation_item_retrieved(self, evt: events.ConversationItemRetrieved): - future = self._retrieve_conversation_item_futures.pop(evt.item.id, None) - if future: - # print(f"[pk] setting result: {evt.item}") - future.set_result(evt.item) + futures = self._retrieve_conversation_item_futures.pop(evt.item.id, None) + if futures: + for future in futures: + future.set_result(evt.item) async def _handle_evt_response_done(self, evt): # todo: figure out whether there's anything we need to do for "cancelled" events @@ -543,9 +549,10 @@ class OpenAIRealtimeBetaLLMService(LLMService): ) if match: item_id = match.group(1) - future = self._retrieve_conversation_item_futures.pop(item_id, None) - if future: - future.set_exception(Exception(evt.error.message)) + futures = self._retrieve_conversation_item_futures.pop(item_id, None) + if futures: + for future in futures: + future.set_exception(Exception(evt.error.message)) return True return False From e707efbffa84cf77dab967fa3c58d5ac2fa68a6f Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 18 Mar 2025 12:30:19 -0400 Subject: [PATCH 101/132] Update changelog with slate of recent updates to OpenAIRealtimeBetaLLMService --- CHANGELOG.md | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d55b97c3..42fad1c44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -128,8 +128,60 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Gemini models. Added foundational example `14p-function-calling-gemini-vertex-ai.py`. +- Added support in `OpenAIRealtimeBetaLLMService` for the + `conversation.item.input_audio_transcription.delta` server message. + +- Added error handling in `OpenAIRealtimeBetaLLMService` for the + `response.done` server message reporting a failure. + +- Added support in `OpenAIRealtimeBetaLLMService` for the new + `gpt-4o-transcribe-latest` input audio transcription model. + +- Added support in `OpenAIRealtimeBetaLLMService` for the new + `input_audio_noise_reduction` session property. + + ```python + session_properties = SessionProperties( + # ... + input_audio_noise_reduction=InputAudioNoiseReduction( + type="near_field" # also supported: "far_field" + ) + # ... + ) + ``` + +- Added support in `OpenAIRealtimeBetaLLMService` for the new + `semantic_vad` `turn_detection` session property, which is a more + sophisticated model for detecting when the user has stopped speaking. + +- Added `on_conversation_item_created` and `on_conversation_item_updated` + events to `OpenAIRealtimeBetaLLMService`. + + ```python + @llm.event_handler("on_conversation_item_created") + async def on_conversation_item_created(llm, item_id, item): + # ... + + @llm.event_handler("on_conversation_item_updated") + async def on_conversation_item_updated(llm, item_id, item): + # `item` may not always be available here + # ... + ``` + +- Added `retrieve_conversation_item(item_id)` to `OpenAIRealtimeBetaLLMService`. + + ```python + item = await llm.retrieve_conversation_item(item_id) + ``` + ### Changed +- Updated the default model for `CartesiaTTSService` and + `CartesiaHttpTTSService` to `sonic-2`. + +- Updated the default model for `OpenAIRealtimeBetaLLMService` to + `gpt-4o-realtime-preview-latest`. + - Function calls are now executed in tasks. This means that the pipeline will not be blocked while the function call is being executed. @@ -216,6 +268,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed an issue in `RimeTTSService` where the last line of text sent didn't result in an audio output being generated. +- Fixed `OpenAIRealtimeBetaLLMService` by adding support for the + `conversation.item.input_audio_transcription.delta` server message, which was + added server-side at some point and not handled client-side. + ### Other - Add foundational example `07w-interruptible-fal.py`, showing `FalSTTService`. From 3dd4ef72303c45c439eebcfd9a4e7c1d1a955047 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 18 Mar 2025 15:52:11 -0400 Subject: [PATCH 102/132] Tweak changelog entries describing slate of recent updates to OpenAIRealtimeBetaLLMService --- CHANGELOG.md | 73 ++++++++++++++++++++++++---------------------------- 1 file changed, 33 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42fad1c44..fe531bf7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -128,51 +128,43 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Gemini models. Added foundational example `14p-function-calling-gemini-vertex-ai.py`. -- Added support in `OpenAIRealtimeBetaLLMService` for the - `conversation.item.input_audio_transcription.delta` server message. +- Added support in `OpenAIRealtimeBetaLLMService` for a slate of new features: -- Added error handling in `OpenAIRealtimeBetaLLMService` for the - `response.done` server message reporting a failure. + - The `'gpt-4o-transcribe-latest'` input audio transcription model. + - The `input_audio_noise_reduction` session property. -- Added support in `OpenAIRealtimeBetaLLMService` for the new - `gpt-4o-transcribe-latest` input audio transcription model. - -- Added support in `OpenAIRealtimeBetaLLMService` for the new - `input_audio_noise_reduction` session property. - - ```python - session_properties = SessionProperties( - # ... - input_audio_noise_reduction=InputAudioNoiseReduction( - type="near_field" # also supported: "far_field" + ```python + session_properties = SessionProperties( + # ... + input_audio_noise_reduction=InputAudioNoiseReduction( + type="near_field" # also supported: "far_field" + ) + # ... ) - # ... - ) - ``` + ``` -- Added support in `OpenAIRealtimeBetaLLMService` for the new - `semantic_vad` `turn_detection` session property, which is a more - sophisticated model for detecting when the user has stopped speaking. + - The `'semantic_vad'` `turn_detection` session property value, a more + sophisticated model for detecting when the user has stopped speaking. + - `on_conversation_item_created` and `on_conversation_item_updated` + events to `OpenAIRealtimeBetaLLMService`. -- Added `on_conversation_item_created` and `on_conversation_item_updated` - events to `OpenAIRealtimeBetaLLMService`. + ```python + @llm.event_handler("on_conversation_item_created") + async def on_conversation_item_created(llm, item_id, item): + # ... - ```python - @llm.event_handler("on_conversation_item_created") - async def on_conversation_item_created(llm, item_id, item): - # ... + @llm.event_handler("on_conversation_item_updated") + async def on_conversation_item_updated(llm, item_id, item): + # `item` may not always be available here + # ... + ``` - @llm.event_handler("on_conversation_item_updated") - async def on_conversation_item_updated(llm, item_id, item): - # `item` may not always be available here - # ... - ``` + - The `retrieve_conversation_item(item_id)` method for introspecting a + conversation item on the server. -- Added `retrieve_conversation_item(item_id)` to `OpenAIRealtimeBetaLLMService`. - - ```python - item = await llm.retrieve_conversation_item(item_id) - ``` + ```python + item = await llm.retrieve_conversation_item(item_id) + ``` ### Changed @@ -268,9 +260,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed an issue in `RimeTTSService` where the last line of text sent didn't result in an audio output being generated. -- Fixed `OpenAIRealtimeBetaLLMService` by adding support for the - `conversation.item.input_audio_transcription.delta` server message, which was - added server-side at some point and not handled client-side. +- Fixed `OpenAIRealtimeBetaLLMService` by adding proper handling for: + - The `conversation.item.input_audio_transcription.delta` server message, + which was added server-side at some point and not handled client-side. + - Errors reported by the `response.done` server message. ### Other From f94a099111a7b23952be85e7b0d13e9a09158117 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 19 Mar 2025 12:00:19 -0400 Subject: [PATCH 103/132] Revert the default model to be "gpt-4o-realtime-preview-2024-12-17" In OpenAIRealtimeBetaLLMService --- CHANGELOG.md | 6 ++---- src/pipecat/services/openai_realtime_beta/openai.py | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe531bf7f..660cf5c82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,7 +130,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added support in `OpenAIRealtimeBetaLLMService` for a slate of new features: - - The `'gpt-4o-transcribe-latest'` input audio transcription model. + - The `'gpt-4o-transcribe-latest'` input audio transcription model, along + with new `language` and `prompt` options specific to that model. - The `input_audio_noise_reduction` session property. ```python @@ -171,9 +172,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated the default model for `CartesiaTTSService` and `CartesiaHttpTTSService` to `sonic-2`. -- Updated the default model for `OpenAIRealtimeBetaLLMService` to - `gpt-4o-realtime-preview-latest`. - - Function calls are now executed in tasks. This means that the pipeline will not be blocked while the function call is being executed. diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 4b993c28a..ab78a4451 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -89,8 +89,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): self, *, api_key: str, - # model: str = "gpt-4o-realtime-preview-2024-12-17", - model: str = "gpt-4o-realtime-preview-latest", + model: str = "gpt-4o-realtime-preview-2024-12-17", base_url: str = "wss://api.openai.com/v1/realtime", session_properties: events.SessionProperties = events.SessionProperties(), start_audio_paused: bool = False, From 0d74bcacb74388724e4b36bcf764869813d3698d Mon Sep 17 00:00:00 2001 From: Chad Bailey Date: Thu, 20 Mar 2025 01:12:13 +0000 Subject: [PATCH 104/132] updated models in the 07g example --- examples/foundational/07g-interruptible-openai.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/foundational/07g-interruptible-openai.py b/examples/foundational/07g-interruptible-openai.py index f9cac5910..fe4b2a53c 100644 --- a/examples/foundational/07g-interruptible-openai.py +++ b/examples/foundational/07g-interruptible-openai.py @@ -51,9 +51,11 @@ async def main(): # api_key="gsk_***", # model="whisper-large-v3", # ) - stt = OpenAISTTService(api_key=os.getenv("OPENAI_API_KEY"), model="whisper-1") + stt = OpenAISTTService( + api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-transcribe-latest" + ) - tts = OpenAITTSService(api_key=os.getenv("OPENAI_API_KEY"), voice="alloy") + tts = OpenAITTSService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini-tts-latest") llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") From 2ae5bdd8a96fe3f3333814846e8863758b4ed9f0 Mon Sep 17 00:00:00 2001 From: Chad Bailey Date: Thu, 20 Mar 2025 01:20:07 +0000 Subject: [PATCH 105/132] lets talk about dogs --- examples/foundational/07g-interruptible-openai.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/foundational/07g-interruptible-openai.py b/examples/foundational/07g-interruptible-openai.py index fe4b2a53c..b66d0346a 100644 --- a/examples/foundational/07g-interruptible-openai.py +++ b/examples/foundational/07g-interruptible-openai.py @@ -52,7 +52,9 @@ async def main(): # model="whisper-large-v3", # ) stt = OpenAISTTService( - api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-transcribe-latest" + api_key=os.getenv("OPENAI_API_KEY"), + model="gpt-4o-transcribe-latest", + prompt="Expect words related to dogs, such as breed names.", ) tts = OpenAITTSService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini-tts-latest") @@ -62,7 +64,7 @@ async def main(): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are very knowledgable about dogs. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", }, ] From f0774268cc83145da2b92786b7646a386c801fe9 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 19 Mar 2025 21:58:46 -0400 Subject: [PATCH 106/132] Rename gpt-4o-transcribe-latest to gpt-4o-transcribe in OpenAIRealtimeBetaLLMService --- CHANGELOG.md | 2 +- src/pipecat/services/openai_realtime_beta/events.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 660cf5c82..6825a62f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,7 +130,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added support in `OpenAIRealtimeBetaLLMService` for a slate of new features: - - The `'gpt-4o-transcribe-latest'` input audio transcription model, along + - The `'gpt-4o-transcribe'` input audio transcription model, along with new `language` and `prompt` options specific to that model. - The `input_audio_noise_reduction` session property. diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 88638b5fd..3a18833d2 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -14,7 +14,7 @@ from pydantic import BaseModel, Field # # session properties # -InputAudioTranscriptionModel = Literal["whisper-1", "gpt-4o-transcribe-latest"] +InputAudioTranscriptionModel = Literal["whisper-1", "gpt-4o-transcribe"] class InputAudioTranscription(BaseModel): @@ -29,9 +29,9 @@ class InputAudioTranscription(BaseModel): prompt: Optional[str] = None, ): super().__init__(model=model, language=language, prompt=prompt) - if self.model != "gpt-4o-transcribe-latest" and (self.language or self.prompt): + if self.model != "gpt-4o-transcribe" and (self.language or self.prompt): raise ValueError( - "Fields 'language' and 'prompt' are only supported when model is 'gpt-4o-transcribe-latest'" + "Fields 'language' and 'prompt' are only supported when model is 'gpt-4o-transcribe'" ) From 70dbf0d6fc952e594b09506744eca9c57142172c Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 20 Mar 2025 11:48:06 -0400 Subject: [PATCH 107/132] Updated default models for OpenAISTTService and OpenAITTSService to gpt-4o based models --- CHANGELOG.md | 5 +++++ src/pipecat/services/openai.py | 12 ++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6825a62f5..ac0a6ef3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -169,6 +169,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Updated `OpenAISTTService` to use `gpt-4o-transcribe` as the default + transcription model. + +- Updated `OpenAITTSService` to use `gpt-4o-mini-tts` as the default TTS model. + - Updated the default model for `CartesiaTTSService` and `CartesiaHttpTTSService` to `sonic-2`. diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index b5f5d9d54..4d3e1d6d7 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -409,13 +409,13 @@ class OpenAIImageGenService(ImageGenService): class OpenAISTTService(BaseWhisperSTTService): - """OpenAI Whisper speech-to-text service. + """OpenAI Speech-to-Text service that generates text from audio. - Uses OpenAI's Whisper API to convert audio to text. Requires an OpenAI API key + Uses OpenAI's transcription API to convert audio to text. Requires an OpenAI API key set via the api_key parameter or OPENAI_API_KEY environment variable. Args: - model: Whisper model to use. Defaults to "whisper-1". + model: Model to use — either gpt-4o or Whisper. Defaults to "gpt-4o-transcribe". api_key: OpenAI API key. Defaults to None. base_url: API base URL. Defaults to None. language: Language of the audio input. Defaults to English. @@ -427,7 +427,7 @@ class OpenAISTTService(BaseWhisperSTTService): def __init__( self, *, - model: str = "whisper-1", + model: str = "gpt-4o-transcribe", api_key: Optional[str] = None, base_url: Optional[str] = None, language: Optional[Language] = Language.EN, @@ -472,7 +472,7 @@ class OpenAITTSService(TTSService): Args: api_key: OpenAI API key. Defaults to None. voice: Voice ID to use. Defaults to "alloy". - model: TTS model to use. Defaults to "tts-1". + model: TTS model to use. Defaults to "gpt-4o-mini-tts". sample_rate: Output audio sample rate in Hz. Defaults to None. **kwargs: Additional keyword arguments passed to TTSService. @@ -487,7 +487,7 @@ class OpenAITTSService(TTSService): *, api_key: Optional[str] = None, voice: str = "alloy", - model: str = "tts-1", + model: str = "gpt-4o-mini-tts", sample_rate: Optional[int] = None, **kwargs, ): From ada68f0699e09323f5d05b0ff5ce461754437f4e Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 19 Mar 2025 22:25:21 -0400 Subject: [PATCH 108/132] More robust handling of conversation item retrieval errors in OpenAIRealtimeBetaLLMService --- .../services/openai_realtime_beta/events.py | 1 + .../services/openai_realtime_beta/openai.py | 23 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 3a18833d2..f4133766b 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -122,6 +122,7 @@ class RealtimeError(BaseModel): code: Optional[str] = "" message: str param: Optional[str] = None + event_id: Optional[str] = None # diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index ab78a4451..324e5653a 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -23,8 +23,6 @@ except ModuleNotFoundError as e: ) raise Exception(f"Missing module: {e}") -import re - from pipecat.frames.frames import ( BotStoppedSpeakingFrame, CancelFrame, @@ -130,14 +128,20 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def retrieve_conversation_item(self, item_id: str): future = self.get_event_loop().create_future() - retrieval_in_progress = False + retrieval_in_flight = False if not self._retrieve_conversation_item_futures.get(item_id): self._retrieve_conversation_item_futures[item_id] = [] else: - retrieval_in_progress = True + retrieval_in_flight = True self._retrieve_conversation_item_futures[item_id].append(future) - if not retrieval_in_progress: - await self.send_client_event(events.ConversationItemRetrieveEvent(item_id=item_id)) + if not retrieval_in_flight: + await self.send_client_event( + # Set event_id to "rci_{item_id}" so that we can identiy an + # error later if the retrieval fails. We don't need a UUID + # suffix to the event_id because we're ensuring only one + # in-flight retrieval per item_id + events.ConversationItemRetrieveEvent(item_id=item_id, event_id=f"rci_{item_id}") + ) return await future # @@ -543,11 +547,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): Otherwise: - return false """ - match = re.match( - r"^Error retrieving item: the item with id '(.*)' does not exist\.$", evt.error.message - ) - if match: - item_id = match.group(1) + if evt.error.code == "item_retrieve_invalid_item_id": + item_id = evt.error.event_id.split("_", 1)[1] # event_id is of the form "rci_{item_id}" futures = self._retrieve_conversation_item_futures.pop(item_id, None) if futures: for future in futures: From 721ee75887441827c46819b4efff4e4895df93c1 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 20 Mar 2025 12:41:40 -0400 Subject: [PATCH 109/132] Comment tweak --- src/pipecat/services/openai_realtime_beta/openai.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 324e5653a..c8f1f597a 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -136,10 +136,11 @@ class OpenAIRealtimeBetaLLMService(LLMService): self._retrieve_conversation_item_futures[item_id].append(future) if not retrieval_in_flight: await self.send_client_event( - # Set event_id to "rci_{item_id}" so that we can identiy an + # Set event_id to "rci_{item_id}" so that we can identify an # error later if the retrieval fails. We don't need a UUID # suffix to the event_id because we're ensuring only one - # in-flight retrieval per item_id + # in-flight retrieval per item_id. (Note: "rci" = "retrieve + # conversation item") events.ConversationItemRetrieveEvent(item_id=item_id, event_id=f"rci_{item_id}") ) return await future From 44380bc8c0370dffe9685b3542f679b5d2b4ada0 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 20 Mar 2025 13:51:16 -0400 Subject: [PATCH 110/132] Remove duplicate changelog entry due to rebase mistake --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac0a6ef3c..065a5ced6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -174,9 +174,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated `OpenAITTSService` to use `gpt-4o-mini-tts` as the default TTS model. -- Updated the default model for `CartesiaTTSService` and - `CartesiaHttpTTSService` to `sonic-2`. - - Function calls are now executed in tasks. This means that the pipeline will not be blocked while the function call is being executed. From efa5f133d747783c43c11ff5b1aad929a1f01484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 11:14:59 -0700 Subject: [PATCH 111/132] openai_realtime: fix and update function calling --- .../services/openai_realtime_beta/context.py | 72 +++---------------- .../services/openai_realtime_beta/openai.py | 4 +- 2 files changed, 11 insertions(+), 65 deletions(-) diff --git a/src/pipecat/services/openai_realtime_beta/context.py b/src/pipecat/services/openai_realtime_beta/context.py index 31639dc6b..c8381976f 100644 --- a/src/pipecat/services/openai_realtime_beta/context.py +++ b/src/pipecat/services/openai_realtime_beta/context.py @@ -12,6 +12,7 @@ from loguru import logger from pipecat.frames.frames import ( Frame, + FunctionCallResultFrame, FunctionCallResultProperties, LLMMessagesUpdateFrame, LLMSetToolsFrame, @@ -174,67 +175,12 @@ class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator): class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator): - async def push_aggregation(self): - # the only thing we implement here is function calling. in all other cases, messages - # are added to the context when we receive openai realtime api events - if not self._function_call_result: - return + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + await super().handle_function_call_result(frame) - properties: Optional[FunctionCallResultProperties] = None - - self.reset() - try: - run_llm = True - frame = self._function_call_result - properties = frame.properties - self._function_call_result = None - if frame.result: - # The "tool_call" message from the LLM that triggered the function call - self._context.add_message( - { - "role": "assistant", - "tool_calls": [ - { - "id": frame.tool_call_id, - "function": { - "name": frame.function_name, - "arguments": json.dumps(frame.arguments), - }, - "type": "function", - } - ], - } - ) - # The result of the function call. Need to add this both to our context here and to - # the openai realtime api context. - result_message = { - "role": "tool", - "content": json.dumps(frame.result), - "tool_call_id": frame.tool_call_id, - } - - self._context.add_message(result_message) - # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself, - # so we didn't have a chance to add the result to the openai realtime api context. Let's push a - # special frame to do that. - await self.push_frame( - RealtimeFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM - ) - if properties and properties.run_llm is not None: - # If the tool call result has a run_llm property, use it - run_llm = properties.run_llm - else: - # Default behavior is to run the LLM if there are no function calls in progress - run_llm = not bool(self._function_calls_in_progress) - - if run_llm: - await self.push_context_frame(FrameDirection.UPSTREAM) - - # Emit the on_context_updated callback once the function call result is added to the context - if properties and properties.on_context_updated is not None: - await properties.on_context_updated() - - await self.push_context_frame() - - except Exception as e: - logger.error(f"Error processing frame: {e}") + # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself, + # so we didn't have a chance to add the result to the openai realtime api context. Let's push a + # special frame to do that. + await self.push_frame( + RealtimeFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM + ) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index c8f1f597a..a1f4bc731 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -579,7 +579,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): arguments = json.loads(item.arguments) if self.has_function(function_name): run_llm = index == total_items - 1 - if function_name in self._callbacks.keys(): + if function_name in self._functions.keys(): await self.call_function( context=self._context, tool_call_id=tool_id, @@ -587,7 +587,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): arguments=arguments, run_llm=run_llm, ) - elif None in self._callbacks.keys(): + elif None in self._functions.keys(): await self.call_function( context=self._context, tool_call_id=tool_id, From 5a39f146f604d7349344d5d46913c2511d648702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 09:47:33 -0700 Subject: [PATCH 112/132] LLMUserContextAggregator: fix emulated user started/stopped speaking issues --- CHANGELOG.md | 7 ++- .../processors/aggregators/llm_response.py | 46 +++++++------------ src/pipecat/transports/base_input.py | 4 +- tests/test_context_aggregators.py | 7 +-- 4 files changed, 22 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 065a5ced6..0a3563229 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -367,6 +367,9 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ### Fixed +- Fixed an issue that would cause undesired interruptions via + `EmulateUserStartedSpeakingFrame`. + - Fixed a `GoogleLLMService` that was causing an exception when sending inline audio in some cases. @@ -383,10 +386,6 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) - Fixed `match_endofsentence` support for ellipses. -- Fixed an issue that would cause undesired interruptions via - `EmulateUserStartedSpeakingFrame` when only interim transcriptions (i.e. no - final transcriptions) where received. - - Fixed an issue where `EndTaskFrame` was not triggering `on_client_disconnected` or closing the WebSocket in FastAPI. diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 9267af364..75435a214 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -5,7 +5,6 @@ # import asyncio -import time from abc import abstractmethod from typing import Dict, List @@ -222,17 +221,15 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): self, context: OpenAILLMContext, aggregation_timeout: float = 1.0, - bot_interruption_timeout: float = 5.0, **kwargs, ): super().__init__(context=context, role="user", **kwargs) self._aggregation_timeout = aggregation_timeout - self._bot_interruption_timeout = bot_interruption_timeout self._seen_interim_results = False self._user_speaking = False - self._last_user_speaking_time = 0 self._emulating_vad = False + self._waiting_for_aggregation = False self._aggregation_event = asyncio.Event() self._aggregation_task = None @@ -240,6 +237,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): def reset(self): super().reset() self._seen_interim_results = False + self._waiting_for_aggregation = False async def handle_aggregation(self, aggregation: str): self._context.add_message({"role": self.role, "content": self._aggregation}) @@ -285,14 +283,11 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): # Reset the aggregation. Reset it before pushing it down, otherwise # if the tasks gets cancelled we won't be able to clear things up. - self._aggregation = "" + self.reset() frame = OpenAILLMContextFrame(self._context) await self.push_frame(frame) - # Reset our accumulator state. - self.reset() - async def _start(self, frame: StartFrame): self._create_aggregation_task() @@ -303,12 +298,14 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): await self._cancel_aggregation_task() async def _handle_user_started_speaking(self, _: UserStartedSpeakingFrame): - self._last_user_speaking_time = time.time() self._user_speaking = True + self._waiting_for_aggregation = True async def _handle_user_stopped_speaking(self, _: UserStoppedSpeakingFrame): - self._last_user_speaking_time = time.time() self._user_speaking = False + # We just stopped speaking. Let's see if there's some aggregation to + # push. If the last thing we saw is an interim transcription, let's wait + # pushing the aggregation as we will probably get a final transcription. if not self._seen_interim_results: await self.push_aggregation() @@ -361,18 +358,13 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): frame we might want to interrupt the bot. """ - if not self._user_speaking: - diff_time = time.time() - self._last_user_speaking_time - if diff_time > self._bot_interruption_timeout: - # If we reach this case we received a transcription but VAD was - # not able to detect voice (e.g. when you whisper a short - # utterance). So, we need to emulate VAD (i.e. user - # start/stopped speaking). - await self.push_frame(EmulateUserStartedSpeakingFrame(), FrameDirection.UPSTREAM) - self._emulating_vad = True - - # Reset time so we don't interrupt again right away. - self._last_user_speaking_time = time.time() + if not self._user_speaking and not self._waiting_for_aggregation: + # If we reach this case we received a transcription but VAD was not + # able to detect voice (e.g. when you whisper a short + # utterance). So, we need to emulate VAD (i.e. user start/stopped + # speaking). + await self.push_frame(EmulateUserStartedSpeakingFrame(), FrameDirection.UPSTREAM) + self._emulating_vad = True class LLMAssistantContextAggregator(LLMContextResponseAggregator): @@ -554,14 +546,11 @@ class LLMUserResponseAggregator(LLMUserContextAggregator): # Reset the aggregation. Reset it before pushing it down, otherwise # if the tasks gets cancelled we won't be able to clear things up. - self._aggregation = "" + self.reset() frame = LLMMessagesFrame(self._context.messages) await self.push_frame(frame) - # Reset our accumulator state. - self.reset() - class LLMAssistantResponseAggregator(LLMAssistantContextAggregator): def __init__(self, messages: List[dict] = [], **kwargs): @@ -573,10 +562,7 @@ class LLMAssistantResponseAggregator(LLMAssistantContextAggregator): # Reset the aggregation. Reset it before pushing it down, otherwise # if the tasks gets cancelled we won't be able to clear things up. - self._aggregation = "" + self.reset() frame = LLMMessagesFrame(self._context.messages) await self.push_frame(frame) - - # Reset our accumulator state. - self.reset() diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index 782ad1333..971dfe066 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -152,6 +152,7 @@ class BaseInputTransport(FrameProcessor): async def _handle_user_interruption(self, frame: Frame): if isinstance(frame, UserStartedSpeakingFrame): logger.debug("User started speaking") + await self.push_frame(frame) # Make sure we notify about interruptions quickly out-of-band. if self.interruptions_allowed: await self._start_interruption() @@ -161,12 +162,11 @@ class BaseInputTransport(FrameProcessor): await self.push_frame(StartInterruptionFrame()) elif isinstance(frame, UserStoppedSpeakingFrame): logger.debug("User stopped speaking") + await self.push_frame(frame) if self.interruptions_allowed: await self._stop_interruption() await self.push_frame(StopInterruptionFrame()) - await self.push_frame(frame) - # # Audio input # diff --git a/tests/test_context_aggregators.py b/tests/test_context_aggregators.py index 0c9b6d5e4..185725632 100644 --- a/tests/test_context_aggregators.py +++ b/tests/test_context_aggregators.py @@ -44,8 +44,6 @@ from pipecat.tests.utils import SleepFrame, run_test AGGREGATION_TIMEOUT = 0.1 AGGREGATION_SLEEP = 0.15 -BOT_INTERRUPTION_TIMEOUT = 0.2 -BOT_INTERRUPTION_SLEEP = 0.25 class BaseTestUserContextAggregator: @@ -388,14 +386,13 @@ class BaseTestUserContextAggregator: aggregator = self.AGGREGATOR_CLASS( context, aggregation_timeout=AGGREGATION_TIMEOUT, - bot_interruption_timeout=BOT_INTERRUPTION_TIMEOUT, ) frames_to_send = [ UserStartedSpeakingFrame(), InterimTranscriptionFrame(text="How ", user_id="cat", timestamp=""), SleepFrame(), UserStoppedSpeakingFrame(), - SleepFrame(BOT_INTERRUPTION_SLEEP), + SleepFrame(AGGREGATION_SLEEP), InterimTranscriptionFrame(text="are you?", user_id="cat", timestamp=""), TranscriptionFrame(text="How are you?", user_id="cat", timestamp=""), SleepFrame(sleep=AGGREGATION_SLEEP), @@ -405,12 +402,10 @@ class BaseTestUserContextAggregator: UserStoppedSpeakingFrame, *self.EXPECTED_CONTEXT_FRAMES, ] - expected_up_frames = [EmulateUserStartedSpeakingFrame, EmulateUserStoppedSpeakingFrame] await run_test( aggregator, frames_to_send=frames_to_send, expected_down_frames=expected_down_frames, - expected_up_frames=expected_up_frames, ) self.check_message_content(context, 0, "How are you?") From 08956e914a93146e7cdec4ad3c27d6375176a708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 10:06:45 -0700 Subject: [PATCH 113/132] livekit: remove unnecessary transport cleanup() function --- src/pipecat/transports/services/livekit.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py index 149ca4b7c..8ce5c885c 100644 --- a/src/pipecat/transports/services/livekit.py +++ b/src/pipecat/transports/services/livekit.py @@ -599,13 +599,6 @@ class LiveKitTransport(BaseTransport): ) await self._output.send_message(frame) - async def cleanup(self): - if self._input: - await self._input.cleanup() - if self._output: - await self._output.cleanup() - await self._client.disconnect() - async def on_room_event(self, event): # Handle room events pass From 66ba1116a40c9f599180f226b1929c6786621321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 10:49:13 -0700 Subject: [PATCH 114/132] pyproject: rollback azure to 1.42.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 87a34f929..4e300bd91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ Website = "https://pipecat.ai" anthropic = [ "anthropic~=0.49.0" ] assemblyai = [ "assemblyai~=0.37.0" ] aws = [ "boto3~=1.37.16" ] -azure = [ "azure-cognitiveservices-speech~=1.43.0"] +azure = [ "azure-cognitiveservices-speech~=1.42.0"] canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.4.0", "websockets~=13.1" ] neuphonic = [ "pyneuphonic~=1.5.13", "websockets~=13.1" ] From b20ce7d655adb5ab87cfd6c9075f6e2a2bf42498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 10:55:11 -0700 Subject: [PATCH 115/132] examples: move 07u-interruptible-neuphonic to 07v --- CHANGELOG.md | 10 ++++++++++ ...tible-ultravox.py => 07u-interruptible-ultravox.py} | 0 ...nic-http.py => 07v-interruptible-neuphonic-http.py} | 0 ...ble-neuphonic.py => 07v-interruptible-neuphonic.py} | 0 4 files changed, 10 insertions(+) rename examples/foundational/{07v-interruptible-ultravox.py => 07u-interruptible-ultravox.py} (100%) rename examples/foundational/{07u-interruptible-neuphonic-http.py => 07v-interruptible-neuphonic-http.py} (100%) rename examples/foundational/{07u-interruptible-neuphonic.py => 07v-interruptible-neuphonic.py} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a3563229..787be8afd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added new `sample_rate` constructor parameter to `TavusVideoService` to allow changing the output sample rate. +- Added new `NeuphonicTTSService`. + (see https://neuphonic.com) + - Added new `UltravoxSTTService`. (see https://github.com/fixie-ai/ultravox) @@ -269,6 +272,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add foundational example `07w-interruptible-fal.py`, showing `FalSTTService`. +- Added a new Ultravox example + `examples/foundational/07u-interruptible-ultravox.py`. + +- Added new Neuphonic examples + `examples/foundational/07v-interruptible-neuphonic.py` and + `examples/foundational/07v-interruptible-neuphonic-http.py`. + - Added a new example `examples/foundational/36-user-email-gathering.py` to show how to gather user emails. The example uses's Cartesia's `` tags and Rime `spell()` function to spell out the emails for confirmation. diff --git a/examples/foundational/07v-interruptible-ultravox.py b/examples/foundational/07u-interruptible-ultravox.py similarity index 100% rename from examples/foundational/07v-interruptible-ultravox.py rename to examples/foundational/07u-interruptible-ultravox.py diff --git a/examples/foundational/07u-interruptible-neuphonic-http.py b/examples/foundational/07v-interruptible-neuphonic-http.py similarity index 100% rename from examples/foundational/07u-interruptible-neuphonic-http.py rename to examples/foundational/07v-interruptible-neuphonic-http.py diff --git a/examples/foundational/07u-interruptible-neuphonic.py b/examples/foundational/07v-interruptible-neuphonic.py similarity index 100% rename from examples/foundational/07u-interruptible-neuphonic.py rename to examples/foundational/07v-interruptible-neuphonic.py From 2133152e5b98f63ff525ec820fd1358cf8816cc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 11:42:54 -0700 Subject: [PATCH 116/132] update CHANGELOG for 0.0.59 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 787be8afd..6d7b6f72e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.59] - 2025-03-20 ### Added From 8f6d92ce7d11a8107e72a5604f704a4156aec371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 13:47:15 -0700 Subject: [PATCH 117/132] update CHANGELOG with BaseOpenAILLMService `default_headers` --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d7b6f72e..3a0524e29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Added `default_headers` parameter to `BaseOpenAILLMService` constructor. + ## [0.0.59] - 2025-03-20 ### Added From 541a4b6063140e7d92c9f7982cbb492f4581a8be Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 20 Mar 2025 15:12:43 -0400 Subject: [PATCH 118/132] Update InputAudioTranscription to use gpt-4o-transcribe model, update 19 examples to use FunctionSchema --- CHANGELOG.md | 10 +++++ .../foundational/19-openai-realtime-beta.py | 41 +++++++++--------- .../foundational/19a-azure-realtime-beta.py | 43 +++++++++---------- .../services/openai_realtime_beta/events.py | 13 ++++-- 4 files changed, 61 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a0524e29..63885acbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `default_headers` parameter to `BaseOpenAILLMService` constructor. +### Changed + +- Changed the default `InputAudioTranscription` model to `gpt-4o-transcribe` + for `OpenAIRealtimeBetaLLMService`. + +### Other + +- Update the `19-openai-realtime-beta.py` and `19a-azure-realtime-beta.py` + examples to use the FunctionSchema format. + ## [0.0.59] - 2025-03-20 ### Added diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 8796e0141..bb62466a7 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -14,6 +14,8 @@ from dotenv import load_dotenv from loguru import logger from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline @@ -46,28 +48,25 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context ) -tools = [ - { - "type": "function", - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], +weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - } -] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", + }, + }, + required=["location", "format"], +) + +# Create tools schema +tools = ToolsSchema(standard_tools=[weather_function]) async def main(): diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py index 14d034836..7ec1d195a 100644 --- a/examples/foundational/19a-azure-realtime-beta.py +++ b/examples/foundational/19a-azure-realtime-beta.py @@ -10,11 +10,12 @@ import sys from datetime import datetime import aiohttp -import websockets from dotenv import load_dotenv from loguru import logger from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline @@ -47,28 +48,26 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context ) -tools = [ - { - "type": "function", - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], +# Define weather function using standardized schema +weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - } -] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", + }, + }, + required=["location", "format"], +) + +# Create tools schema +tools = ToolsSchema(standard_tools=[weather_function]) async def main(): diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index f4133766b..c2dcb7f09 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -14,17 +14,24 @@ from pydantic import BaseModel, Field # # session properties # -InputAudioTranscriptionModel = Literal["whisper-1", "gpt-4o-transcribe"] class InputAudioTranscription(BaseModel): - model: InputAudioTranscriptionModel + """Configuration for audio transcription settings. + + Attributes: + model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1"). + language: Optional language code for transcription. + prompt: Optional transcription hint text. + """ + + model: str = "gpt-4o-transcribe" language: Optional[str] prompt: Optional[str] def __init__( self, - model: Optional[InputAudioTranscriptionModel] = "whisper-1", + model: Optional[str] = "gpt-4o-transcribe", language: Optional[str] = None, prompt: Optional[str] = None, ): From 41688205be87f443cbc73e08b215150befdbafa4 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 20 Mar 2025 15:23:57 -0400 Subject: [PATCH 119/132] Provide new settings in OpenAI Realtime example --- examples/foundational/19-openai-realtime-beta.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index bb62466a7..f2349f6d2 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -26,7 +26,8 @@ from pipecat.services.openai_realtime_beta import ( InputAudioTranscription, OpenAIRealtimeBetaLLMService, SessionProperties, - TurnDetection, + SemanticTurnDetection, + InputAudioNoiseReduction ) from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -91,9 +92,10 @@ async def main(): input_audio_transcription=InputAudioTranscription(), # Set openai TurnDetection parameters. Not setting this at all will turn it # on by default - turn_detection=TurnDetection(silence_duration_ms=1000), + turn_detection=SemanticTurnDetection(), # Or set to False to disable openai turn detection and use transport VAD # turn_detection=False, + input_audio_noise_reduction=InputAudioNoiseReduction(type='near_field'), # tools=tools, instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. From 2ac8f2ec2d5acd26527a2e37510ddf17a521f5a0 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 20 Mar 2025 15:50:51 -0400 Subject: [PATCH 120/132] Fix linting --- .../foundational/19-openai-realtime-beta.py | 6 +++--- .../foundational/19a-azure-realtime-beta.py | 1 - src/pipecat/services/openai.py | 19 ++++++++++++++++--- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index f2349f6d2..3aff14e65 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -23,11 +23,11 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai_realtime_beta import ( + InputAudioNoiseReduction, InputAudioTranscription, OpenAIRealtimeBetaLLMService, - SessionProperties, SemanticTurnDetection, - InputAudioNoiseReduction + SessionProperties, ) from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -95,7 +95,7 @@ async def main(): turn_detection=SemanticTurnDetection(), # Or set to False to disable openai turn detection and use transport VAD # turn_detection=False, - input_audio_noise_reduction=InputAudioNoiseReduction(type='near_field'), + input_audio_noise_reduction=InputAudioNoiseReduction(type="near_field"), # tools=tools, instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py index 7ec1d195a..2eefd4ec9 100644 --- a/examples/foundational/19a-azure-realtime-beta.py +++ b/examples/foundational/19a-azure-realtime-beta.py @@ -26,7 +26,6 @@ from pipecat.services.openai_realtime_beta import ( AzureRealtimeBetaLLMService, InputAudioTranscription, SessionProperties, - TurnDetection, ) from pipecat.transports.services.daily import DailyParams, DailyTransport diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 48b108ad0..ff7bc0442 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -129,10 +129,23 @@ class BaseOpenAILLMService(LLMService): } self.set_model_name(model) self._client = self.create_client( - api_key=api_key, base_url=base_url, organization=organization, project=project, default_headers=default_headers, **kwargs + api_key=api_key, + base_url=base_url, + organization=organization, + project=project, + default_headers=default_headers, + **kwargs, ) - def create_client(self, api_key=None, base_url=None, organization=None, project=None, default_headers=None, **kwargs): + def create_client( + self, + api_key=None, + base_url=None, + organization=None, + project=None, + default_headers=None, + **kwargs, + ): return AsyncOpenAI( api_key=api_key, base_url=base_url, @@ -143,7 +156,7 @@ class BaseOpenAILLMService(LLMService): max_keepalive_connections=100, max_connections=1000, keepalive_expiry=None ) ), - default_headers=default_headers + default_headers=default_headers, ) def can_generate_metrics(self) -> bool: From 66e42ae4104b1e54806fb95ceab2824e2a75024e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 16:14:24 -0700 Subject: [PATCH 121/132] pyproject: rollback deepgram-sdk to 3.8.0 --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63885acbb..7db8d9dc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Rollback to `deepgram-sdk` 3.8.0 since 3.10.1 was causing connections issues. + - Changed the default `InputAudioTranscription` model to `gpt-4o-transcribe` for `OpenAIRealtimeBetaLLMService`. diff --git a/pyproject.toml b/pyproject.toml index 4e300bd91..0d1e46ed9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ neuphonic = [ "pyneuphonic~=1.5.13", "websockets~=13.1" ] cerebras = [] deepseek = [] daily = [ "daily-python~=0.15.0" ] -deepgram = [ "deepgram-sdk~=3.10.1" ] +deepgram = [ "deepgram-sdk~=3.8.0" ] elevenlabs = [ "websockets~=13.1" ] fal = [ "fal-client~=0.5.9" ] fish = [ "ormsgpack~=1.7.0", "websockets~=13.1" ] From f2b9789acfeb5d793f19a436bcf287043ca361c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 16:17:34 -0700 Subject: [PATCH 122/132] update CHANGELOG for 0.0.60 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7db8d9dc9..343efff17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.60] - 2025-03-20 ### Added From 3b4d91e1c17726667605a5e3e7f1abbd43db1443 Mon Sep 17 00:00:00 2001 From: milo157 <43028253+milo157@users.noreply.github.com> Date: Fri, 21 Mar 2025 03:55:43 +0200 Subject: [PATCH 123/132] Fixed ultravox service bugs (#1420) --- .../07u-interruptible-ultravox.py | 1 + src/pipecat/services/ultravox.py | 38 +++++++++++-------- tests/test_user_idle_processor.py | 6 +-- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/examples/foundational/07u-interruptible-ultravox.py b/examples/foundational/07u-interruptible-ultravox.py index 3ae4540f0..429e5a9fb 100644 --- a/examples/foundational/07u-interruptible-ultravox.py +++ b/examples/foundational/07u-interruptible-ultravox.py @@ -14,6 +14,7 @@ from loguru import logger from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask diff --git a/src/pipecat/services/ultravox.py b/src/pipecat/services/ultravox.py index 40029e673..a17ee3e85 100644 --- a/src/pipecat/services/ultravox.py +++ b/src/pipecat/services/ultravox.py @@ -17,6 +17,13 @@ from loguru import logger from pipecat.frames.frames import ( AudioRawFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMTextFrame, + TranscriptionFrame, + TextFrame, + StartFrame, + EndFrame, CancelFrame, EndFrame, ErrorFrame, @@ -339,6 +346,12 @@ class UltravoxSTTService(AIService): # Concatenate audio frames - all should be int16 now audio_data = np.concatenate(audio_arrays) + audio_int16 = audio_data # Already in int16 format + # Save int16 audio + + # Convert int16 to float32 and normalize for model input + audio_float32 = audio_int16.astype(np.float32) / 32768.0 + # Generate text using the model if self._model: try: @@ -349,11 +362,11 @@ class UltravoxSTTService(AIService): await self.start_ttfb_metrics() await self.start_processing_metrics() - async for response in self._model.generate( + async for response in self.model.generate( messages=[{"role": "user", "content": "<|audio|>\n"}], - temperature=self._temperature, - max_tokens=self._max_tokens, - audio=audio_data, + temperature=self.temperature, + max_tokens=self.max_tokens, + audio=audio_float32, ): # Stop TTFB metrics after first response await self.stop_ttfb_metrics() @@ -369,18 +382,13 @@ class UltravoxSTTService(AIService): await self.stop_processing_metrics() logger.info(f"Generated text: {full_response}") - # Create a transcription frame with the generated text - transcription = full_response.strip() - if transcription: - yield TranscriptionFrame( - user_id="", - text=transcription, - timestamp=time_now_iso8601(), - ) - else: - logger.warning("Empty transcription result") - yield ErrorFrame("Empty transcription result") + yield LLMFullResponseStartFrame() + + text_frame = LLMTextFrame(text=full_response.strip()) + yield text_frame + + yield LLMFullResponseEndFrame() except Exception as e: logger.error(f"Error generating text from model: {e}") diff --git a/tests/test_user_idle_processor.py b/tests/test_user_idle_processor.py index a2f2fd386..7ea6f8744 100644 --- a/tests/test_user_idle_processor.py +++ b/tests/test_user_idle_processor.py @@ -86,9 +86,9 @@ class TestUserIdleProcessor(unittest.IsolatedAsyncioTestCase): expected_down_frames=expected_down_frames, ) - assert not callback_called.is_set(), ( - "Idle callback was called even though bot speaking frames reset the timer" - ) + assert ( + not callback_called.is_set() + ), "Idle callback was called even though bot speaking frames reset the timer" async def test_idle_retry_callback(self): """Test that retry count increases until user activity resets it.""" From d71b5201539209be80c165e003bd744eec4256b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 18:58:06 -0700 Subject: [PATCH 124/132] update CHANGELOG.md and fix formatting --- CHANGELOG.md | 7 +++++++ tests/test_user_idle_processor.py | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 343efff17..701b8a81c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed + +- Fixed an issue in `UltravoxSTTService` that caused improper audio processing + and incorrect LLM frame output. + ## [0.0.60] - 2025-03-20 ### Added diff --git a/tests/test_user_idle_processor.py b/tests/test_user_idle_processor.py index 7ea6f8744..a2f2fd386 100644 --- a/tests/test_user_idle_processor.py +++ b/tests/test_user_idle_processor.py @@ -86,9 +86,9 @@ class TestUserIdleProcessor(unittest.IsolatedAsyncioTestCase): expected_down_frames=expected_down_frames, ) - assert ( - not callback_called.is_set() - ), "Idle callback was called even though bot speaking frames reset the timer" + assert not callback_called.is_set(), ( + "Idle callback was called even though bot speaking frames reset the timer" + ) async def test_idle_retry_callback(self): """Test that retry count increases until user activity resets it.""" From fc78e6fc5a0062a266ec122a0e52297cb3a416da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 19:13:23 -0700 Subject: [PATCH 125/132] ElevenLabs: add support for a sample rate of 8000 --- CHANGELOG.md | 4 ++++ src/pipecat/services/elevenlabs.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 701b8a81c..c79c8f3b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- ElevenLabs TTS services now support a sample rate of 8000. + ### Fixed - Fixed an issue in `UltravoxSTTService` that caused improper audio processing diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index 568b9eb64..68c71a144 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -102,6 +102,8 @@ def language_to_elevenlabs_language(language: Language) -> Optional[str]: def output_format_from_sample_rate(sample_rate: int) -> str: match sample_rate: + case 8000: + return "pcm_8000" case 16000: return "pcm_16000" case 22050: @@ -113,7 +115,7 @@ def output_format_from_sample_rate(sample_rate: int) -> str: logger.warning( f"ElevenLabsTTSService: No output format available for {sample_rate} sample rate" ) - return "pcm_16000" + return "pcm_24000" def build_elevenlabs_voice_settings( From 442f18d47b323cb0494a37f876bb1cf01377251b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 19:16:56 -0700 Subject: [PATCH 126/132] ultravox: fix formatting --- src/pipecat/services/ultravox.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/pipecat/services/ultravox.py b/src/pipecat/services/ultravox.py index a17ee3e85..52a5d05eb 100644 --- a/src/pipecat/services/ultravox.py +++ b/src/pipecat/services/ultravox.py @@ -17,25 +17,19 @@ from loguru import logger from pipecat.frames.frames import ( AudioRawFrame, - LLMFullResponseEndFrame, - LLMFullResponseStartFrame, - LLMTextFrame, - TranscriptionFrame, - TextFrame, - StartFrame, - EndFrame, CancelFrame, EndFrame, ErrorFrame, Frame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMTextFrame, StartFrame, - TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import AIService -from pipecat.utils.time import time_now_iso8601 try: from transformers import AutoTokenizer From e77f7c84566dce84a5dd7b2b4ac9ac9bb6b11ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 19:16:49 -0700 Subject: [PATCH 127/132] update ruff and pyright versions --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 2b55adcbe..e65c2755c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,10 +3,10 @@ coverage~=7.6.12 grpcio-tools~=1.67.1 pip-tools~=7.4.1 pre-commit~=4.0.1 -pyright~=1.1.394 +pyright~=1.1.397 pytest~=8.3.4 pytest-asyncio~=0.25.3 -ruff~=0.9.7 +ruff~=0.11.1 setuptools~=70.0.0 setuptools_scm~=8.1.0 python-dotenv~=1.0.1 From 04d462ff022b92a98e59591f9300609da01c4322 Mon Sep 17 00:00:00 2001 From: allenmylath Date: Fri, 21 Mar 2025 10:09:09 +0530 Subject: [PATCH 128/132] Update requirements.txt example uses cartesia not elevenlabs --- examples/telnyx-chatbot/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/telnyx-chatbot/requirements.txt b/examples/telnyx-chatbot/requirements.txt index 630a98dd6..e103e438d 100644 --- a/examples/telnyx-chatbot/requirements.txt +++ b/examples/telnyx-chatbot/requirements.txt @@ -1,4 +1,4 @@ -pipecat-ai[openai,silero,deepgram,elevenlabs] +pipecat-ai[openai,silero,deepgram,cartesia] fastapi uvicorn python-dotenv From dd81048ddb8513f824fc796694da00b32e450f07 Mon Sep 17 00:00:00 2001 From: allenmylath Date: Fri, 21 Mar 2025 10:11:28 +0530 Subject: [PATCH 129/132] Update env.example EXAMPLE USES CARTESI NOT ELEVNE LABS --- examples/telnyx-chatbot/env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/telnyx-chatbot/env.example b/examples/telnyx-chatbot/env.example index 5de3bace2..1da398649 100644 --- a/examples/telnyx-chatbot/env.example +++ b/examples/telnyx-chatbot/env.example @@ -1,3 +1,3 @@ OPENAI_API_KEY= DEEPGRAM_API_KEY= -ELEVENLABS_API_KEY= +CARTESIA_API_KEY= From 3ed764a7699f8d290687f23d7ff30c7cc8f932d9 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Fri, 21 Mar 2025 12:56:05 -0300 Subject: [PATCH 130/132] Only checking the length if tools is a list. --- src/pipecat/processors/aggregators/openai_llm_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index 2e5ade0a0..948e3e101 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -157,7 +157,7 @@ class OpenAILLMContext: self._tool_choice = tool_choice def set_tools(self, tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = NOT_GIVEN): - if tools != NOT_GIVEN and len(tools) == 0: + if tools != NOT_GIVEN and isinstance(tools, list) and len(tools) == 0: tools = NOT_GIVEN self._tools = tools From 3311afc5819f1a59938999d9611ac0f3ae992cc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sat, 22 Mar 2025 21:57:32 -0700 Subject: [PATCH 131/132] examples: add foundational 07x-interruptible-local.py --- CHANGELOG.md | 5 + .../foundational/07x-interruptible-local.py | 91 +++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 examples/foundational/07x-interruptible-local.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c79c8f3b3..4749329ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed an issue in `UltravoxSTTService` that caused improper audio processing and incorrect LLM frame output. +### Other + +- Added `examples/foundational/07x-interruptible-local.py` to show how a local + transport can be used. + ## [0.0.60] - 2025-03-20 ### Added diff --git a/examples/foundational/07x-interruptible-local.py b/examples/foundational/07x-interruptible-local.py new file mode 100644 index 000000000..54942ad97 --- /dev/null +++ b/examples/foundational/07x-interruptible-local.py @@ -0,0 +1,91 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.local.audio import LocalAudioTransport, LocalAudioTransportParams + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + transport = LocalAudioTransport( + LocalAudioTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ) + ) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) From 48e8d3968a3680ee8c60782c0763f038b53033d4 Mon Sep 17 00:00:00 2001 From: "Thomas B." Date: Mon, 24 Mar 2025 03:29:52 +0100 Subject: [PATCH 132/132] fix: recognition language correctly set for Azure STT (#1436) --- src/pipecat/services/azure.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/azure.py b/src/pipecat/services/azure.py index c59cd29c2..9df1a8ef1 100644 --- a/src/pipecat/services/azure.py +++ b/src/pipecat/services/azure.py @@ -686,8 +686,11 @@ class AzureSTTService(STTService): ): super().__init__(sample_rate=sample_rate, **kwargs) - self._speech_config = SpeechConfig(subscription=api_key, region=region) - self._speech_config.speech_recognition_language = language + self._speech_config = SpeechConfig( + subscription=api_key, + region=region, + speech_recognition_language=language_to_azure_language(language), + ) self._audio_stream = None self._speech_recognizer = None