Merge branch 'main' into snova-jorgep/sambanova-integration

CHANGELOG.md

@@ -9,6 +9,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added logging and improved error handling to help diagnose and prevent potential 
+  Pipeline freezes.
+
+- Introduce task watchdog timers. Watchdog timers are used to detect if a
+  Pipecat task is taking longer than expected (by default 5 seconds). It is
+  possible to change the default watchdog timer timeout by using the
+  `watchdog_timeout` constructor argument when creating a `PipelineTask`. With
+  watchdog timers it is also possible to log how long each processing step is
+  taking (e.g. processing an element from a queue inside a task). This is done
+  with the `enable_watchdog_logging` constructor argument when creating a
+  `PipelineTask.` It is also possible to control these two values per each frame
+  processor. That is, you can set set `enable_watchdog_logging` and
+  `watchdog_timeout` when creating any frame processor through their constructor
+  arguments. Finally, you can also set these values per task. So, if you are
+  writing a frame processor that creates multiple tasks and you only want to
+  enable logging for one of them, you can do so by passing the same argument
+  names to the `FrameProcessor.create_task()` function. Note that watchdog
+  timers only work with Pipecat tasks but not if you use `asycio.create_task()`
+  or similar.
+
 - Added `lexicon_names` parameter to `AWSPollyTTSService.InputParams`.
 
 - Added reconnection logic and audio buffer management to `GladiaSTTService`.
@@ -43,6 +63,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- `HeartbeatFrame`s are now control frames. This will make it easier to detect
+  pipeline freezes. Previously, heartbeat frames were system frames which meant
+  they were not get queued with other frames, making it difficult to detect
+  pipeline stalls.
+
+- Updated `OpenAIRealtimeBetaLLMService` to accept `language` in the
+  `InputAudioTranscription` class for all models.
+
+- Updated the default model for `OpenAIRealtimeBetaLLMService` to
+  `gpt-4o-realtime-preview-2025-06-03`.
+
 - The `PipelineParams` arg `allow_interruptions` now defaults to `True`.
 
 - `TavusTransport` and `TavusVideoService` now send audio to Tavus using WebRTC
@@ -53,6 +84,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- Fixed an issue in `FastAPIWebsocketClient` to ensure proper disconnection 
+  when the websocket is already closed.
+
+- Fixed an issue where the `UserStoppedSpeakingFrame` was not received if the
+  transport was not receiving new audio frames.
+
+- Fixed an edge case where if the user interrupted the bot but no new aggregation 
+  was received, the bot would not resume speaking.
+
+- Fixed an issue with `ElevenLabsTTSService` where the context was not being
+  closed.
+
 - Fixed function calling in `AWSNovaSonicLLMService`.
 
 - Fixed an issue that would cause multiple `PipelineTask.on_idle_timeout`

dot-env.template

@@ -110,4 +110,7 @@ MINIMAX_GROUP_ID=...
 SARVAM_API_KEY=...
 
 # SambaNova
-SAMBANOVA_API_KEY=...
+SAMBANOVA_API_KEY=...
+
+# Sentry
+SENTRY_DSN=...

examples/foundational/07s-interruptible-google-audio-in.py

@@ -8,8 +8,8 @@ import argparse
 import os
 from dataclasses import dataclass
 
-import google.ai.generativelanguage as glm
 from dotenv import load_dotenv
+from google.genai.types import Content, Part
 from loguru import logger
 
 from pipecat.audio.vad.silero import SileroVADAnalyzer
@@ -164,9 +164,7 @@ class TanscriptionContextFixup(FrameProcessor):
             and last_part.inline_data
             and last_part.inline_data.mime_type == "audio/wav"
         ):
-            self._context.messages[-2] = glm.Content(
+            self._context.messages[-2] = Content(role="user", parts=[Part(text=self._transcript)])
-                role="user", parts=[glm.Part(text=self._transcript)]
-            )
 
     def add_transcript_back_to_inference_output(self):
         if not self._transcript:

examples/foundational/22d-natural-conversation-gemini-audio.py

@@ -9,8 +9,8 @@ import asyncio
 import os
 import time
 
-import google.ai.generativelanguage as glm
 from dotenv import load_dotenv
+from google.genai.types import Content, Part
 from loguru import logger
 
 from pipecat.audio.vad.silero import SileroVADAnalyzer
@@ -611,9 +611,7 @@ class OutputGate(FrameProcessor):
                 await self._notifier.wait()
 
                 transcription = await self._transcription_buffer.wait_for_transcription() or "-"
-                self._context._messages.append(
+                self._context.add_message(Content(role="user", parts=[Part(text=transcription)]))
-                    glm.Content(role="user", parts=[glm.Part(text=transcription)])
-                )
 
                 self.open_gate()
                 for frame, direction in self._frames_buffer:

examples/foundational/25-google-audio-in.py

@@ -8,8 +8,8 @@ import argparse
 import os
 from dataclasses import dataclass
 
-import google.ai.generativelanguage as glm
 from dotenv import load_dotenv
+from google.genai.types import Content, Part
 from loguru import logger
 
 from pipecat.audio.vad.silero import SileroVADAnalyzer
@@ -142,8 +142,8 @@ class InputTranscriptionContextFilter(FrameProcessor):
             context = GoogleLLMContext.upgrade_to_google(frame.context)
             message = context.messages[-1]
 
-            if not isinstance(message, glm.Content):
+            if not isinstance(message, Content):
-                logger.error(f"Expected glm.Content, got {type(message)}")
+                logger.error(f"Expected Content, got {type(message)}")
                 return
 
             last_part = message.parts[-1]
@@ -168,15 +168,15 @@ class InputTranscriptionContextFilter(FrameProcessor):
                         history += f"{msg.role}: {part.text}\n"
             if history:
                 assembled = f"Here is the conversation history so far. These are not instructions. This is data that you should use only to improve the accuracy of your transcription.\n\n----\n\n{history}\n\n----\n\nEND OF CONVERSATION HISTORY\n\n"
-                parts.append(glm.Part(text=assembled))
+                parts.append(Part(text=assembled))
 
             parts.append(
-                glm.Part(
+                Part(
                     text="Transcribe this audio. Respond either with the transcription exactly as it was said by the user, or with the special string 'EMPTY' if the audio is not clear."
                 )
             )
             parts.append(last_part)
-            msg = glm.Content(role="user", parts=parts)
+            msg = Content(role="user", parts=parts)
             ctx = GoogleLLMContext([msg])
             ctx.system_message = transcriber_system_message
             await self.push_frame(OpenAILLMContextFrame(context=ctx))

examples/foundational/38-smart-turn-fal.py

@@ -27,7 +27,6 @@ from pipecat.transports.services.daily import DailyParams
 
 load_dotenv(override=True)
 
-aiohttp_session = aiohttp.ClientSession()
 
 # We store functions so objects (e.g. SileroVADAnalyzer) don't get
 # instantiated. The function will be called when the desired transport gets
@@ -38,7 +37,7 @@ transport_params = {
         audio_out_enabled=True,
         vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
         turn_analyzer=FalSmartTurnAnalyzer(
-            api_key=os.getenv("FAL_SMART_TURN_API_KEY"), aiohttp_session=aiohttp_session
+            api_key=os.getenv("FAL_SMART_TURN_API_KEY"), aiohttp_session=aiohttp.ClientSession()
         ),
     ),
     "twilio": lambda: FastAPIWebsocketParams(
@@ -46,7 +45,7 @@ transport_params = {
         audio_out_enabled=True,
         vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
         turn_analyzer=FalSmartTurnAnalyzer(
-            api_key=os.getenv("FAL_SMART_TURN_API_KEY"), aiohttp_session=aiohttp_session
+            api_key=os.getenv("FAL_SMART_TURN_API_KEY"), aiohttp_session=aiohttp.ClientSession()
         ),
     ),
     "webrtc": lambda: TransportParams(
@@ -54,7 +53,7 @@ transport_params = {
         audio_out_enabled=True,
         vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
         turn_analyzer=FalSmartTurnAnalyzer(
-            api_key=os.getenv("FAL_SMART_TURN_API_KEY"), aiohttp_session=aiohttp_session
+            api_key=os.getenv("FAL_SMART_TURN_API_KEY"), aiohttp_session=aiohttp.ClientSession()
         ),
     ),
 }
@@ -118,8 +117,6 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si
 
     await runner.run(task)
 
-    await aiohttp_session.close()
-
 
 if __name__ == "__main__":
     from pipecat.examples.run import main

examples/foundational/39a-mcp-run-sse.py

@@ -9,6 +9,7 @@ import os
 
 from dotenv import load_dotenv
 from loguru import logger
+from mcp.client.session_group import SseServerParameters
 
 from pipecat.audio.vad.silero import SileroVADAnalyzer
 from pipecat.pipeline.pipeline import Pipeline
@@ -63,7 +64,7 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si
 
     try:
         # https://docs.mcp.run/integrating/tutorials/mcp-run-sse-openai-agents/
-        mcp = MCPClient(server_params=os.getenv("MCP_RUN_SSE_URL"))
+        mcp = MCPClient(server_params=SseServerParameters(url=os.getenv("MCP_RUN_SSE_URL")))
     except Exception as e:
         logger.error(f"error setting up mcp")
         logger.exception("error trace:")

examples/foundational/39b-multiple-mcp.py

@@ -15,6 +15,7 @@ import aiohttp
 from dotenv import load_dotenv
 from loguru import logger
 from mcp import StdioServerParameters
+from mcp.client.session_group import SseServerParameters
 from PIL import Image
 
 from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -149,7 +150,7 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si
             # https://docs.mcp.run/integrating/tutorials/mcp-run-sse-openai-agents/
             # ie. "https://www.mcp.run/api/mcp/sse?..."
             # ensure the profile has a tool or few installed
-            mcp_run = MCPClient(server_params=os.getenv("MCP_RUN_SSE_URL"))
+            mcp_run = MCPClient(server_params=SseServerParameters(url=os.getenv("MCP_RUN_SSE_URL")))
         except Exception as e:
             logger.error(f"error setting up mcp.run")
             logger.exception("error trace:")

examples/freeze-test/README.md

@@ -0,0 +1,59 @@
+# Freeze Test Client
+
+The purpose of this example is to create an environment for testing the bot and try to create freezing conditions.
+
+### Approach 1: Server-Side Testing with `SimulateFreezeInput`
+
+- Utilize only the bot `freeze_test_bot.py` with the `SimulateFreezeInput` processor. This input continuously injects frames, simulating user speech interruptions at random intervals.
+- This approach excludes the use of input transport and speech-to-text (STT) functionalities.
+
+### Approach 2: Server-Side with TypeScript Client
+
+- Combine server-side operations with a TypeScript client.
+- The client initially records a segment of audio, e.g., 5–10 seconds long. It can be anything.
+- After that, it replays this recorded audio to the server at random intervals, mimicking user input interruptions.
+- This helps testing interruptions in the pipeline as if real users were interacting with the bot.
+
+## Setup
+
+Follow these steps to set up and run the Freeze Test Client:
+
+1. **Run the Bot Server**  
+   - Set up and activate your virtual environment:
+       ```bash
+       python3 -m venv venv
+       source venv/bin/activate  # On Windows: venv\Scripts\activate
+       ```
+
+   - Install dependencies:
+      ```bash
+      pip install -r requirements.txt
+      ```
+
+   - Create your `.env` file and set your env vars:
+      ```bash
+      cp env.example .env
+      ```
+   
+   - Run the server:
+      ```bash
+      python freeze_test_bot.py
+      ```
+
+2. **Navigate to the Client Directory**
+   ```bash
+   cd client
+   ```
+
+3. **Install Dependencies**
+   ```bash
+   npm install
+   ```
+
+4. **Run the Client Application**
+   ```bash
+   npm run dev
+   ```
+
+5. **Access the Client in Your Browser**  
+   Visit [http://localhost:5173](http://localhost:5173) to interact with the Freeze Test Client.

examples/freeze-test/client/index.html

@@ -0,0 +1,43 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>AI Chatbot</title>
+</head>
+
+<body>
+<div class="container">
+  <div class="status-bar">
+    <div class="status">
+      Transport: <span id="connection-status">Disconnected</span>
+    </div>
+    <div class="controls">
+      <button id="connect-btn">Connect</button>
+      <button id="disconnect-btn" disabled>Disconnect</button>
+    </div>
+  </div>
+  <div class="status-bar">
+    <div class="status">
+      Playing audio: <span id="play-audio-status"></span>
+    </div>
+    <div class="controls">
+      <button id="play-btn">Start</button>
+      <button id="stop-btn" disabled>Stop</button>
+    </div>
+  </div>
+
+  <audio id="bot-audio" autoplay></audio>
+
+  <div class="debug-panel">
+    <h3>Debug Info</h3>
+    <div id="debug-log"></div>
+  </div>
+</div>
+
+<script type="module" src="/src/app.ts"></script>
+<link rel="stylesheet" href="/src/style.css">
+</body>
+
+</html>

examples/freeze-test/client/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "client",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc && vite build",
+    "preview": "vite preview"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "description": "",
+  "devDependencies": {
+    "@types/node": "^22.15.30",
+    "@types/protobufjs": "^6.0.0",
+    "@vitejs/plugin-react-swc": "^3.10.1",
+    "typescript": "^5.8.3",
+    "vite": "^6.3.5"
+  },
+  "dependencies": {
+    "@pipecat-ai/client-js": "^0.4.0",
+    "@pipecat-ai/websocket-transport": "^0.4.1",
+    "protobufjs": "^7.4.0"
+  }
+}

examples/freeze-test/client/src/app.ts

@@ -0,0 +1,328 @@
+/**
+ * Copyright (c) 2024–2025, Daily
+ *
+ * SPDX-License-Identifier: BSD 2-Clause License
+ */
+
+/**
+ * RTVI Client Implementation
+ *
+ * This client connects to an RTVI-compatible bot server using WebSocket.
+ *
+ * Requirements:
+ * - A running RTVI bot server (defaults to http://localhost:7860)
+ */
+
+import {
+  RTVIClient,
+  RTVIClientOptions,
+  RTVIEvent,
+} from '@pipecat-ai/client-js';
+import {
+  ProtobufFrameSerializer,
+  WebSocketTransport
+} from "@pipecat-ai/websocket-transport";
+
+class RecordingSerializer extends ProtobufFrameSerializer {
+
+  private lastTimestamp: number | null = null;
+  private recordingAudioToSend: boolean = false;
+  private _recordedAudio: { data: ArrayBuffer; delay: number }[] = [];
+
+  public startRecording() {
+    this.recordingAudioToSend = true;
+    this._recordedAudio = [];
+    this.lastTimestamp = null;
+  }
+
+  public stopRecording() {
+    this.recordingAudioToSend = false;
+  }
+
+  // @ts-ignore
+  serializeAudio(data: ArrayBuffer, sampleRate: number, numChannels: number): Uint8Array | null {
+    if (this.recordingAudioToSend) {
+      const now = Date.now();
+      // Compute delay since last packet
+      const delay = this.lastTimestamp ? now - this.lastTimestamp : 0;
+      this.lastTimestamp = now;
+      // Save audio chunk and delay
+      this._recordedAudio.push({ data, delay });
+      return null;
+    } else {
+      return super.serializeAudio(data, sampleRate, numChannels);
+    }
+  }
+
+  public get recordedAudio() {
+    return this._recordedAudio
+  }
+}
+
+class WebsocketClientApp {
+  private ENABLE_RECORDING_MODE = false
+  private RECORDING_TIME_MS = 10000
+
+  private rtviClient: RTVIClient | null = null;
+  private connectBtn: HTMLButtonElement | null = null;
+  private disconnectBtn: HTMLButtonElement | null = null;
+  private statusSpan: HTMLElement | null = null;
+  private debugLog: HTMLElement | null = null;
+  private botAudio: HTMLAudioElement;
+
+  private declare websocketTransport: WebSocketTransport;
+  private sendRecordedAudio: boolean = false
+  private declare recordingSerializer: RecordingSerializer;
+
+  private playBtn: HTMLButtonElement | null = null;
+  private stopBtn: HTMLButtonElement | null = null;
+
+  constructor() {
+    this.botAudio = document.createElement('audio');
+    this.botAudio.autoplay = true;
+    //this.botAudio.playsInline = true;
+    document.body.appendChild(this.botAudio);
+
+    this.setupDOMElements();
+    this.setupEventListeners();
+  }
+
+  /**
+   * Set up references to DOM elements and create necessary media elements
+   */
+  private setupDOMElements(): void {
+    this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement;
+    this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement;
+    this.statusSpan = document.getElementById('connection-status');
+    this.debugLog = document.getElementById('debug-log');
+    this.playBtn = document.getElementById('play-btn') as HTMLButtonElement;
+    this.stopBtn = document.getElementById('stop-btn') as HTMLButtonElement;
+  }
+
+  /**
+   * Set up event listeners for connect/disconnect buttons
+   */
+  private setupEventListeners(): void {
+    this.connectBtn?.addEventListener('click', () => this.connect());
+    this.disconnectBtn?.addEventListener('click', () => this.disconnect());
+    this.playBtn?.addEventListener('click', () => this.startSendingRecordedAudio());
+    this.stopBtn?.addEventListener('click', () => this.stopSendingRecordedAudio());
+  }
+
+  /**
+   * Add a timestamped message to the debug log
+   */
+  private log(message: string): void {
+    if (!this.debugLog) return;
+    const entry = document.createElement('div');
+    entry.textContent = `${new Date().toISOString()} - ${message}`;
+    if (message.startsWith('User: ')) {
+      entry.style.color = '#2196F3';
+    } else if (message.startsWith('Bot: ')) {
+      entry.style.color = '#4CAF50';
+    }
+    this.debugLog.appendChild(entry);
+    this.debugLog.scrollTop = this.debugLog.scrollHeight;
+    console.log(message);
+  }
+
+  /**
+   * Update the connection status display
+   */
+  private updateStatus(status: string): void {
+    if (this.statusSpan) {
+      this.statusSpan.textContent = status;
+    }
+    this.log(`Status: ${status}`);
+  }
+
+  /**
+   * Check for available media tracks and set them up if present
+   * This is called when the bot is ready or when the transport state changes to ready
+   */
+  setupMediaTracks() {
+    if (!this.rtviClient) return;
+    const tracks = this.rtviClient.tracks();
+    if (tracks.bot?.audio) {
+      this.setupAudioTrack(tracks.bot.audio);
+    }
+  }
+
+  /**
+   * Set up listeners for track events (start/stop)
+   * This handles new tracks being added during the session
+   */
+  setupTrackListeners() {
+    if (!this.rtviClient) return;
+
+    // Listen for new tracks starting
+    this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
+      // Only handle non-local (bot) tracks
+      if (!participant?.local && track.kind === 'audio') {
+        this.setupAudioTrack(track);
+      }
+    });
+
+    // Listen for tracks stopping
+    this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
+      this.log(`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`);
+    });
+  }
+
+  /**
+   * Set up an audio track for playback
+   * Handles both initial setup and track updates
+   */
+  private setupAudioTrack(track: MediaStreamTrack): void {
+    this.log('Setting up audio track');
+    if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
+      const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
+      if (oldTrack?.id === track.id) return;
+    }
+    this.botAudio.srcObject = new MediaStream([track]);
+  }
+
+  /**
+   * Initialize and connect to the bot
+   * This sets up the RTVI client, initializes devices, and establishes the connection
+   */
+  public async connect(): Promise<void> {
+    try {
+      const startTime = Date.now();
+
+      this.recordingSerializer = new RecordingSerializer()
+      const transport = this.ENABLE_RECORDING_MODE ? new WebSocketTransport({serializer: this.recordingSerializer}) : new WebSocketTransport();
+      this.websocketTransport = transport
+
+      const RTVIConfig: RTVIClientOptions = {
+        transport,
+        params: {
+          // The baseURL and endpoint of your bot server that the client will connect to
+          baseUrl: 'http://localhost:7860',
+          endpoints: { connect: '/connect' },
+        },
+        enableMic: true,
+        enableCam: false,
+        callbacks: {
+          onConnected: () => {
+            this.updateStatus('Connected');
+            if (this.connectBtn) this.connectBtn.disabled = true;
+            if (this.disconnectBtn) this.disconnectBtn.disabled = false;
+          },
+          onDisconnected: () => {
+            this.updateStatus('Disconnected');
+            if (this.connectBtn) this.connectBtn.disabled = false;
+            if (this.disconnectBtn) this.disconnectBtn.disabled = true;
+            this.log('Client disconnected');
+          },
+          onBotReady: (data) => {
+            this.log(`Bot ready: ${JSON.stringify(data)}`);
+            this.setupMediaTracks();
+          },
+          onUserTranscript: (data) => {
+            if (data.final) {
+              this.log(`User: ${data.text}`);
+            }
+          },
+          onBotTranscript: (data) => this.log(`Bot: ${data.text}`),
+          onMessageError: (error) => console.error('Message error:', error),
+          onError: (error) => console.error('Error:', error),
+        },
+      }
+      this.rtviClient = new RTVIClient(RTVIConfig);
+      this.setupTrackListeners();
+
+      this.log('Initializing devices...');
+      await this.rtviClient.initDevices();
+
+      this.log('Connecting to bot...');
+      await this.rtviClient.connect();
+
+      const timeTaken = Date.now() - startTime;
+      this.log(`Connection complete, timeTaken: ${timeTaken}`);
+
+      if (this.ENABLE_RECORDING_MODE) {
+        this.log(`Starting to recording the next ${(this.RECORDING_TIME_MS/1000)}s of audio`);
+        this.recordingSerializer.startRecording()
+        await this.sleep(this.RECORDING_TIME_MS)
+        this.recordingSerializer.stopRecording()
+        this.log("Recording stopped");
+        this.rtviClient.enableMic(false)
+        this.startSendingRecordedAudio()
+      }
+    } catch (error) {
+      this.log(`Error connecting: ${(error as Error).message}`);
+      this.updateStatus('Error');
+      // Clean up if there's an error
+      if (this.rtviClient) {
+        try {
+          await this.rtviClient.disconnect();
+        } catch (disconnectError) {
+          this.log(`Error during disconnect: ${disconnectError}`);
+        }
+      }
+    }
+  }
+
+  /**
+   * Disconnect from the bot and clean up media resources
+   */
+  public async disconnect(): Promise<void> {
+    if (this.rtviClient) {
+      try {
+        this.stopSendingRecordedAudio()
+        await this.rtviClient.disconnect();
+        this.rtviClient = null;
+        if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
+          this.botAudio.srcObject.getAudioTracks().forEach((track) => track.stop());
+          this.botAudio.srcObject = null;
+        }
+      } catch (error) {
+        this.log(`Error disconnecting: ${(error as Error).message}`);
+      }
+    }
+  }
+
+  private startSendingRecordedAudio() {
+    this.sendRecordedAudio = true
+    if (this.playBtn) this.playBtn.disabled = true;
+    if (this.stopBtn) this.stopBtn.disabled = false;
+    void this.replayAudio()
+  }
+
+  private stopSendingRecordedAudio() {
+    if (this.stopBtn) this.stopBtn.disabled = true;
+    if (this.playBtn) this.playBtn.disabled = false;
+    this.sendRecordedAudio = false
+  }
+
+  private async replayAudio() {
+    if (this.sendRecordedAudio) {
+      this.log("Sending recorded audio")
+      for (const chunk of this.recordingSerializer.recordedAudio) {
+        await this.sleep(chunk.delay);
+        this.websocketTransport.handleUserAudioStream(chunk.data);
+      }
+      const randomDelay = 1000 + Math.random() * (10000 - 500);
+      await this.sleep(randomDelay);
+
+      void this.replayAudio()
+    }
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+}
+
+declare global {
+  interface Window {
+    WebsocketClientApp: typeof WebsocketClientApp;
+  }
+}
+
+window.addEventListener('DOMContentLoaded', () => {
+  window.WebsocketClientApp = WebsocketClientApp;
+  new WebsocketClientApp();
+});

examples/freeze-test/client/src/style.css

@@ -0,0 +1,98 @@
+body {
+    margin: 0;
+    padding: 20px;
+    font-family: Arial, sans-serif;
+    background-color: #f0f0f0;
+}
+
+.container {
+    max-width: 1200px;
+    margin: 0 auto;
+}
+
+.status-bar {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    padding: 10px;
+    background-color: #fff;
+    border-radius: 8px;
+    margin-bottom: 20px;
+}
+
+.controls button {
+    padding: 8px 16px;
+    margin-left: 10px;
+    border: none;
+    border-radius: 4px;
+    cursor: pointer;
+}
+
+#connect-btn {
+    background-color: #4caf50;
+    color: white;
+}
+
+#disconnect-btn {
+    background-color: #f44336;
+    color: white;
+}
+
+button:disabled {
+    opacity: 0.5;
+    cursor: not-allowed;
+}
+
+.main-content {
+    background-color: #fff;
+    border-radius: 8px;
+    padding: 20px;
+    margin-bottom: 20px;
+}
+
+.bot-container {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+}
+
+#bot-video-container {
+    width: 640px;
+    height: 360px;
+    background-color: #e0e0e0;
+    border-radius: 8px;
+    margin: 20px auto;
+    overflow: hidden;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+}
+
+#bot-video-container video {
+    width: 100%;
+    height: 100%;
+    object-fit: cover;
+}
+
+.debug-panel {
+    background-color: #fff;
+    border-radius: 8px;
+    padding: 20px;
+}
+
+.debug-panel h3 {
+    margin: 0 0 10px 0;
+    font-size: 16px;
+    font-weight: bold;
+}
+
+#debug-log {
+    height: 500px;
+    overflow-y: auto;
+    background-color: #f8f8f8;
+    padding: 10px;
+    border-radius: 4px;
+    font-family: monospace;
+    font-size: 12px;
+    line-height: 1.4;
+}

examples/freeze-test/client/tsconfig.json

@@ -0,0 +1,111 @@
+{
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig to read more about this file */
+
+    /* Projects */
+    // "incremental": true,                              /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
+    // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
+    // "tsBuildInfoFile": "./.tsbuildinfo",              /* Specify the path to .tsbuildinfo incremental compilation file. */
+    // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects. */
+    // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
+    // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
+
+    /* Language and Environment */
+    "target": "es2016",                                  /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
+    // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
+    // "jsx": "preserve",                                /* Specify what JSX code is generated. */
+    // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */
+    // "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
+    // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
+    // "jsxFragmentFactory": "",                         /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
+    // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
+    // "reactNamespace": "",                             /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
+    // "noLib": true,                                    /* Disable including any library files, including the default lib.d.ts. */
+    // "useDefineForClassFields": true,                  /* Emit ECMAScript-standard-compliant class fields. */
+    // "moduleDetection": "auto",                        /* Control what method is used to detect module-format JS files. */
+
+    /* Modules */
+    "module": "commonjs",                                /* Specify what module code is generated. */
+    // "rootDir": "./",                                  /* Specify the root folder within your source files. */
+    // "moduleResolution": "node10",                     /* Specify how TypeScript looks up a file from a given module specifier. */
+    // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
+    // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
+    // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
+    // "typeRoots": [],                                  /* Specify multiple folders that act like './node_modules/@types'. */
+    // "types": [],                                      /* Specify type package names to be included without being referenced in a source file. */
+    // "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
+    // "moduleSuffixes": [],                             /* List of file name suffixes to search when resolving a module. */
+    // "allowImportingTsExtensions": true,               /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
+    // "rewriteRelativeImportExtensions": true,          /* Rewrite '.ts', '.tsx', '.mts', and '.cts' file extensions in relative import paths to their JavaScript equivalent in output files. */
+    // "resolvePackageJsonExports": true,                /* Use the package.json 'exports' field when resolving package imports. */
+    // "resolvePackageJsonImports": true,                /* Use the package.json 'imports' field when resolving imports. */
+    // "customConditions": [],                           /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
+    // "noUncheckedSideEffectImports": true,             /* Check side effect imports. */
+    // "resolveJsonModule": true,                        /* Enable importing .json files. */
+    // "allowArbitraryExtensions": true,                 /* Enable importing files with any extension, provided a declaration file is present. */
+    // "noResolve": true,                                /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
+
+    /* JavaScript Support */
+    // "allowJs": true,                                  /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
+    // "checkJs": true,                                  /* Enable error reporting in type-checked JavaScript files. */
+    // "maxNodeModuleJsDepth": 1,                        /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
+
+    /* Emit */
+    // "declaration": true,                              /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
+    // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
+    // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
+    // "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
+    // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */
+    // "noEmit": true,                                   /* Disable emitting files from a compilation. */
+    // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
+    // "outDir": "./",                                   /* Specify an output folder for all emitted files. */
+    // "removeComments": true,                           /* Disable emitting comments. */
+    // "importHelpers": true,                            /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
+    // "downlevelIteration": true,                       /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
+    // "sourceRoot": "",                                 /* Specify the root path for debuggers to find the reference source code. */
+    // "mapRoot": "",                                    /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSources": true,                            /* Include source code in the sourcemaps inside the emitted JavaScript. */
+    // "emitBOM": true,                                  /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
+    // "newLine": "crlf",                                /* Set the newline character for emitting files. */
+    // "stripInternal": true,                            /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
+    // "noEmitHelpers": true,                            /* Disable generating custom helper functions like '__extends' in compiled output. */
+    // "noEmitOnError": true,                            /* Disable emitting files if any type checking errors are reported. */
+    // "preserveConstEnums": true,                       /* Disable erasing 'const enum' declarations in generated code. */
+    // "declarationDir": "./",                           /* Specify the output directory for generated declaration files. */
+
+    /* Interop Constraints */
+    // "isolatedModules": true,                          /* Ensure that each file can be safely transpiled without relying on other imports. */
+    // "verbatimModuleSyntax": true,                     /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
+    // "isolatedDeclarations": true,                     /* Require sufficient annotation on exports so other tools can trivially generate declaration files. */
+    // "allowSyntheticDefaultImports": true,             /* Allow 'import x from y' when a module doesn't have a default export. */
+    "esModuleInterop": true,                             /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
+    // "preserveSymlinks": true,                         /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
+    "forceConsistentCasingInFileNames": true,            /* Ensure that casing is correct in imports. */
+
+    /* Type Checking */
+    "strict": true,                                      /* Enable all strict type-checking options. */
+    // "noImplicitAny": true,                            /* Enable error reporting for expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,                         /* When type checking, take into account 'null' and 'undefined'. */
+    // "strictFunctionTypes": true,                      /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
+    // "strictBindCallApply": true,                      /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
+    // "strictPropertyInitialization": true,             /* Check for class properties that are declared but not set in the constructor. */
+    // "strictBuiltinIteratorReturn": true,              /* Built-in iterators are instantiated with a 'TReturn' type of 'undefined' instead of 'any'. */
+    // "noImplicitThis": true,                           /* Enable error reporting when 'this' is given the type 'any'. */
+    // "useUnknownInCatchVariables": true,               /* Default catch clause variables as 'unknown' instead of 'any'. */
+    // "alwaysStrict": true,                             /* Ensure 'use strict' is always emitted. */
+    // "noUnusedLocals": true,                           /* Enable error reporting when local variables aren't read. */
+    // "noUnusedParameters": true,                       /* Raise an error when a function parameter isn't read. */
+    // "exactOptionalPropertyTypes": true,               /* Interpret optional property types as written, rather than adding 'undefined'. */
+    // "noImplicitReturns": true,                        /* Enable error reporting for codepaths that do not explicitly return in a function. */
+    // "noFallthroughCasesInSwitch": true,               /* Enable error reporting for fallthrough cases in switch statements. */
+    // "noUncheckedIndexedAccess": true,                 /* Add 'undefined' to a type when accessed using an index. */
+    // "noImplicitOverride": true,                       /* Ensure overriding members in derived classes are marked with an override modifier. */
+    // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type. */
+    // "allowUnusedLabels": true,                        /* Disable error reporting for unused labels. */
+    // "allowUnreachableCode": true,                     /* Disable error reporting for unreachable code. */
+
+    /* Completeness */
+    // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
+    "skipLibCheck": true                                 /* Skip type checking all .d.ts files. */
+  }
+}

examples/freeze-test/client/vite.config.js

@@ -0,0 +1,15 @@
+import { defineConfig } from 'vite';
+import react from '@vitejs/plugin-react-swc';
+
+export default defineConfig({
+    plugins: [react()],
+    server: {
+        proxy: {
+            // Proxy /api requests to the backend server
+            '/connect': {
+                target: 'http://0.0.0.0:7860', // Replace with your backend URL
+                changeOrigin: true,
+            },
+        },
+    },
+});

examples/freeze-test/freeze_test_bot.py

@@ -0,0 +1,312 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import argparse
+import asyncio
+import os
+import random
+from contextlib import asynccontextmanager
+from typing import Any, Dict
+
+import uvicorn
+from dotenv import load_dotenv
+from fastapi import FastAPI, Request, WebSocket
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import RedirectResponse
+from loguru import logger
+from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI
+
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.frames.frames import (
+    CancelFrame,
+    EndFrame,
+    Frame,
+    InterimTranscriptionFrame,
+    LLMFullResponseEndFrame,
+    LLMTextFrame,
+    StartFrame,
+    StartInterruptionFrame,
+    StopFrame,
+    StopInterruptionFrame,
+    TranscriptionFrame,
+    TTSTextFrame,
+    UserStartedSpeakingFrame,
+    UserStoppedSpeakingFrame,
+)
+from pipecat.observers.loggers.debug_log_observer import DebugLogObserver
+from pipecat.pipeline.parallel_pipeline import ParallelPipeline
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import (
+    OpenAILLMContext,
+    OpenAILLMContextFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIProcessor
+from pipecat.serializers.protobuf import ProtobufFrameSerializer
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.deepgram import DeepgramSTTService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.network.fastapi_websocket import (
+    FastAPIWebsocketParams,
+    FastAPIWebsocketTransport,
+)
+from pipecat.utils.time import time_now_iso8601
+
+load_dotenv(override=True)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Handles FastAPI startup and shutdown."""
+    yield  # Run app
+
+
+# Initialize FastAPI app with lifespan manager
+app = FastAPI(lifespan=lifespan)
+
+# Configure CORS to allow requests from any origin
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Mount the frontend at /
+app.mount("/client", SmallWebRTCPrebuiltUI)
+
+
+class SimulateFreezeInput(FrameProcessor):
+    def __init__(
+        self,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        # Whether we have seen a StartFrame already.
+        self._initialized = False
+        self._send_frames_task = None
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+        if isinstance(frame, StartFrame):
+            # Push StartFrame before start(), because we want StartFrame to be
+            # processed by every processor before any other frame is processed.
+            await self.push_frame(frame, direction)
+            await self._start(frame)
+        elif isinstance(frame, CancelFrame):
+            logger.info("SimulateFreezeInput: Received cancel frame")
+            await self._stop()
+            await self.push_frame(frame, direction)
+        elif isinstance(frame, EndFrame):
+            logger.info("SimulateFreezeInput: Received end frame")
+            await self.push_frame(frame, direction)
+            await self._stop()
+        elif isinstance(frame, StopFrame):
+            logger.info("SimulateFreezeInput: Received stop frame")
+            await self.push_frame(frame, direction)
+            await self._stop()
+
+    async def _start(self, frame: StartFrame):
+        if self._initialized:
+            return
+        logger.info(f"Starting SimulateFreezeInput")
+        self._initialized = True
+        if not self._send_frames_task:
+            self._send_frames_task = self.create_task(self._send_frames())
+
+    async def _stop(self):
+        logger.info(f"Stopping SimulateFreezeInput")
+        self._initialized = False
+        if self._send_frames_task:
+            await self.cancel_task(self._send_frames_task)
+            self._send_frames_task = None
+
+    async def _send_user_text(self, text: str):
+        # Emulation as if the user has spoken and the stt transcribed
+        await self.push_frame(UserStartedSpeakingFrame())
+        await self.push_frame(StartInterruptionFrame())
+        await self.push_frame(
+            TranscriptionFrame(
+                text,
+                "",
+                time_now_iso8601(),
+            )
+        )
+        # Need to wait before sending the UserStoppedSpeakingFrame,
+        # otherwise TranscriptionFrame will be processed
+        # later than the UserStoppedSpeakingFrame
+        await asyncio.sleep(0.1)
+        await self.push_frame(UserStoppedSpeakingFrame())
+        await self.push_frame(StopInterruptionFrame())
+
+    async def _send_frames(self):
+        try:
+            i = 0
+            while True:
+                logger.debug("SimulateFreezeInput _send_frames")
+                await self._send_user_text("Tell me a brief history of Brazil!")
+                await asyncio.sleep(3)
+                await self._send_user_text("")
+                break
+                # i += 1
+                # if i >= 5:
+                #    break
+                # sleeping 1s before interrupting
+                # wait_time = random.uniform(1, 10)
+                # await asyncio.sleep(wait_time)
+        except Exception as e:
+            logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
+
+
+async def run_example(websocket_client):
+    logger.info(f"Starting bot")
+
+    # Create a transport using the WebRTC connection
+    transport = FastAPIWebsocketTransport(
+        websocket=websocket_client,
+        params=FastAPIWebsocketParams(
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+            add_wav_header=False,
+            vad_analyzer=SileroVADAnalyzer(),
+            serializer=ProtobufFrameSerializer(),
+        ),
+    )
+
+    freeze = SimulateFreezeInput()
+
+    stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
+
+    tts = CartesiaTTSService(
+        api_key=os.getenv("CARTESIA_API_KEY"),
+        voice_id="71a7ad14-091c-4e8e-a314-022ece01c121",  # British Reading Lady
+    )
+
+    llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
+
+    rtvi = RTVIProcessor(config=RTVIConfig(config=[]))
+
+    messages = [
+        {
+            "role": "system",
+            "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
+        },
+    ]
+
+    context = OpenAILLMContext(messages)
+    context_aggregator = llm.create_context_aggregator(context)
+
+    pipeline = Pipeline(
+        [
+            ParallelPipeline(
+                [
+                    freeze,
+                ],
+                [
+                    transport.input(),
+                    stt,
+                ],
+            ),
+            rtvi,
+            context_aggregator.user(),  # User responses
+            llm,  # LLM
+            tts,  # TTS
+            transport.output(),  # Transport bot output
+            context_aggregator.assistant(),  # Assistant spoken responses
+        ]
+    )
+
+    task = PipelineTask(
+        pipeline,
+        params=PipelineParams(
+            allow_interruptions=True,
+            enable_metrics=True,
+            enable_usage_metrics=True,
+            report_only_initial_ttfb=True,
+        ),
+        idle_timeout_secs=120,
+        observers=[
+            DebugLogObserver(
+                frame_types={
+                    InterimTranscriptionFrame: None,
+                    TranscriptionFrame: None,
+                    # TTSTextFrame: None,
+                    # LLMTextFrame: None,
+                    OpenAILLMContextFrame: None,
+                    LLMFullResponseEndFrame: None,
+                },
+                exclude_fields={
+                    "result",
+                    "metadata",
+                    "audio",
+                    "image",
+                    "images",
+                },
+            ),
+        ],
+    )
+
+    @transport.event_handler("on_client_connected")
+    async def on_client_connected(transport, client):
+        logger.info(f"Client connected")
+
+    @rtvi.event_handler("on_client_ready")
+    async def on_client_ready(rtvi):
+        logger.info(f"Client ready")
+        await rtvi.set_bot_ready()
+        # Kick off the conversation.
+        # messages.append({"role": "system", "content": "Please introduce yourself to the user."})
+        # await task.queue_frames([context_aggregator.user().get_context_frame()])
+
+    @transport.event_handler("on_client_disconnected")
+    async def on_client_disconnected(transport, client):
+        logger.info(f"Client disconnected")
+        await task.cancel()
+
+    runner = PipelineRunner(handle_sigint=False)
+
+    await runner.run(task)
+
+
+@app.get("/", include_in_schema=False)
+async def root_redirect():
+    return RedirectResponse(url="/client/")
+
+
+@app.websocket("/ws")
+async def websocket_endpoint(websocket: WebSocket):
+    await websocket.accept()
+    print("WebSocket connection accepted")
+    try:
+        await run_example(websocket)
+    except Exception as e:
+        print(f"Exception in run_bot: {e}")
+
+
+@app.post("/connect")
+async def bot_connect(request: Request) -> Dict[Any, Any]:
+    server_mode = os.getenv("WEBSOCKET_SERVER", "fast_api")
+    if server_mode == "websocket_server":
+        ws_url = "ws://localhost:8765"
+    else:
+        ws_url = "ws://localhost:7860/ws"
+    return {"ws_url": ws_url}
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Pipecat Bot Runner")
+    parser.add_argument(
+        "--host", default="localhost", help="Host for HTTP server (default: localhost)"
+    )
+    parser.add_argument(
+        "--port", type=int, default=7860, help="Port for HTTP server (default: 7860)"
+    )
+    args = parser.parse_args()
+
+    uvicorn.run(app, host=args.host, port=args.port)

examples/moondream-chatbot/bot.py

@@ -143,6 +143,7 @@ async def main():
             DailyParams(
                 audio_in_enabled=True,
                 audio_out_enabled=True,
+                video_in_enabled=True,
                 video_out_enabled=True,
                 video_out_width=1024,
                 video_out_height=576,

examples/sentry-metrics/bot.py

@@ -49,7 +49,7 @@ async def main():
 
         # Initialize Sentry
         sentry_sdk.init(
-            dsn="your-project-dsn",
+            dsn=os.getenv("SENTRY_DSN"),
             traces_sample_rate=1.0,
         )
 

src/pipecat/audio/turn/base_turn_analyzer.py

@@ -78,3 +78,8 @@ class BaseTurnAnalyzer(ABC):
             EndOfTurnState: The result of the end of turn analysis.
         """
         pass
+
+    @abstractmethod
+    def clear(self):
+        """Reset the turn analyzer to its initial state."""
+        pass

src/pipecat/audio/turn/smart_turn/base_smart_turn.py

@@ -98,6 +98,9 @@ class BaseSmartTurn(BaseTurnAnalyzer):
         logger.debug(f"End of Turn result: {state}")
         return state, result
 
+    def clear(self):
+        self._clear(EndOfTurnState.COMPLETE)
+
     def _clear(self, turn_state: EndOfTurnState):
         # If the state is still incomplete, keep the _speech_triggered as True
         self._speech_triggered = turn_state == EndOfTurnState.INCOMPLETE

src/pipecat/frames/frames.py

@@ -7,6 +7,7 @@
 from dataclasses import dataclass, field
 from enum import Enum
 from typing import (
+    TYPE_CHECKING,
     Any,
     Awaitable,
     Callable,
@@ -26,6 +27,9 @@ from pipecat.transcriptions.language import Language
 from pipecat.utils.time import nanoseconds_to_str
 from pipecat.utils.utils import obj_count, obj_id
 
+if TYPE_CHECKING:
+    from pipecat.processors.frame_processor import FrameProcessor
+
 
 class KeypadEntry(str, Enum):
     """DTMF entries."""
@@ -485,16 +489,6 @@ class FatalErrorFrame(ErrorFrame):
     fatal: bool = field(default=True, init=False)
 
 
-@dataclass
-class HeartbeatFrame(SystemFrame):
-    """This frame is used by the pipeline task as a mechanism to know if the
-    pipeline is running properly.
-
-    """
-
-    timestamp: int
-
-
 @dataclass
 class EndTaskFrame(SystemFrame):
     """This is used to notify the pipeline task that the pipeline should be
@@ -529,25 +523,25 @@ class StopTaskFrame(SystemFrame):
 
 @dataclass
 class FrameProcessorPauseUrgentFrame(SystemFrame):
-    """This processor is used to pause frame processing for the given processor
+    """This frame is used to pause frame processing for the given processor as
-    as fast as possible. Pausing frame processing will keep frames in the
+    fast as possible. Pausing frame processing will keep frames in the internal
-    internal queue which will then be processed when frame processing is resumed
+    queue which will then be processed when frame processing is resumed with
-    with `FrameProcessorResumeFrame`.
+    `FrameProcessorResumeFrame`.
 
     """
 
-    processor: str
+    processor: "FrameProcessor"
 
 
 @dataclass
 class FrameProcessorResumeUrgentFrame(SystemFrame):
-    """This processor is used to resume frame processing for the given processor
+    """This frame is used to resume frame processing for the given processor
     if it was previously paused as fast as possible. After resuming frame
     processing all queued frames will be processed in the order received.
 
     """
 
-    processor: str
+    processor: "FrameProcessor"
 
 
 @dataclass
@@ -877,25 +871,37 @@ class StopFrame(ControlFrame):
     pass
 
 
+@dataclass
+class HeartbeatFrame(ControlFrame):
+    """This frame is used by the pipeline task as a mechanism to know if the
+    pipeline is running properly.
+
+    """
+
+    timestamp: int
+
+
 @dataclass
 class FrameProcessorPauseFrame(ControlFrame):
-    """This processor is used to pause frame processing for the given
+    """This frame is used to pause frame processing for the given
     processor. Pausing frame processing will keep frames in the internal queue
     which will then be processed when frame processing is resumed with
-    `FrameProcessorResumeFrame`."""
+    `FrameProcessorResumeFrame`.
 
-    processor: str
+    """
+
+    processor: "FrameProcessor"
 
 
 @dataclass
 class FrameProcessorResumeFrame(ControlFrame):
-    """This processor is used to resume frame processing for the given processor
+    """This frame is used to resume frame processing for the given processor if
-    if it was previously paused. After resuming frame processing all queued
+    it was previously paused. After resuming frame processing all queued frames
-    frames will be processed in the order received.
+    will be processed in the order received.
 
     """
 
-    processor: str
+    processor: "FrameProcessor"
 
 
 @dataclass

src/pipecat/pipeline/base_task.py

@@ -6,18 +6,21 @@
 
 import asyncio
 from abc import abstractmethod
+from dataclasses import dataclass
 from typing import AsyncIterable, Iterable
 
 from pipecat.frames.frames import Frame
 from pipecat.utils.base_object import BaseObject
 
 
-class BaseTask(BaseObject):
+@dataclass
-    @abstractmethod
+class PipelineTaskParams:
-    def set_event_loop(self, loop: asyncio.AbstractEventLoop):
+    """Specific configuration for the pipeline task."""
-        """Sets the event loop that this task will run on."""
-        pass
 
+    loop: asyncio.AbstractEventLoop
+
+
+class BasePipelineTask(BaseObject):
     @abstractmethod
     def has_finished(self) -> bool:
         """Indicates whether the tasks has finished. That is, all processors
@@ -40,7 +43,7 @@ class BaseTask(BaseObject):
         pass
 
     @abstractmethod
-    async def run(self):
+    async def run(self, params: PipelineTaskParams):
         """Starts running the given pipeline."""
         pass
 

src/pipecat/pipeline/parallel_pipeline.py

@@ -202,14 +202,18 @@ class ParallelPipeline(BasePipeline):
     async def _process_up_queue(self):
         while True:
             frame = await self._up_queue.get()
+            self.start_watchdog()
             await self._parallel_push_frame(frame, FrameDirection.UPSTREAM)
             self._up_queue.task_done()
+            self.reset_watchdog()
 
     async def _process_down_queue(self):
         running = True
         while running:
             frame = await self._down_queue.get()
 
+            self.start_watchdog()
+
             endframe_counter = self._endframe_counter.get(frame.id, 0)
 
             # If we have a counter, decrement it.
@@ -224,3 +228,5 @@ class ParallelPipeline(BasePipeline):
             running = not (endframe_counter == 0 and isinstance(frame, EndFrame))
 
             self._down_queue.task_done()
+
+            self.reset_watchdog()

src/pipecat/pipeline/runner.py

@@ -11,6 +11,7 @@ from typing import Optional
 
 from loguru import logger
 
+from pipecat.pipeline.base_task import PipelineTaskParams
 from pipecat.pipeline.task import PipelineTask
 from pipecat.utils.base_object import BaseObject
 
@@ -37,8 +38,8 @@ class PipelineRunner(BaseObject):
     async def run(self, task: PipelineTask):
         logger.debug(f"Runner {self} started running {task}")
         self._tasks[task.name] = task
-        task.set_event_loop(self._loop)
+        params = PipelineTaskParams(loop=self._loop)
-        await task.run()
+        await task.run(params)
         del self._tasks[task.name]
 
         # Cleanup base object.

src/pipecat/pipeline/task.py

@@ -6,7 +6,8 @@
 
 import asyncio
 import time
-from typing import Any, AsyncIterable, Dict, Iterable, List, Optional, Sequence, Tuple, Type
+from collections import deque
+from typing import Any, AsyncIterable, Deque, Dict, Iterable, List, Optional, Tuple, Type
 
 from loguru import logger
 from pydantic import BaseModel, ConfigDict, Field
@@ -23,6 +24,7 @@ from pipecat.frames.frames import (
     ErrorFrame,
     Frame,
     HeartbeatFrame,
+    InputAudioRawFrame,
     LLMFullResponseEndFrame,
     MetricsFrame,
     StartFrame,
@@ -33,19 +35,22 @@ from pipecat.metrics.metrics import ProcessingMetricsData, TTFBMetricsData
 from pipecat.observers.base_observer import BaseObserver
 from pipecat.observers.turn_tracking_observer import TurnTrackingObserver
 from pipecat.pipeline.base_pipeline import BasePipeline
-from pipecat.pipeline.base_task import BaseTask
+from pipecat.pipeline.base_task import BasePipelineTask, PipelineTaskParams
 from pipecat.pipeline.task_observer import TaskObserver
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
-from pipecat.utils.asyncio import BaseTaskManager, TaskManager
+from pipecat.utils.asyncio import WATCHDOG_TIMEOUT, BaseTaskManager, TaskManager, TaskManagerParams
 from pipecat.utils.tracing.setup import is_tracing_available
 from pipecat.utils.tracing.turn_trace_observer import TurnTraceObserver
 
 HEARTBEAT_SECONDS = 1.0
-HEARTBEAT_MONITOR_SECONDS = HEARTBEAT_SECONDS * 5
+HEARTBEAT_MONITOR_SECONDS = HEARTBEAT_SECONDS * 10
 
 
 class PipelineParams(BaseModel):
-    """Configuration parameters for pipeline execution.
+    """Configuration parameters for pipeline execution. These parameters are
+    usually passed to all frame processors using through `StartFrame`. For other
+    generic pipeline task parameters use `PipelineTask` constructor arguments
+    instead.
 
     Attributes:
         allow_interruptions: Whether to allow pipeline interruptions.
@@ -60,6 +65,7 @@ class PipelineParams(BaseModel):
         send_initial_empty_metrics: Whether to send initial empty metrics.
         start_metadata: Additional metadata for pipeline start.
         interruption_strategies: Strategies for bot interruption behavior.
+
     """
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -71,11 +77,11 @@ class PipelineParams(BaseModel):
     enable_metrics: bool = False
     enable_usage_metrics: bool = False
     heartbeats_period_secs: float = HEARTBEAT_SECONDS
+    interruption_strategies: List[BaseInterruptionStrategy] = Field(default_factory=list)
     observers: List[BaseObserver] = Field(default_factory=list)
     report_only_initial_ttfb: bool = False
     send_initial_empty_metrics: bool = True
     start_metadata: Dict[str, Any] = Field(default_factory=dict)
-    interruption_strategies: List[BaseInterruptionStrategy] = Field(default_factory=list)
 
 
 class PipelineTaskSource(FrameProcessor):
@@ -125,7 +131,7 @@ class PipelineTaskSink(FrameProcessor):
         await self._down_queue.put(frame)
 
 
-class PipelineTask(BaseTask):
+class PipelineTask(BasePipelineTask):
     """Manages the execution of a pipeline, handling frame processing and task lifecycle.
 
     It has a couple of event handlers `on_frame_reached_upstream` and
@@ -172,21 +178,24 @@ class PipelineTask(BaseTask):
     Args:
         pipeline: The pipeline to execute.
         params: Configuration parameters for the pipeline.
-        observers: List of observers for monitoring pipeline execution.
+        additional_span_attributes: Optional dictionary of attributes to propagate as
-        clock: Clock implementation for timing operations.
+            OpenTelemetry conversation span attributes.
+        cancel_on_idle_timeout: Whether the pipeline task should be cancelled if
+            the idle timeout is reached.
         check_dangling_tasks: Whether to check for processors' tasks finishing properly.
+        clock: Clock implementation for timing operations.
+        conversation_id: Optional custom ID for the conversation.
+        enable_tracing: Whether to enable tracing.
+        enable_turn_tracking: Whether to enable turn tracking.
+        enable_watchdog_logging: Whether to print task processing times.
+        idle_timeout_frames: A tuple with the frames that should trigger an idle
+            timeout if not received withing `idle_timeout_seconds`.
         idle_timeout_secs: Timeout (in seconds) to consider pipeline idle or
             None. If a pipeline is idle the pipeline task will be cancelled
             automatically.
-        idle_timeout_frames: A tuple with the frames that should trigger an idle
+        observers: List of observers for monitoring pipeline execution.
-            timeout if not received withing `idle_timeout_seconds`.
+        watchdog_timeout_secs: Watchdog timer timeout (in seconds). A warning
-        cancel_on_idle_timeout: Whether the pipeline task should be cancelled if
+            will be logged if the watchdog timer is not reset before this timeout.
-            the idle timeout is reached.
-        enable_turn_tracking: Whether to enable turn tracking.
-        enable_turn_tracing: Whether to enable turn tracing.
-        conversation_id: Optional custom ID for the conversation.
-        additional_span_attributes: Optional dictionary of attributes to propagate as
-            OpenTelemetry conversation span attributes.
     """
 
     def __init__(
@@ -194,33 +203,37 @@ class PipelineTask(BaseTask):
         pipeline: BasePipeline,
         *,
         params: Optional[PipelineParams] = None,
-        observers: Optional[List[BaseObserver]] = None,
+        additional_span_attributes: Optional[dict] = None,
-        clock: Optional[BaseClock] = None,
+        cancel_on_idle_timeout: bool = True,
-        task_manager: Optional[BaseTaskManager] = None,
         check_dangling_tasks: bool = True,
-        idle_timeout_secs: Optional[float] = 300,
+        clock: Optional[BaseClock] = None,
+        conversation_id: Optional[str] = None,
+        enable_tracing: bool = False,
+        enable_turn_tracking: bool = True,
+        enable_watchdog_logging: bool = False,
         idle_timeout_frames: Tuple[Type[Frame], ...] = (
             BotSpeakingFrame,
             LLMFullResponseEndFrame,
         ),
-        cancel_on_idle_timeout: bool = True,
+        idle_timeout_secs: Optional[float] = 300,
-        enable_turn_tracking: bool = True,
+        observers: Optional[List[BaseObserver]] = None,
-        enable_tracing: bool = False,
+        task_manager: Optional[BaseTaskManager] = None,
-        conversation_id: Optional[str] = None,
+        watchdog_timeout_secs: float = WATCHDOG_TIMEOUT,
-        additional_span_attributes: Optional[dict] = None,
     ):
         super().__init__()
         self._pipeline = pipeline
-        self._clock = clock or SystemClock()
         self._params = params or PipelineParams()
-        self._check_dangling_tasks = check_dangling_tasks
-        self._idle_timeout_secs = idle_timeout_secs
-        self._idle_timeout_frames = idle_timeout_frames
-        self._cancel_on_idle_timeout = cancel_on_idle_timeout
-        self._enable_turn_tracking = enable_turn_tracking
-        self._enable_tracing = enable_tracing and is_tracing_available()
-        self._conversation_id = conversation_id
         self._additional_span_attributes = additional_span_attributes or {}
+        self._cancel_on_idle_timeout = cancel_on_idle_timeout
+        self._check_dangling_tasks = check_dangling_tasks
+        self._clock = clock or SystemClock()
+        self._conversation_id = conversation_id
+        self._enable_tracing = enable_tracing and is_tracing_available()
+        self._enable_turn_tracking = enable_turn_tracking
+        self._enable_watchdog_logging = enable_watchdog_logging
+        self._idle_timeout_frames = idle_timeout_frames
+        self._idle_timeout_secs = idle_timeout_secs
+        self._watchdog_timeout_secs = watchdog_timeout_secs
         if self._params.observers:
             import warnings
 
@@ -322,9 +335,6 @@ class PipelineTask(BaseTask):
     async def remove_observer(self, observer: BaseObserver):
         await self._observer.remove_observer(observer)
 
-    def set_event_loop(self, loop: asyncio.AbstractEventLoop):
-        self._task_manager.set_event_loop(loop)
-
     def set_reached_upstream_filter(self, types: Tuple[Type[Frame], ...]):
         """Sets which frames will be checked before calling the
         on_frame_reached_upstream event handler.
@@ -358,14 +368,14 @@ class PipelineTask(BaseTask):
         """Stops the running pipeline immediately."""
         await self._cancel()
 
-    async def run(self):
+    async def run(self, params: PipelineTaskParams):
         """Starts and manages the pipeline execution until completion or cancellation."""
         if self.has_finished():
             return
         cleanup_pipeline = True
         try:
             # Setup processors.
-            await self._setup()
+            await self._setup(params)
 
             # Create all main tasks and wait of the main push task. This is the
             # task that pushes frames to the very beginning of our pipeline (our
@@ -485,7 +495,14 @@ class PipelineTask(BaseTask):
         await self._pipeline_end_event.wait()
         self._pipeline_end_event.clear()
 
-    async def _setup(self):
+    async def _setup(self, params: PipelineTaskParams):
+        mgr_params = TaskManagerParams(
+            loop=params.loop,
+            enable_watchdog_logging=self._enable_watchdog_logging,
+            watchdog_timeout=self._watchdog_timeout_secs,
+        )
+        self._task_manager.setup(mgr_params)
+
         setup = FrameProcessorSetup(
             clock=self._clock,
             task_manager=self._task_manager,
@@ -509,6 +526,8 @@ class PipelineTask(BaseTask):
             await self._pipeline.cleanup()
         await self._sink.cleanup()
 
+        await self._task_manager.cleanup()
+
     async def _process_push_queue(self):
         """This is the task that runs the pipeline for the first time by sending
         a StartFrame and by pushing any other frames queued by the user. It runs
@@ -646,12 +665,17 @@ class PipelineTask(BaseTask):
         """
         running = True
         last_frame_time = 0
+        frame_buffer = deque(maxlen=10)  # Store last 10 frames
+
         while running:
             try:
                 frame = await asyncio.wait_for(
                     self._idle_queue.get(), timeout=self._idle_timeout_secs
                 )
 
+                if not isinstance(frame, InputAudioRawFrame):
+                    frame_buffer.append(frame)
+
                 if isinstance(frame, StartFrame) or isinstance(frame, self._idle_timeout_frames):
                     # If we find a StartFrame or one of the frames that prevents a
                     # time out we update the time.
@@ -662,7 +686,7 @@ class PipelineTask(BaseTask):
                     # valid frames.
                     diff_time = time.time() - last_frame_time
                     if diff_time >= self._idle_timeout_secs:
-                        running = await self._idle_timeout_detected()
+                        running = await self._idle_timeout_detected(frame_buffer)
                         # Reset `last_frame_time` so we don't trigger another
                         # immediate idle timeout if we are not cancelling. For
                         # example, we might want to force the bot to say goodbye
@@ -670,15 +694,20 @@ class PipelineTask(BaseTask):
                         last_frame_time = time.time()
 
                 self._idle_queue.task_done()
-            except asyncio.TimeoutError:
-                running = await self._idle_timeout_detected()
 
-    async def _idle_timeout_detected(self) -> bool:
+            except asyncio.TimeoutError:
+                running = await self._idle_timeout_detected(frame_buffer)
+
+    async def _idle_timeout_detected(self, last_frames: Deque[Frame]) -> bool:
         """Logic for when the pipeline is idle.
 
         Returns:
             bool: Whther the pipeline task is being cancelled or not.
         """
+        logger.warning("Idle timeout detected. Last 10 frames received:")
+        for i, frame in enumerate(last_frames, 1):
+            logger.warning(f"Frame {i}: {frame}")
+
         await self._call_event_handler("on_idle_timeout")
         if self._cancel_on_idle_timeout:
             logger.warning(f"Idle pipeline detected, cancelling pipeline task...")

src/pipecat/processors/aggregators/llm_response.py

@@ -266,6 +266,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
 
         self._user_speaking = False
         self._bot_speaking = False
+        self._was_bot_speaking = False
         self._emulating_vad = False
         self._seen_interim_results = False
         self._waiting_for_aggregation = False
@@ -275,6 +276,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
 
     async def reset(self):
         await super().reset()
+        self._was_bot_speaking = False
         self._seen_interim_results = False
         self._waiting_for_aggregation = False
         [await s.reset() for s in self._interruption_strategies]
@@ -355,6 +357,20 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
             else:
                 # No interruption config - normal behavior (always push aggregation)
                 await self._process_aggregation()
+        # Handles the case where both the user and the bot are not speaking,
+        # and the bot was previously speaking before the user interruption.
+        # Normally, when the user stops speaking, new text is expected,
+        # which triggers the bot to respond. However, if no new text
+        # is received, this safeguard ensures
+        # the bot doesn't hang indefinitely while waiting to speak again.
+        elif not self._seen_interim_results and self._was_bot_speaking and not self._bot_speaking:
+            logger.warning(
+                "User stopped speaking but no new aggregation received. Forcing aggregation processing to resume bot response."
+            )
+            # Resetting it so we don't trigger this twice
+            self._was_bot_speaking = False
+            # We are just pushing the same previous context to be processed again in this case
+            await self.push_frame(OpenAILLMContextFrame(self._context))
 
     async def _should_interrupt_based_on_strategies(self) -> bool:
         """Check if interruption should occur based on configured strategies."""
@@ -381,6 +397,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
     async def _handle_user_started_speaking(self, frame: UserStartedSpeakingFrame):
         self._user_speaking = True
         self._waiting_for_aggregation = True
+        self._was_bot_speaking = self._bot_speaking
 
         # If we get a non-emulated UserStartedSpeakingFrame but we are in the
         # middle of emulating VAD, let's stop emulating VAD (i.e. don't send the
@@ -393,7 +410,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
         # We just stopped speaking. Let's see if there's some aggregation to
         # push. If the last thing we saw is an interim transcription, let's wait
         # pushing the aggregation as we will probably get a final transcription.
-        if not self._seen_interim_results:
+        if len(self._aggregation) > 0 and not self._seen_interim_results:
             await self.push_aggregation()
 
     async def _handle_bot_started_speaking(self, _: BotStartedSpeakingFrame):

src/pipecat/processors/consumer_processor.py

@@ -61,5 +61,7 @@ class ConsumerProcessor(FrameProcessor):
     async def _consumer_task_handler(self):
         while True:
             frame = await self._queue.get()
+            self.start_watchdog()
             new_frame = await self._transformer(frame)
             await self.push_frame(new_frame, self._direction)
+            self.reset_watchdog()

src/pipecat/processors/frame_processor.py

@@ -51,6 +51,8 @@ class FrameProcessor(BaseObject):
         *,
         name: Optional[str] = None,
         metrics: Optional[FrameProcessorMetrics] = None,
+        enable_watchdog_logging: Optional[bool] = None,
+        watchdog_timeout_secs: Optional[float] = None,
         **kwargs,
     ):
         super().__init__(name=name)
@@ -58,6 +60,12 @@ class FrameProcessor(BaseObject):
         self._prev: Optional["FrameProcessor"] = None
         self._next: Optional["FrameProcessor"] = None
 
+        # Enable watchdog logging for all tasks created by this frame processor.
+        self._enable_watchdog_logging = enable_watchdog_logging
+
+        # Allow this frame processor to control their tasks timeout.
+        self._watchdog_timeout = watchdog_timeout_secs
+
         # Clock
         self._clock: Optional[BaseClock] = None
 
@@ -171,24 +179,42 @@ class FrameProcessor(BaseObject):
         await self.stop_ttfb_metrics()
         await self.stop_processing_metrics()
 
-    def create_task(self, coroutine: Coroutine, name: Optional[str] = None) -> asyncio.Task:
+    def create_task(
-        if not self._task_manager:
+        self,
-            raise Exception(f"{self} TaskManager is still not initialized.")
+        coroutine: Coroutine,
+        name: Optional[str] = None,
+        *,
+        enable_watchdog_logging: Optional[bool] = None,
+        watchdog_timeout_secs: Optional[float] = None,
+    ) -> asyncio.Task:
         if name:
             name = f"{self}::{name}"
         else:
             name = f"{self}::{coroutine.cr_code.co_name}"
-        return self._task_manager.create_task(coroutine, name)
+        return self.get_task_manager().create_task(
+            coroutine,
+            name,
+            enable_watchdog_logging=(
+                enable_watchdog_logging
+                if enable_watchdog_logging
+                else self._enable_watchdog_logging
+            ),
+            watchdog_timeout=(
+                watchdog_timeout_secs if watchdog_timeout_secs else self._watchdog_timeout
+            ),
+        )
 
     async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None):
-        if not self._task_manager:
+        await self.get_task_manager().cancel_task(task, timeout)
-            raise Exception(f"{self} TaskManager is still not initialized.")
-        await self._task_manager.cancel_task(task, timeout)
 
     async def wait_for_task(self, task: asyncio.Task, timeout: Optional[float] = None):
-        if not self._task_manager:
+        await self.get_task_manager().wait_for_task(task, timeout)
-            raise Exception(f"{self} TaskManager is still not initialized.")
+
-        await self._task_manager.wait_for_task(task, timeout)
+    def start_watchdog(self):
+        self.get_task_manager().start_watchdog(asyncio.current_task())
+
+    def reset_watchdog(self):
+        self.get_task_manager().reset_watchdog(asyncio.current_task())
 
     async def setup(self, setup: FrameProcessorSetup):
         self._clock = setup.clock
@@ -206,9 +232,7 @@ class FrameProcessor(BaseObject):
         logger.debug(f"Linking {self} -> {self._next}")
 
     def get_event_loop(self) -> asyncio.AbstractEventLoop:
-        if not self._task_manager:
+        return self.get_task_manager().get_event_loop()
-            raise Exception(f"{self} TaskManager is still not initialized.")
-        return self._task_manager.get_event_loop()
 
     def set_parent(self, parent: "FrameProcessor"):
         self._parent = parent
@@ -296,11 +320,11 @@ class FrameProcessor(BaseObject):
         await self.__cancel_push_task()
 
     async def __pause(self, frame: FrameProcessorPauseFrame | FrameProcessorPauseUrgentFrame):
-        if frame.name == self.name:
+        if frame.processor.name == self.name:
             await self.pause_processing_frames()
 
     async def __resume(self, frame: FrameProcessorResumeFrame | FrameProcessorResumeUrgentFrame):
-        if frame.name == self.name:
+        if frame.processor.name == self.name:
             await self.resume_processing_frames()
 
     #
@@ -315,9 +339,8 @@ class FrameProcessor(BaseObject):
             # Cancel the input task. This will stop processing queued frames.
             await self.__cancel_input_task()
         except Exception as e:
-            logger.exception(f"Uncaught exception in {self}: {e}")
+            logger.exception(f"Uncaught exception in {self} when handling _start_interruption: {e}")
             await self.push_error(ErrorFrame(str(e)))
-            raise
 
         # Create a new input queue and task.
         self.__create_input_task()
@@ -360,7 +383,6 @@ class FrameProcessor(BaseObject):
         except Exception as e:
             logger.exception(f"Uncaught exception in {self}: {e}")
             await self.push_error(ErrorFrame(str(e)))
-            raise
 
     def _check_started(self, frame: Frame):
         if not self.__started:
@@ -389,15 +411,19 @@ class FrameProcessor(BaseObject):
                 logger.trace(f"{self}: frame processing resumed")
 
             (frame, direction, callback) = await self.__input_queue.get()
-
+            try:
-            # Process the frame.
+                self.start_watchdog()
-            await self.process_frame(frame, direction)
+                # Process the frame.
-
+                await self.process_frame(frame, direction)
-            # If this frame has an associated callback, call it now.
+                # If this frame has an associated callback, call it now.
-            if callback:
+                if callback:
-                await callback(self, frame, direction)
+                    await callback(self, frame, direction)
-
+            except Exception as e:
-            self.__input_queue.task_done()
+                logger.exception(f"{self}: error processing frame: {e}")
+                await self.push_error(ErrorFrame(str(e)))
+            finally:
+                self.__input_queue.task_done()
+                self.reset_watchdog()
 
     def __create_push_task(self):
         if not self.__push_frame_task:
@@ -412,5 +438,7 @@ class FrameProcessor(BaseObject):
     async def __push_frame_task_handler(self):
         while True:
             (frame, direction) = await self.__push_queue.get()
+            self.start_watchdog()
             await self.__internal_push_frame(frame, direction)
             self.__push_queue.task_done()
+            self.reset_watchdog()

src/pipecat/processors/frameworks/rtvi.py

@@ -783,14 +783,18 @@ class RTVIProcessor(FrameProcessor):
     async def _action_task_handler(self):
         while True:
             frame = await self._action_queue.get()
+            self.start_watchdog()
             await self._handle_action(frame.message_id, frame.rtvi_action_run)
             self._action_queue.task_done()
+            self.reset_watchdog()
 
     async def _message_task_handler(self):
         while True:
             message = await self._message_queue.get()
+            self.start_watchdog()
             await self._handle_message(message)
             self._message_queue.task_done()
+            self.reset_watchdog()
 
     async def _handle_transport_message(self, frame: TransportMessageUrgentFrame):
         try:

src/pipecat/services/assemblyai/stt.py

@@ -190,6 +190,7 @@ class AssemblyAISTTService(STTService):
             while self._connected:
                 try:
                     message = await self._websocket.recv()
+                    self.start_watchdog()
                     data = json.loads(message)
                     await self._handle_message(data)
                 except websockets.exceptions.ConnectionClosedOK:
@@ -197,6 +198,8 @@ class AssemblyAISTTService(STTService):
                 except Exception as e:
                     logger.error(f"Error processing WebSocket message: {e}")
                     break
+                finally:
+                    self.reset_watchdog()
 
         except Exception as e:
             logger.error(f"Fatal error in receive handler: {e}")

src/pipecat/services/aws/stt.py

@@ -285,6 +285,9 @@ class AWSTranscribeSTTService(STTService):
 
             try:
                 response = await self._ws_client.recv()
+
+                self.start_watchdog()
+
                 headers, payload = decode_event(response)
 
                 if headers.get(":message-type") == "event":
@@ -342,3 +345,5 @@ class AWSTranscribeSTTService(STTService):
             except Exception as e:
                 logger.error(f"{self} Unexpected error in receive loop: {e}")
                 break
+            finally:
+                self.reset_watchdog()

src/pipecat/services/aws_nova_sonic/aws.py

@@ -699,6 +699,8 @@ class AWSNovaSonicLLMService(LLMService):
                 output = await self._stream.await_output()
                 result = await output[1].receive()
 
+                self.start_watchdog()
+
                 if result.value and result.value.bytes_:
                     response_data = result.value.bytes_.decode("utf-8")
                     json_data = json.loads(response_data)
@@ -731,6 +733,8 @@ class AWSNovaSonicLLMService(LLMService):
             logger.error(f"{self} error processing responses: {e}")
             if self._wants_connection:
                 await self.reset_conversation()
+        finally:
+            self.reset_watchdog()
 
     async def _handle_completion_start_event(self, event_json):
         pass

src/pipecat/services/elevenlabs/tts.py

@@ -284,7 +284,6 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
         logger.trace(f"{self}: flushing audio")
         msg = {"context_id": self._context_id, "flush": True}
         await self._websocket.send(json.dumps(msg))
-        self._context_id = None
 
     async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
         await super().push_frame(frame, direction)
@@ -380,6 +379,12 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
         if self._context_id and self._websocket:
             logger.trace(f"Closing context {self._context_id} due to interruption")
             try:
+                # ElevenLabs requires that Pipecat manages the contexts and closes them
+                # when they're not longer in use. Since a StartInterruptionFrame is pushed
+                # every time the user speaks, we'll use this as a trigger to close the context
+                # and reset the state.
+                # Note: We do not need to call remove_audio_context here, as the context is
+                # automatically reset when super ()._handle_interruption is called.
                 await self._websocket.send(
                     json.dumps({"context_id": self._context_id, "close_context": True})
                 )
@@ -391,10 +396,20 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
     async def _receive_messages(self):
         async for message in self._get_websocket():
             msg = json.loads(message)
-            # Check if this message belongs to the current context
+
             received_ctx_id = msg.get("contextId")
+
+            # Handle final messages first, regardless of context availability
+            # At the moment, this message is received AFTER the close_context message is
+            # sent, so it doesn't serve any functional purpose. For now, we'll just log it.
+            if msg.get("isFinal") is True:
+                logger.trace(f"Received final message for context {received_ctx_id}")
+                continue
+
+            # Check if this message belongs to the current context.
+            # This should never happen, so warn about it.
             if not self.audio_context_available(received_ctx_id):
-                logger.trace(f"Ignoring message from unavailable context: {received_ctx_id}")
+                logger.warning(f"Ignoring message from unavailable context: {received_ctx_id}")
                 continue
 
             if msg.get("audio"):
@@ -408,21 +423,26 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
                 word_times = calculate_word_times(msg["alignment"], self._cumulative_time)
                 await self.add_word_timestamps(word_times)
                 self._cumulative_time = word_times[-1][1]
-            if msg.get("isFinal"):
-                logger.trace(f"Received final message for context {received_ctx_id}")
-                await self.remove_audio_context(received_ctx_id)
-                # Reset context tracking if this was our active context
-                if self._context_id == received_ctx_id:
-                    self._context_id = None
-                    self._started = False
 
     async def _keepalive_task_handler(self):
         while True:
             await asyncio.sleep(10)
             try:
-                # Send an empty message to keep the connection alive
                 if self._websocket and self._websocket.open:
-                    await self._websocket.send(json.dumps({}))
+                    if self._context_id:
+                        # Send keepalive with context ID to keep the connection alive
+                        keepalive_message = {
+                            "text": "",
+                            "context_id": self._context_id,
+                        }
+                        logger.trace(f"Sending keepalive for context {self._context_id}")
+                    else:
+                        # It's possible to have a user interruption which clears the context
+                        # without generating a new TTS response. In this case, we'll just send
+                        # an empty message to keep the connection alive.
+                        keepalive_message = {"text": ""}
+                        logger.trace("Sending keepalive without context")
+                    await self._websocket.send(json.dumps(keepalive_message))
             except websockets.ConnectionClosed as e:
                 logger.warning(f"{self} keepalive error: {e}")
                 break
@@ -441,14 +461,6 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
                 await self._connect()
 
             try:
-                # Close previous context if there was one
-                if self._context_id and not self._started:
-                    await self._websocket.send(
-                        json.dumps({"context_id": self._context_id, "close_context": True})
-                    )
-                    await self.remove_audio_context(self._context_id)
-                    self._context_id = None
-
                 if not self._started:
                     await self.start_ttfb_metrics()
                     yield TTSStartedFrame()
@@ -473,9 +485,6 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
                 logger.error(f"{self} error sending message: {e}")
                 yield TTSStoppedFrame()
                 self._started = False
-                if self._context_id:
-                    await self.remove_audio_context(self._context_id)
-                    self._context_id = None
                 return
             yield None
         except Exception as e:

src/pipecat/services/gemini_multimodal_live/gemini.py

@@ -687,6 +687,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
 
     async def _receive_task_handler(self):
         async for message in self._websocket:
+            self.start_watchdog()
+
             evt = events.parse_server_event(message)
             # logger.debug(f"Received event: {message[:500]}")
             # logger.debug(f"Received event: {evt}")
@@ -708,8 +710,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
                 await self._handle_evt_error(evt)
                 # errors are fatal, so exit the receive loop
                 return
-            else:
+
-                pass
+            self.reset_watchdog()
 
     #
     #

src/pipecat/services/gladia/stt.py

@@ -502,6 +502,8 @@ class GladiaSTTService(STTService):
     async def _receive_task_handler(self):
         try:
             async for message in self._websocket:
+                self.start_watchdog()
+
                 content = json.loads(message)
 
                 # Handle audio chunk acknowledgments
@@ -559,11 +561,15 @@ class GladiaSTTService(STTService):
                                 translation, "", time_now_iso8601(), translated_language
                             )
                         )
+
+                self.reset_watchdog()
         except websockets.exceptions.ConnectionClosed:
             # Expected when closing the connection
             pass
         except Exception as e:
             logger.error(f"Error in Gladia WebSocket handler: {e}")
+        finally:
+            self.reset_watchdog()
 
     async def _maybe_reconnect(self) -> bool:
         """Handle exponential backoff reconnection logic."""

src/pipecat/services/google/stt.py

@@ -747,9 +747,12 @@ class GoogleSTTService(STTService):
         try:
             while True:
                 try:
+                    self.start_watchdog()
+
                     if self._request_queue.empty():
                         # wait for 10ms in case we don't have audio
                         await asyncio.sleep(0.01)
+                        self.reset_watchdog()
                         continue
 
                     # Start bi-directional streaming
@@ -760,12 +763,13 @@ class GoogleSTTService(STTService):
                     # Process responses
                     await self._process_responses(streaming_recognize)
 
+                    self.reset_watchdog()
+
                     # If we're here, check if we need to reconnect
                     if (int(time.time() * 1000) - self._stream_start_time) > self.STREAMING_LIMIT:
                         logger.debug("Reconnecting stream after timeout")
                         # Reset stream start time
                         self._stream_start_time = int(time.time() * 1000)
-                        continue
                     else:
                         # Normal stream end
                         break
@@ -775,7 +779,8 @@ class GoogleSTTService(STTService):
 
                     await asyncio.sleep(1)  # Brief delay before reconnecting
                     self._stream_start_time = int(time.time() * 1000)
-                    continue
+                finally:
+                    self.reset_watchdog()
 
         except Exception as e:
             logger.error(f"Error in streaming task: {e}")
@@ -800,12 +805,16 @@ class GoogleSTTService(STTService):
         """Process streaming recognition responses."""
         try:
             async for response in streaming_recognize:
+                self.start_watchdog()
+
                 # Check streaming limit
                 if (int(time.time() * 1000) - self._stream_start_time) > self.STREAMING_LIMIT:
                     logger.debug("Stream timeout reached in response processing")
+                    self.reset_watchdog()
                     break
 
                 if not response.results:
+                    self.reset_watchdog()
                     continue
 
                 for result in response.results:
@@ -848,8 +857,10 @@ class GoogleSTTService(STTService):
                             )
                         )
 
+                self.reset_watchdog()
         except Exception as e:
             logger.error(f"Error processing Google STT responses: {e}")
-
+            self.reset_watchdog()
-            # Re-raise the exception to let it propagate (e.g. in the case of a timeout, propagate to _stream_audio to reconnect)
+            # Re-raise the exception to let it propagate (e.g. in the case of a
+            # timeout, propagate to _stream_audio to reconnect)
             raise

src/pipecat/services/mcp_service.py

@@ -8,8 +8,8 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
 from pipecat.utils.base_object import BaseObject
 
 try:
-    from mcp import ClientSession, StdioServerParameters, types
+    from mcp import ClientSession, StdioServerParameters
-    from mcp.client.session import ClientSession
+    from mcp.client.session_group import SseServerParameters
     from mcp.client.sse import sse_client
     from mcp.client.stdio import stdio_client
 except ModuleNotFoundError as e:
@@ -21,7 +21,7 @@ except ModuleNotFoundError as e:
 class MCPClient(BaseObject):
     def __init__(
         self,
-        server_params: Union[StdioServerParameters, str],
+        server_params: Union[StdioServerParameters, SseServerParameters],
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -30,12 +30,12 @@ class MCPClient(BaseObject):
         if isinstance(server_params, StdioServerParameters):
             self._client = stdio_client
             self._register_tools = self._stdio_register_tools
-        elif isinstance(server_params, str):
+        elif isinstance(server_params, SseServerParameters):
             self._client = sse_client
             self._register_tools = self._sse_register_tools
         else:
             raise TypeError(
-                f"{self} invalid argument type: `server_params` must be either StdioServerParameters or an SSE server url string."
+                f"{self} invalid argument type: `server_params` must be either StdioServerParameters or SseServerParameters."
             )
 
     async def register_tools(self, llm) -> ToolsSchema:
@@ -90,7 +90,12 @@ class MCPClient(BaseObject):
             logger.debug(f"Executing tool '{function_name}' with call ID: {tool_call_id}")
             logger.trace(f"Tool arguments: {json.dumps(arguments, indent=2)}")
             try:
-                async with self._client(self._server_params) as (read, write):
+                async with self._client(
+                    url=self._server_params.url,
+                    headers=self._server_params.headers,
+                    timeout=self._server_params.timeout,
+                    sse_read_timeout=self._server_params.sse_read_timeout,
+                ) as (read, write):
                     async with self._session(read, write) as session:
                         await session.initialize()
                         await self._call_tool(session, function_name, arguments, result_callback)
@@ -100,10 +105,14 @@ class MCPClient(BaseObject):
                 logger.exception("Full exception details:")
                 await result_callback(error_msg)
 
-        logger.debug("Starting registration of mcp.run tools")
+        logger.debug(f"SSE server parameters: {self._server_params}")
-        tool_schemas: List[FunctionSchema] = []
 
-        async with self._client(self._server_params) as (read, write):
+        async with self._client(
+            url=self._server_params.url,
+            headers=self._server_params.headers,
+            timeout=self._server_params.timeout,
+            sse_read_timeout=self._server_params.sse_read_timeout,
+        ) as (read, write):
             async with self._session(read, write) as session:
                 await session.initialize()
                 tools_schema = await self._list_tools(session, mcp_tool_wrapper, llm)

src/pipecat/services/openai_realtime_beta/events.py

@@ -36,10 +36,6 @@ class InputAudioTranscription(BaseModel):
         prompt: Optional[str] = None,
     ):
         super().__init__(model=model, language=language, prompt=prompt)
-        if self.model != "gpt-4o-transcribe" and (self.language or self.prompt):
-            raise ValueError(
-                "Fields 'language' and 'prompt' are only supported when model is 'gpt-4o-transcribe'"
-            )
 
 
 class TurnDetection(BaseModel):
@@ -207,12 +203,11 @@ class ResponseCancelEvent(ClientEvent):
 
 
 class ServerEvent(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
     event_id: str
     type: str
 
-    class Config:
-        arbitrary_types_allowed = True
-
 
 class SessionCreatedEvent(ServerEvent):
     type: Literal["session.created"]

src/pipecat/services/openai_realtime_beta/openai.py

@@ -86,7 +86,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
         self,
         *,
         api_key: str,
-        model: str = "gpt-4o-realtime-preview-2024-12-17",
+        model: str = "gpt-4o-realtime-preview-2025-06-03",
         base_url: str = "wss://api.openai.com/v1/realtime",
         session_properties: Optional[events.SessionProperties] = None,
         start_audio_paused: bool = False,
@@ -370,6 +370,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
 
     async def _receive_task_handler(self):
         async for message in self._websocket:
+            self.start_watchdog()
             evt = events.parse_server_event(message)
             if evt.type == "session.created":
                 await self._handle_evt_session_created(evt)
@@ -400,6 +401,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
                     await self._handle_evt_error(evt)
                     # errors are fatal, so exit the receive loop
                     return
+            self.reset_watchdog()
 
     @traced_openai_realtime(operation="llm_setup")
     async def _handle_evt_session_created(self, evt):

src/pipecat/services/riva/stt.py

@@ -224,11 +224,13 @@ class RivaSTTService(STTService):
             streaming_config=self._config,
         )
         for response in responses:
+            self.start_watchdog()
             if not response.results:
                 continue
             asyncio.run_coroutine_threadsafe(
                 self._response_queue.put(response), self.get_event_loop()
             )
+            self.reset_watchdog()
 
     async def _thread_task_handler(self):
         try:
@@ -283,7 +285,9 @@ class RivaSTTService(STTService):
     async def _response_task_handler(self):
         while True:
             response = await self._response_queue.get()
+            self.start_watchdog()
             await self._handle_response(response)
+            self.reset_watchdog()
 
     async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
         await self.start_ttfb_metrics()

src/pipecat/services/simli/video.py

@@ -62,6 +62,7 @@ class SimliVideoService(FrameProcessor):
     async def _consume_and_process_audio(self):
         await self._pipecat_resampler_event.wait()
         async for audio_frame in self._simli_client.getAudioStreamIterator():
+            self.start_watchdog()
             resampled_frames = self._pipecat_resampler.resample(audio_frame)
             for resampled_frame in resampled_frames:
                 audio_array = resampled_frame.to_ndarray()
@@ -74,10 +75,12 @@ class SimliVideoService(FrameProcessor):
                             num_channels=1,
                         ),
                     )
+            self.reset_watchdog()
 
     async def _consume_and_process_video(self):
         await self._pipecat_resampler_event.wait()
         async for video_frame in self._simli_client.getVideoStreamIterator(targetFormat="rgb24"):
+            self.start_watchdog()
             # Process the video frame
             convertedFrame: OutputImageRawFrame = OutputImageRawFrame(
                 image=video_frame.to_rgb().to_image().tobytes(),
@@ -86,6 +89,7 @@ class SimliVideoService(FrameProcessor):
             )
             convertedFrame.pts = video_frame.pts
             await self.push_frame(convertedFrame)
+            self.reset_watchdog()
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         await super().process_frame(frame, direction)

src/pipecat/services/tavus/video.py

@@ -217,5 +217,7 @@ class TavusVideoService(AIService):
     async def _send_task_handler(self):
         while True:
             frame = await self._queue.get()
-            if isinstance(frame, OutputAudioRawFrame):
+            self.start_watchdog()
+            if isinstance(frame, OutputAudioRawFrame) and self._client:
                 await self._client.write_audio_frame(frame)
+            self.reset_watchdog()

src/pipecat/transports/base_input.py

@@ -43,6 +43,8 @@ from pipecat.metrics.metrics import MetricsData
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 from pipecat.transports.base_transport import TransportParams
 
+AUDIO_INPUT_TIMEOUT_SECS = 0.5
+
 
 class BaseInputTransport(FrameProcessor):
     def __init__(self, params: TransportParams, **kwargs):
@@ -56,6 +58,9 @@ class BaseInputTransport(FrameProcessor):
         # Track bot speaking state for interruption logic
         self._bot_speaking = False
 
+        # Track user speaking state for interruption logic
+        self._user_speaking = False
+
         # We read audio from a single queue one at a time and we then run VAD in
         # a thread. Therefore, only one thread should be necessary.
         self._executor = ThreadPoolExecutor(max_workers=1)
@@ -130,6 +135,7 @@ class BaseInputTransport(FrameProcessor):
 
     async def start(self, frame: StartFrame):
         self._paused = False
+        self._user_speaking = False
 
         self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
 
@@ -240,6 +246,7 @@ class BaseInputTransport(FrameProcessor):
     async def _handle_user_interruption(self, frame: Frame):
         if isinstance(frame, UserStartedSpeakingFrame):
             logger.debug("User started speaking")
+            self._user_speaking = True
             await self.push_frame(frame)
 
             # Only push StartInterruptionFrame if:
@@ -263,6 +270,7 @@ class BaseInputTransport(FrameProcessor):
                 )
         elif isinstance(frame, UserStoppedSpeakingFrame):
             logger.debug("User stopped speaking")
+            self._user_speaking = False
             await self.push_frame(frame)
             if self.interruptions_allowed:
                 await self._stop_interruption()
@@ -355,26 +363,42 @@ class BaseInputTransport(FrameProcessor):
     async def _audio_task_handler(self):
         vad_state: VADState = VADState.QUIET
         while True:
-            frame: InputAudioRawFrame = await self._audio_in_queue.get()
+            try:
+                frame: InputAudioRawFrame = await asyncio.wait_for(
+                    self._audio_in_queue.get(), timeout=AUDIO_INPUT_TIMEOUT_SECS
+                )
 
-            # If an audio filter is available, run it before VAD.
+                self.start_watchdog()
-            if self._params.audio_in_filter:
-                frame.audio = await self._params.audio_in_filter.filter(frame.audio)
 
-            # Check VAD and push event if necessary. We just care about
+                # If an audio filter is available, run it before VAD.
-            # changes from QUIET to SPEAKING and vice versa.
+                if self._params.audio_in_filter:
-            previous_vad_state = vad_state
+                    frame.audio = await self._params.audio_in_filter.filter(frame.audio)
-            if self._params.vad_analyzer:
-                vad_state = await self._handle_vad(frame, vad_state)
 
-            if self._params.turn_analyzer:
+                # Check VAD and push event if necessary. We just care about
-                await self._run_turn_analyzer(frame, vad_state, previous_vad_state)
+                # changes from QUIET to SPEAKING and vice versa.
+                previous_vad_state = vad_state
+                if self._params.vad_analyzer:
+                    vad_state = await self._handle_vad(frame, vad_state)
 
-            # Push audio downstream if passthrough is set.
+                if self._params.turn_analyzer:
-            if self._params.audio_in_passthrough:
+                    await self._run_turn_analyzer(frame, vad_state, previous_vad_state)
-                await self.push_frame(frame)
 
-            self._audio_in_queue.task_done()
+                # Push audio downstream if passthrough is set.
+                if self._params.audio_in_passthrough:
+                    await self.push_frame(frame)
+
+                self._audio_in_queue.task_done()
+            except asyncio.TimeoutError:
+                if self._user_speaking:
+                    logger.warning(
+                        "Forcing user stopped speaking due to timeout receiving audio frame!"
+                    )
+                    vad_state = VADState.QUIET
+                    if self._params.turn_analyzer:
+                        self._params.turn_analyzer.clear()
+                    await self._handle_user_interruption(UserStoppedSpeakingFrame())
+            finally:
+                self.reset_watchdog()
 
     async def _handle_prediction_result(self, result: MetricsData):
         """Handle a prediction result event from the turn analyzer.

src/pipecat/transports/network/fastapi_websocket.py

@@ -70,11 +70,22 @@ class FastAPIWebsocketClient:
         return self._websocket.iter_bytes() if self._is_binary else self._websocket.iter_text()
 
     async def send(self, data: str | bytes):
-        if self._can_send():
+        try:
-            if self._is_binary:
+            if self._can_send():
-                await self._websocket.send_bytes(data)
+                if self._is_binary:
-            else:
+                    await self._websocket.send_bytes(data)
-                await self._websocket.send_text(data)
+                else:
+                    await self._websocket.send_text(data)
+        except Exception as e:
+            logger.error(
+                f"{self} exception sending data: {e.__class__.__name__} ({e}), application_state: {self._websocket.application_state}"
+            )
+            # For some reason the websocket is disconnected, and we are not able to send data
+            # So let's properly handle it and disconnect the transport
+            if self._websocket.application_state == WebSocketState.DISCONNECTED:
+                logger.warning("Closing already disconnected websocket!")
+                self._closing = True
+                await self.trigger_client_disconnected()
 
     async def disconnect(self):
         self._leave_counter -= 1
@@ -171,6 +182,8 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
                 if not self._params.serializer:
                     continue
 
+                self.start_watchdog()
+
                 frame = await self._params.serializer.deserialize(message)
 
                 if not frame:
@@ -180,9 +193,13 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
                     await self.push_audio_frame(frame)
                 else:
                     await self.push_frame(frame)
+
+                self.reset_watchdog()
         except Exception as e:
             logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
 
+        self.reset_watchdog()
+
         await self._client.trigger_client_disconnected()
 
     async def _monitor_websocket(self):

src/pipecat/transports/network/small_webrtc.py

@@ -423,8 +423,10 @@ class SmallWebRTCInputTransport(BaseInputTransport):
     async def _receive_audio(self):
         try:
             async for audio_frame in self._client.read_audio_frame():
+                self.start_watchdog()
                 if audio_frame:
                     await self.push_audio_frame(audio_frame)
+                self.reset_watchdog()
 
         except Exception as e:
             logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
@@ -432,6 +434,7 @@ class SmallWebRTCInputTransport(BaseInputTransport):
     async def _receive_video(self):
         try:
             async for video_frame in self._client.read_video_frame():
+                self.start_watchdog()
                 if video_frame:
                     await self.push_video_frame(video_frame)
 
@@ -450,6 +453,7 @@ class SmallWebRTCInputTransport(BaseInputTransport):
                             await self.push_video_frame(image_frame)
                             # Remove from pending requests
                             del self._image_requests[req_id]
+                self.reset_watchdog()
 
         except Exception as e:
             logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")

src/pipecat/transports/services/livekit.py

@@ -415,6 +415,7 @@ class LiveKitInputTransport(BaseInputTransport):
         logger.info("Audio input task started")
         while True:
             audio_data = await self._client.get_next_audio_frame()
+            self.start_watchdog()
             if audio_data:
                 audio_frame_event, participant_id = audio_data
                 pipecat_audio_frame = await self._convert_livekit_audio_to_pipecat(
@@ -427,6 +428,7 @@ class LiveKitInputTransport(BaseInputTransport):
                     num_channels=pipecat_audio_frame.num_channels,
                 )
                 await self.push_audio_frame(input_audio_frame)
+            self.reset_watchdog()
 
     async def _convert_livekit_audio_to_pipecat(
         self, audio_frame_event: rtc.AudioFrameEvent

src/pipecat/utils/asyncio.py

@@ -5,15 +5,30 @@
 #
 
 import asyncio
+import time
 from abc import ABC, abstractmethod
-from typing import Coroutine, Dict, Optional, Sequence, Set
+from dataclasses import dataclass
+from typing import Coroutine, Dict, List, Optional, Sequence
 
 from loguru import logger
 
+WATCHDOG_TIMEOUT = 5.0
+
+
+@dataclass
+class TaskManagerParams:
+    loop: asyncio.AbstractEventLoop
+    enable_watchdog_logging: bool = False
+    watchdog_timeout: float = WATCHDOG_TIMEOUT
+
 
 class BaseTaskManager(ABC):
     @abstractmethod
-    def set_event_loop(self, loop: asyncio.AbstractEventLoop):
+    def setup(self, params: TaskManagerParams):
+        pass
+
+    @abstractmethod
+    async def cleanup(self):
         pass
 
     @abstractmethod
@@ -21,7 +36,14 @@ class BaseTaskManager(ABC):
         pass
 
     @abstractmethod
-    def create_task(self, coroutine: Coroutine, name: str) -> asyncio.Task:
+    def create_task(
+        self,
+        coroutine: Coroutine,
+        name: str,
+        *,
+        enable_watchdog_logging: Optional[bool] = None,
+        watchdog_timeout: Optional[float] = None,
+    ) -> asyncio.Task:
         """
         Creates and schedules a new asyncio Task that runs the given coroutine.
 
@@ -31,6 +53,8 @@ class BaseTaskManager(ABC):
             loop (asyncio.AbstractEventLoop): The event loop to use for creating the task.
             coroutine (Coroutine): The coroutine to be executed within the task.
             name (str): The name to assign to the task for identification.
+            enable_watchdog_logging(bool): whether this task should log watchdog processing times.
+            watchdog_timeout(float): watchdog timer timeout for this task.
 
         Returns:
             asyncio.Task: The created task object.
@@ -73,21 +97,64 @@ class BaseTaskManager(ABC):
         """Returns the list of currently created/registered tasks."""
         pass
 
+    @abstractmethod
+    def start_watchdog(self, task: asyncio.Task):
+        """Starts the given task watchdog timer. If not reset, a warning will be
+        logged indicating the task is stalling.
+
+        """
+        pass
+
+    @abstractmethod
+    def reset_watchdog(self, task: asyncio.Task):
+        """Resets the given task watchdog timer. If not reset, a warning will be
+        logged indicating the task is stalling.
+
+        """
+        pass
+
+
+@dataclass
+class TaskData:
+    task: asyncio.Task
+    watchdog_start: asyncio.Event
+    watchdog_timer: asyncio.Event
+    enable_watchdog_logging: bool
+    watchdog_timeout: float
+
 
 class TaskManager(BaseTaskManager):
     def __init__(self) -> None:
-        self._tasks: Dict[str, asyncio.Task] = {}
+        self._tasks: Dict[str, TaskData] = {}
-        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._params: Optional[TaskManagerParams] = None
+        self._watchdog_tasks: List[asyncio.Task] = []
 
-    def set_event_loop(self, loop: asyncio.AbstractEventLoop):
+    def setup(self, params: TaskManagerParams):
-        self._loop = loop
+        if not self._params:
+            self._params = params
+
+    async def cleanup(self):
+        for task in self._watchdog_tasks:
+            try:
+                task.cancel()
+                await task
+            except asyncio.CancelledError:
+                # This is expected, no need to re-raise.
+                pass
 
     def get_event_loop(self) -> asyncio.AbstractEventLoop:
-        if not self._loop:
+        if not self._params:
-            raise Exception("TaskManager missing event loop, use TaskManager.set_event_loop().")
+            raise Exception("TaskManager is not setup: unable to get event loop")
-        return self._loop
+        return self._params.loop
 
-    def create_task(self, coroutine: Coroutine, name: str) -> asyncio.Task:
+    def create_task(
+        self,
+        coroutine: Coroutine,
+        name: str,
+        *,
+        enable_watchdog_logging: Optional[bool] = None,
+        watchdog_timeout: Optional[float] = None,
+    ) -> asyncio.Task:
         """
         Creates and schedules a new asyncio Task that runs the given coroutine.
 
@@ -97,6 +164,8 @@ class TaskManager(BaseTaskManager):
             loop (asyncio.AbstractEventLoop): The event loop to use for creating the task.
             coroutine (Coroutine): The coroutine to be executed within the task.
             name (str): The name to assign to the task for identification.
+            enable_watchdog_logging(bool): whether this task should log watchdog processing time.
+            watchdog_timeout(float): watchdog timer timeout for this task.
 
         Returns:
             asyncio.Task: The created task object.
@@ -112,12 +181,26 @@ class TaskManager(BaseTaskManager):
             except Exception as e:
                 logger.exception(f"{name}: unexpected exception: {e}")
 
-        if not self._loop:
+        if not self._params:
-            raise Exception("TaskManager missing event loop, use TaskManager.set_event_loop().")
+            raise Exception("TaskManager is not setup: unable to get event loop")
 
-        task = self._loop.create_task(run_coroutine())
+        task = self._params.loop.create_task(run_coroutine())
         task.set_name(name)
-        self._add_task(task)
+        self._add_task(
+            TaskData(
+                task=task,
+                watchdog_start=asyncio.Event(),
+                watchdog_timer=asyncio.Event(),
+                enable_watchdog_logging=(
+                    enable_watchdog_logging
+                    if enable_watchdog_logging
+                    else self._params.enable_watchdog_logging
+                ),
+                watchdog_timeout=(
+                    watchdog_timeout if watchdog_timeout else self._params.watchdog_timeout
+                ),
+            )
+        )
         logger.trace(f"{name}: task created")
         return task
 
@@ -165,6 +248,8 @@ class TaskManager(BaseTaskManager):
         name = task.get_name()
         task.cancel()
         try:
+            # Make sure to reset watchdog if a task is cancelled.
+            self.reset_watchdog(task)
             if timeout:
                 await asyncio.wait_for(task, timeout=timeout)
             else:
@@ -176,16 +261,51 @@ class TaskManager(BaseTaskManager):
             pass
         except Exception as e:
             logger.exception(f"{name}: unexpected exception while cancelling task: {e}")
+        except BaseException as e:
+            logger.critical(f"{name}: fatal base exception while cancelling task: {e}")
+            raise
         finally:
             self._remove_task(task)
 
     def current_tasks(self) -> Sequence[asyncio.Task]:
         """Returns the list of currently created/registered tasks."""
-        return list(self._tasks.values())
+        return [data.task for data in self._tasks.values()]
 
-    def _add_task(self, task: asyncio.Task):
+    def start_watchdog(self, task: asyncio.Task):
+        """Starts the given task watchdog timer. If not reset, a warning will be
+        logged indicating the task is stalling. If the timer was already started
+        a warning will be logged.
+
+        """
         name = task.get_name()
-        self._tasks[name] = task
+        if name in self._tasks:
+            if self._tasks[name].watchdog_start.is_set():
+                logger.warning(f"Watchdog timer for task {name} already started")
+            else:
+                self._tasks[name].watchdog_timer.clear()
+                self._tasks[name].watchdog_start.set()
+        else:
+            logger.warning(f"Unable to start watchdog timer: task {name} does not exist")
+
+    def reset_watchdog(self, task: asyncio.Task):
+        """Resets the given task watchdog timer. If not reset, a warning will be
+        logged indicating the task is stalling.
+
+        """
+        name = task.get_name()
+        if name in self._tasks:
+            self._tasks[name].watchdog_start.clear()
+            self._tasks[name].watchdog_timer.set()
+        else:
+            logger.warning(f"Unable to reset watchdog timer: task {name} does not exist")
+
+    def _add_task(self, task_data: TaskData):
+        name = task_data.task.get_name()
+        self._tasks[name] = task_data
+        watchdog_task = self.get_event_loop().create_task(
+            self._watchdog_task_handler(self._tasks[name])
+        )
+        self._watchdog_tasks.append(watchdog_task)
 
     def _remove_task(self, task: asyncio.Task):
         name = task.get_name()
@@ -193,3 +313,33 @@ class TaskManager(BaseTaskManager):
             del self._tasks[name]
         except KeyError as e:
             logger.trace(f"{name}: unable to remove task (already removed?): {e}")
+
+    async def _watchdog_task_handler(self, task_data: TaskData):
+        name = task_data.task.get_name()
+        start = task_data.watchdog_start
+        timer = task_data.watchdog_timer
+        enable_watchdog_logging = task_data.enable_watchdog_logging
+        watchdog_timeout = task_data.watchdog_timeout
+
+        async def wait_for_reset():
+            waiting = True
+            while waiting:
+                try:
+                    start_time = time.time()
+                    await asyncio.wait_for(timer.wait(), timeout=watchdog_timeout)
+                    total_time = time.time() - start_time
+                    if enable_watchdog_logging:
+                        logger.debug(f"{name} task processing time: {total_time:.20f}")
+                    waiting = False
+                except asyncio.TimeoutError:
+                    logger.warning(
+                        f"{name}: task is taking too long {WATCHDOG_TIMEOUT} second(s) (forgot to reset watchdog?)"
+                    )
+                finally:
+                    timer.clear()
+
+        while True:
+            # Wait for the user to start the watchdog timer.
+            await start.wait()
+            # Now, waiting for the task to finish.
+            await wait_for_reset()

tests/test_pipeline.py

@@ -17,6 +17,7 @@ from pipecat.frames.frames import (
     TextFrame,
 )
 from pipecat.observers.base_observer import BaseObserver, FramePushed
+from pipecat.pipeline.base_task import PipelineTaskParams
 from pipecat.pipeline.parallel_pipeline import ParallelPipeline
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.task import PipelineParams, PipelineTask
@@ -96,11 +97,10 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
     async def test_task_single(self):
         pipeline = Pipeline([IdentityFilter()])
         task = PipelineTask(pipeline)
-        task.set_event_loop(asyncio.get_event_loop())
 
         await task.queue_frame(TextFrame(text="Hello!"))
         await task.queue_frames([TextFrame(text="Bye!"), EndFrame()])
-        await task.run()
+        await task.run(PipelineTaskParams(loop=asyncio.get_event_loop()))
         assert task.has_finished()
 
     async def test_task_observers(self):
@@ -116,10 +116,9 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
         identity = IdentityFilter()
         pipeline = Pipeline([identity])
         task = PipelineTask(pipeline, observers=[CustomObserver()])
-        task.set_event_loop(asyncio.get_event_loop())
 
         await task.queue_frames([TextFrame(text="Hello Downstream!"), EndFrame()])
-        await task.run()
+        await task.run(PipelineTaskParams(loop=asyncio.get_event_loop()))
         assert frame_received
 
     async def test_task_add_observer(self):
@@ -156,8 +155,6 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
         observer1 = CustomAddObserver1()
         task.add_observer(observer1)
 
-        task.set_event_loop(asyncio.get_event_loop())
-
         async def delayed_add_observer():
             observer2 = CustomAddObserver2()
             # Wait after the pipeline is started and add another observer.
@@ -176,7 +173,9 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
             # Finally end the pipeline.
             await task.queue_frame(EndFrame())
 
-        await asyncio.gather(task.run(), delayed_add_observer())
+        await asyncio.gather(
+            task.run(PipelineTaskParams(loop=asyncio.get_event_loop())), delayed_add_observer()
+        )
 
         assert frame_received
         assert frame_count_1 == 1
@@ -189,7 +188,6 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
         identity = IdentityFilter()
         pipeline = Pipeline([identity])
         task = PipelineTask(pipeline)
-        task.set_event_loop(asyncio.get_event_loop())
 
         @task.event_handler("on_pipeline_started")
         async def on_pipeline_started(task, frame: StartFrame):
@@ -202,7 +200,7 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
             end_received = True
 
         await task.queue_frame(EndFrame())
-        await task.run()
+        await task.run(PipelineTaskParams(loop=asyncio.get_event_loop()))
 
         assert start_received
         assert end_received
@@ -213,7 +211,6 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
         identity = IdentityFilter()
         pipeline = Pipeline([identity])
         task = PipelineTask(pipeline)
-        task.set_event_loop(asyncio.get_event_loop())
 
         @task.event_handler("on_pipeline_stopped")
         async def on_pipeline_ended(task, frame: StopFrame):
@@ -221,7 +218,7 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
             stop_received = True
 
         await task.queue_frame(StopFrame())
-        await task.run()
+        await task.run(PipelineTaskParams(loop=asyncio.get_event_loop()))
 
         assert stop_received
 
@@ -232,7 +229,6 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
         identity = IdentityFilter()
         pipeline = Pipeline([identity])
         task = PipelineTask(pipeline, cancel_on_idle_timeout=False)
-        task.set_event_loop(asyncio.get_event_loop())
         task.set_reached_upstream_filter((TextFrame,))
         task.set_reached_downstream_filter((TextFrame,))
 
@@ -254,7 +250,10 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
         await task.queue_frame(TextFrame(text="Hello Downstream!"))
 
         try:
-            await asyncio.wait_for(asyncio.shield(task.run()), timeout=1.0)
+            await asyncio.wait_for(
+                asyncio.shield(task.run(PipelineTaskParams(loop=asyncio.get_event_loop()))),
+                timeout=1.0,
+            )
         except asyncio.TimeoutError:
             pass
 
@@ -282,13 +281,15 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
             observers=[heartbeats_observer],
             cancel_on_idle_timeout=False,
         )
-        task.set_event_loop(asyncio.get_event_loop())
 
         expected_heartbeats = 1.0 / 0.2
 
         await task.queue_frame(TextFrame(text="Hello!"))
         try:
-            await asyncio.wait_for(asyncio.shield(task.run()), timeout=1.0)
+            await asyncio.wait_for(
+                asyncio.shield(task.run(PipelineTaskParams(loop=asyncio.get_event_loop()))),
+                timeout=1.0,
+            )
         except asyncio.TimeoutError:
             pass
         assert heartbeats_counter == expected_heartbeats
@@ -297,17 +298,18 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
         identity = IdentityFilter()
         pipeline = Pipeline([identity])
         task = PipelineTask(pipeline, idle_timeout_secs=0.2)
-        task.set_event_loop(asyncio.get_event_loop())
+        await task.run(PipelineTaskParams(loop=asyncio.get_event_loop()))
-        await task.run()
         assert True
 
     async def test_no_idle_task(self):
         identity = IdentityFilter()
         pipeline = Pipeline([identity])
         task = PipelineTask(pipeline, idle_timeout_secs=0.2, cancel_on_idle_timeout=False)
-        task.set_event_loop(asyncio.get_event_loop())
         try:
-            await asyncio.wait_for(asyncio.shield(task.run()), timeout=0.3)
+            await asyncio.wait_for(
+                asyncio.shield(task.run(PipelineTaskParams(loop=asyncio.get_event_loop()))),
+                timeout=0.3,
+            )
         except asyncio.TimeoutError:
             assert True
         else:
@@ -324,15 +326,13 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
             ),
             idle_timeout_secs=0.3,
         )
-        task.set_event_loop(asyncio.get_event_loop())
+        await task.run(PipelineTaskParams(loop=asyncio.get_event_loop()))
-        await task.run()
         assert True
 
     async def test_idle_task_event_handler_no_frames(self):
         identity = IdentityFilter()
         pipeline = Pipeline([identity])
         task = PipelineTask(pipeline, idle_timeout_secs=0.2, cancel_on_idle_timeout=False)
-        task.set_event_loop(asyncio.get_event_loop())
 
         idle_timeout = False
 
@@ -342,14 +342,13 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
             idle_timeout = True
             await task.cancel()
 
-        await task.run()
+        await task.run(PipelineTaskParams(loop=asyncio.get_event_loop()))
         assert idle_timeout
 
     async def test_idle_task_event_handler_quiet_user(self):
         identity = IdentityFilter()
         pipeline = Pipeline([identity])
         task = PipelineTask(pipeline, idle_timeout_secs=0.2, cancel_on_idle_timeout=False)
-        task.set_event_loop(asyncio.get_event_loop())
 
         idle_timeout = 0
 
@@ -373,7 +372,9 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
                 )
                 await asyncio.sleep(0.01)
 
-        await asyncio.gather(send_audio(), task.run())
+        await asyncio.gather(
+            send_audio(), task.run(PipelineTaskParams(loop=asyncio.get_event_loop()))
+        )
         assert idle_timeout == 1
 
     async def test_idle_task_frames(self):
@@ -387,7 +388,6 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
             idle_timeout_secs=idle_timeout_secs,
             idle_timeout_frames=(TextFrame,),
         )
-        task.set_event_loop(asyncio.get_event_loop())
 
         async def delayed_frames():
             await asyncio.sleep(sleep_time_secs)
@@ -399,7 +399,10 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase):
 
         start_time = time.time()
 
-        tasks = {asyncio.create_task(task.run()), asyncio.create_task(delayed_frames())}
+        tasks = [
+            asyncio.create_task(task.run(PipelineTaskParams(loop=asyncio.get_event_loop()))),
+            asyncio.create_task(delayed_frames()),
+        ]
 
         await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)