Merge pull request #2130 from pipecat-ai/aleix/pipeline-task-cancel-queue

PipelineTask: cancel idle queue before cancelling task

.github/ISSUE_TEMPLATE/1-bug_report.yml

@@ -0,0 +1,87 @@
+name: Bug report
+description: Report a bug or unexpected behavior
+type: Bug
+body:
+  - type: markdown
+    attributes:
+      value: |
+        ## Bug Report
+
+        Thank you for taking the time to fill out this bug report.
+
+  - type: markdown
+    attributes:
+      value: |
+        ### Environment
+
+  - type: input
+    id: pipecat-version
+    attributes:
+      label: pipecat version
+      description: Which version are you using?
+      placeholder: e.g., 0.0.63
+    validations:
+      required: true
+
+  - type: input
+    id: python-version
+    attributes:
+      label: Python version
+      description: Which Python version are you using?
+      placeholder: e.g., 3.12.8
+    validations:
+      required: true
+
+  - type: input
+    id: os
+    attributes:
+      label: Operating System
+      description: Which OS are you using?
+      placeholder: e.g., Ubuntu 24.04, Windows 11, macOS 12.5
+    validations:
+      required: true
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Issue description
+      description: Provide a clear description of the issue.
+    validations:
+      required: true
+
+  - type: textarea
+    id: repro
+    attributes:
+      label: Reproduction steps
+      description: List the steps to reproduce the issue.
+      placeholder: |
+        1. Do this...
+        2. Then do that...
+        3. Observe the error...
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: What did you expect to happen?
+    validations:
+      required: true
+
+  - type: textarea
+    id: actual
+    attributes:
+      label: Actual behavior
+      description: What actually happened?
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs
+      description: If applicable, include any relevant logs or error messages
+      render: shell
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/2-question.yml

@@ -0,0 +1,67 @@
+name: Question
+description: Ask a question or get help
+type: Question
+body:
+  - type: markdown
+    attributes:
+      value: |
+        ## Question
+
+        Use this form to ask a question about pipecat.
+
+  - type: markdown
+    attributes:
+      value: |
+        ### Environment (if applicable)
+
+  - type: input
+    id: pipecat-version
+    attributes:
+      label: pipecat version
+      description: Which version are you using? (if applicable)
+      placeholder: e.g., 0.0.63
+    validations:
+      required: false
+
+  - type: input
+    id: python-version
+    attributes:
+      label: Python version
+      description: Which Python version are you using? (if applicable)
+      placeholder: e.g., 3.12.8
+    validations:
+      required: false
+
+  - type: input
+    id: os
+    attributes:
+      label: Operating System
+      description: Which OS are you using? (if applicable)
+      placeholder: e.g., Ubuntu 24.04, Windows 11, macOS 12.5
+    validations:
+      required: false
+
+  - type: textarea
+    id: question
+    attributes:
+      label: Question
+      description: Provide your question in detail here.
+    validations:
+      required: true
+
+  - type: textarea
+    id: tried
+    attributes:
+      label: What I've tried
+      description: Describe what you've already tried or research you've done.
+      placeholder: I've looked at the documentation and tried...
+    validations:
+      required: false
+
+  - type: textarea
+    id: context
+    attributes:
+      label: Context
+      description: Any additional context or information that might help others understand your question better.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/3-feature_request.yml

@@ -0,0 +1,52 @@
+name: Feature request
+description: Suggest an enhancement or new feature
+type: Enhancement
+body:
+  - type: markdown
+    attributes:
+      value: |
+        ## Feature Request
+
+        Thank you for suggesting an enhancement to pipecat.
+
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem Statement
+      description: A clear description of the problem this feature would solve.
+      placeholder: I'm always frustrated when...
+    validations:
+      required: true
+
+  - type: textarea
+    id: solution
+    attributes:
+      label: Proposed Solution
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternative Solutions
+      description: Any alternative solutions or features you've considered.
+    validations:
+      required: false
+
+  - type: textarea
+    id: context
+    attributes:
+      label: Additional Context
+      description: Add any other context, mockups, or screenshots about the feature request here.
+      placeholder: You can drag and drop images here to include them.
+    validations:
+      required: false
+
+  - type: checkboxes
+    id: contribution
+    attributes:
+      label: Would you be willing to help implement this feature?
+      options:
+        - label: Yes, I'd like to contribute
+        - label: No, I'm just suggesting

.github/ISSUE_TEMPLATE/4-service-issue.yml

@@ -0,0 +1,82 @@
+name: Service Issue
+description: An issue with a third-party service
+type: Service Issue
+body:
+  - type: markdown
+    attributes:
+      value: |
+        ## Service Issue
+
+        Use this form to report an issue with a third-party service integration.
+
+  - type: input
+    id: pipecat-version
+    attributes:
+      label: pipecat version
+      description: Which version are you using?
+      placeholder: e.g., 0.0.63
+    validations:
+      required: true
+
+  - type: input
+    id: service-name
+    attributes:
+      label: Service Name
+      description: Which third-party service is having issues?
+      placeholder: e.g., OpenAI, ElevenLabs, Anthropic
+    validations:
+      required: true
+
+  - type: input
+    id: service-version
+    attributes:
+      label: Service or model version
+      description: Which version of the service API or model are you using?
+      placeholder: e.g., v1, gpt-4.1
+    validations:
+      required: false
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Issue Description
+      description: Provide a clear description of the service issue.
+    validations:
+      required: true
+
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Reproduction Steps
+      description: Provide steps to reproduce the issue.
+      placeholder: |
+        1. Configure service X
+        2. Call method Y
+        3. See error Z
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected Behavior
+      description: What did you expect to happen?
+    validations:
+      required: true
+
+  - type: textarea
+    id: actual
+    attributes:
+      label: Actual Behavior
+      description: What actually happened?
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Error Logs
+      description: If available, include any error messages or logs.
+      render: shell
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/5-new-service.yml

@@ -0,0 +1,56 @@
+name: New Service
+description: Request to support a new third-party service
+type: New Service
+body:
+  - type: markdown
+    attributes:
+      value: |
+        ## New Service Request
+
+        Use this form to request support for a new third-party service in pipecat.
+
+  - type: input
+    id: service-name
+    attributes:
+      label: Service Name
+      description: What is the name of the third-party service?
+      placeholder: e.g., NewAPI, SomeService
+    validations:
+      required: true
+
+  - type: input
+    id: service-website
+    attributes:
+      label: Service Website
+      description: Link to the service's website or documentation
+      placeholder: e.g., https://newapi.com
+    validations:
+      required: true
+
+  - type: textarea
+    id: service-description
+    attributes:
+      label: Service Description
+      description: Briefly describe what this service does and how it works.
+    validations:
+      required: true
+
+  - type: textarea
+    id: api-info
+    attributes:
+      label: API Information
+      description: If available, provide details about the service's API.
+      placeholder: |
+        - API documentation link
+        - Authentication method
+        - Key endpoints you'd like supported
+    validations:
+      required: false
+
+  - type: checkboxes
+    id: contribution
+    attributes:
+      label: Would you be willing to help implement this service?
+      options:
+        - label: Yes, I'd like to contribute
+        - label: No, I'm just suggesting

.github/ISSUE_TEMPLATE/6-dependency.yml

@@ -0,0 +1,74 @@
+name: Dependency Issue
+description: An issue with a Pipecat dependency (not a third-party service)
+type: Dependency Issue
+body:
+  - type: markdown
+    attributes:
+      value: |
+        ## Dependency Issue
+
+        Use this form to report an issue with a Pipecat dependency.
+
+  - type: input
+    id: pipecat-version
+    attributes:
+      label: pipecat version
+      description: Which version are you using?
+      placeholder: e.g., 0.0.63
+    validations:
+      required: true
+
+  - type: input
+    id: dependency-name
+    attributes:
+      label: Dependency Name
+      description: Which Pipecat dependency is causing the issue?
+      placeholder: e.g., openai, anthropic, fastapi
+    validations:
+      required: true
+
+  - type: input
+    id: dependency-version
+    attributes:
+      label: Dependency Version
+      description: Which version of the dependency are you using?
+      placeholder: e.g., 1.2.3
+    validations:
+      required: true
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Issue Description
+      description: Provide a clear description of the dependency issue.
+    validations:
+      required: true
+
+  - type: textarea
+    id: impact
+    attributes:
+      label: Impact
+      description: How is this dependency issue affecting your usage of pipecat?
+    validations:
+      required: true
+
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Reproduction Steps
+      description: If applicable, provide steps to reproduce the issue.
+      placeholder: |
+        1. Install dependency X
+        2. Run command Y
+        3. See error Z
+    validations:
+      required: false
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Error Logs
+      description: If applicable, include any relevant error messages or logs.
+      render: shell
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/7-troubleshooting.yml

@@ -0,0 +1,70 @@
+name: Troubleshooting
+description: Help with a specific use case
+type: Troubleshooting
+body:
+  - type: markdown
+    attributes:
+      value: |
+        ## Troubleshooting Request
+
+        Use this form to get help with a specific use case or implementation.
+
+  - type: input
+    id: pipecat-version
+    attributes:
+      label: pipecat version
+      description: Which version are you using?
+      placeholder: e.g., 0.0.63
+    validations:
+      required: true
+
+  - type: input
+    id: python-version
+    attributes:
+      label: Python version
+      description: Which version of Python are you using?
+      placeholder: e.g., 3.12.8
+    validations:
+      required: true
+
+  - type: input
+    id: os
+    attributes:
+      label: Operating System
+      description: Which OS are you using?
+      placeholder: e.g., Ubuntu 24.04, Windows 11, macOS 12.5
+    validations:
+      required: true
+
+  - type: textarea
+    id: use-case
+    attributes:
+      label: Use Case Description
+      description: Describe what you're trying to accomplish with pipecat.
+    validations:
+      required: true
+
+  - type: textarea
+    id: current-approach
+    attributes:
+      label: Current Approach
+      description: What have you tried so far? Include code snippets if relevant.
+      render: python
+    validations:
+      required: true
+
+  - type: textarea
+    id: errors
+    attributes:
+      label: Errors or Unexpected Behavior
+      description: Describe any errors or unexpected behavior you're encountering.
+    validations:
+      required: true
+
+  - type: textarea
+    id: additional-context
+    attributes:
+      label: Additional Context
+      description: Any other information that might help us understand your situation.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: false

.github/workflows/android.yaml

@@ -6,11 +6,13 @@ on:
       - main
     paths:
       - "examples/simple-chatbot/client/android/**"
+      - "examples/p2p-webrtc/video-transform/client/android/**"
   pull_request:
     branches:
       - "**"
     paths:
       - "examples/simple-chatbot/client/android/**"
+      - "examples/p2p-webrtc/video-transform/client/android/**"
   workflow_dispatch:
     inputs:
       sdk_git_ref:
@@ -23,7 +25,7 @@ concurrency:
 
 jobs:
   sdk:
-    name: "Simple chatbot demo"
+    name: "Demo apps"
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
@@ -37,12 +39,22 @@ jobs:
           distribution: 'temurin'
           java-version: '17'
 
-      - name: Build demo app
+      - name: "Example app: Simple Chatbot"
         working-directory: examples/simple-chatbot/client/android
         run: ./gradlew :simple-chatbot-client:assembleDebug
 
-      - name: Upload demo APK
+      - name: Upload Simple Chatbot APK
         uses: actions/upload-artifact@v4
         with:
           name: Simple Chatbot Android Client
           path: examples/simple-chatbot/client/android/simple-chatbot-client/build/outputs/apk/debug/simple-chatbot-client-debug.apk
+
+      - name: "Example app: Small WebRTC Client"
+        working-directory: examples/p2p-webrtc/video-transform/client/android
+        run: ./gradlew :small-webrtc-client:assembleDebug
+
+      - name: Upload Small WebRTC APK
+        uses: actions/upload-artifact@v4
+        with:
+          name: Small WebRTC Android Client
+          path: examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/build/outputs/apk/debug/small-webrtc-client-debug.apk

.github/workflows/format.yaml

@@ -17,7 +17,7 @@ concurrency:
 
 jobs:
   ruff-format:
-    name: "Formatting checker"
+    name: "Code quality checks"
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
@@ -39,8 +39,8 @@ jobs:
         run: |
           source .venv/bin/activate
           ruff format --diff
-      - name: Ruff import linter
+      - name: Ruff linter (all rules)
         id: ruff-check
         run: |
           source .venv/bin/activate
-          ruff check --select I
+          ruff check

.gitignore

@@ -7,7 +7,7 @@ venv
 /.idea
 #*#
 
-# Distribution / packaging
+# Distribution / Packaging
 .Python
 build/
 develop-eggs/
@@ -30,24 +30,24 @@ MANIFEST
 .env
 fly.toml
 
-# Example files
-pipecat/examples/twilio-chatbot/templates/streams.xml
-pipecat/examples/bot-ready-signalling/client/react-native/node_modules/
-pipecat/examples/bot-ready-signalling/client/react-native/.expo/
-pipecat/examples/bot-ready-signalling/client/react-native/dist/
-pipecat/examples/bot-ready-signalling/client/react-native/npm-debug.*
-pipecat/examples/bot-ready-signalling/client/react-native/*.jks
-pipecat/examples/bot-ready-signalling/client/react-native/*.p8
-pipecat/examples/bot-ready-signalling/client/react-native/*.p12
-pipecat/examples/bot-ready-signalling/client/react-native/*.key
-pipecat/examples/bot-ready-signalling/client/react-native/*.mobileprovision
-pipecat/examples/bot-ready-signalling/client/react-native/*.orig.*
-pipecat/examples/bot-ready-signalling/client/react-native/web-build/
+# Examples
+examples/telnyx-chatbot/templates/streams.xml
+examples/twilio-chatbot/templates/streams.xml
+examples/**/node_modules/
+examples/**/.expo/
+examples/**/dist/
+examples/**/npm-debug.*
+examples/**/*.jks
+examples/**/*.p8
+examples/**/*.p12
+examples/**/*.key
+examples/**/*.mobileprovision
+examples/**/*.orig.*
+examples/**/web-build/
 
 # macOS
 .DS_Store
 
-
 # Documentation
 docs/api/_build/
 docs/api/api

CHANGELOG.md

@@ -9,7 +9,959 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
-- Added a new iOS client option to the `SmallWebRTCTransport` **video-transform** example.
+- Added `run_llm` field to `LLMMessagesAppendFrame` and `LLMMessagesUpdateFrame`
+  frames. If true, a context frame will be pushed triggering the LLM to respond.
+
+- Added a new `SOXRStreamAudioResampler` for processing audio in chunks or
+  streams. If you write your own processor and need to use an audio resampler,
+  use the new `create_stream_resampler()`.
+
+- Added new `DailyParams.audio_in_user_tracks` to allow receiving one track per
+  user (default) or a single track from the room (all participants mixed).
+
+- Added support for providing "direct" functions, which don't need an
+  accompanying `FunctionSchema` or function definition dict. Instead, metadata
+  (i.e. `name`, `description`, `properties`, and `required`) are automatically
+  extracted from a combination of the function signature and docstring.
+
+  Usage:
+
+  ```python
+  # "Direct" function
+  # `params` must be the first parameter
+  async def do_something(params: FunctionCallParams, foo: int, bar: str = ""):
+    """
+    Do something interesting.
+
+    Args:
+      foo (int): The foo to do something interesting with.
+      bar (string): The bar to do something interesting with.
+    """
+
+    result = await process(foo, bar)
+    await params.result_callback({"result": result})
+
+  # ...
+
+  llm.register_direct_function(do_something)
+
+  # ...
+
+  tools = ToolsSchema(standard_tools=[do_something])
+  ```
+
+  - `user_id` is now populated in the `TranscriptionFrame` and
+    `InterimTranscriptionFrame` when using a transport that provides a
+    `user_id`, like `DailyTransport` or `LiveKitTransport`.
+
+- Added `watchdog_coroutine()`. This is a watchdog helper for couroutines. So,
+  if you have a coroutine that is waiting for a result and that takes a long
+  time, you will need to wrap it with `watchdog_coroutine()` so the watchdog
+  timers are reset regularly.
+
+- Added `session_token` parameter to `AWSNovaSonicLLMService`.
+
+### Changed
+
+- Updated all the services to use the new `SOXRStreamAudioResampler`, ensuring smooth
+  transitions and eliminating clicks.
+
+- Upgraded `daily-python` to 0.19.4.
+
+- Updated `google` optional dependency to use `google-genai` version `1.24.0`.
+
+### Fixed
+
+- Fixed an issue where audio would get stuck in the queue when an interrupt occurs 
+  during Azure TTS synthesis.
+
+- Fixed a race condition that occurs in Python 3.10+ where the task could miss
+  the `CancelledError` and continue running indefinitely, freezing the pipeline.
+
+- Fixed a `AWSNovaSonicLLMService` issue introduced in 0.0.72.
+
+## [0.0.73] - 2025-06-26
+
+### Fixed
+
+- Fixed an issue introduced in 0.0.72 that would cause `ElevenLabsTTSService`,
+  `GladiaSTTService`, `NeuphonicTTSService` and `OpenAIRealtimeBetaLLMService`
+  to throw an error.
+
+## [0.0.72] - 2025-06-26
+
+### Added
+
+- Added logging and improved error handling to help diagnose and prevent potential
+  Pipeline freezes.
+
+- Added `WatchdogQueue`, `WatchdogPriorityQueue`, `WatchdogEvent` and
+  `WatchdogAsyncIterator`. These helper utilities reset watchdog timers
+  appropriately before they expire. When watchdog timers are disabled, the
+  utilities behave as standard counterparts without side effects.
+
+- Introduce task watchdog timers. Watchdog timers are used to detect if a
+  Pipecat task is taking longer than expected (by default 5 seconds). Watchdog
+  timers are disabled by default and can be enabled globally by passing
+  `enable_watchdog_timers` argument to `PipelineTask` constructor. It is
+  possible to change the default watchdog timer timeout by using the
+  `watchdog_timeout` argument. You can also log how long it takes to reset the
+  watchdog timers which is done with the `enable_watchdog_logging`. You can
+  control all these settings per each frame processor or even per task. That is,
+  you can set `enable_watchdog_timers`, `enable_watchdog_logging` and
+  `watchdog_timeout` when creating any frame processor through their constructor
+  arguments or when you create a task with `FrameProcessor.create_task()`. Note
+  that watchdog timers only work with Pipecat tasks and will not work if you use
+  `asycio.create_task()` or similar.
+
+- Added `lexicon_names` parameter to `AWSPollyTTSService.InputParams`.
+
+- Added reconnection logic and audio buffer management to `GladiaSTTService`.
+
+- The `TurnTrackingObserver` now ends a turn upon observing an `EndFrame` or
+  `CancelFrame`.
+
+- Added Polish support to `AWSTranscribeSTTService`.
+
+- Added new frames `FrameProcessorPauseFrame` and `FrameProcessorResumeFrame`
+  which allow pausing and resuming frame processing for a given frame
+  processor. These are control frames, so they are ordered. Pausing frame
+  processor will keep old frames in the internal queues until resume takes
+  place. Frames being pushed while a frame processor is paused will be pushed to
+  the queues. When frame processing is resumed all queued frames will be
+  processed in order. Also added `FrameProcessorPauseUrgentFrame` and
+  `FrameProcessorResumeUrgentFrame` which are system frames and therefore they
+  have high priority.
+
+- Added a property called `has_function_calls_in_progress` in
+  `LLMAssistantContextAggregator` that exposes whether a function call is in
+  progress.
+
+- Added `SambaNovaLLMService` which provides llm api integration with an
+  OpenAI-compatible interface.
+
+- Added `SambaNovaTTSService` which provides speech-to-text functionality using
+  SambaNovas's (whisper) API.
+
+- Add fundational examples for function calling and transcription
+  `14s-function-calling-sambanova.py`, `13g-sambanova-transcription.py`
+
+### Changed
+
+- `HeartbeatFrame`s are now control frames. This will make it easier to detect
+  pipeline freezes. Previously, heartbeat frames were system frames which meant
+  they were not get queued with other frames, making it difficult to detect
+  pipeline stalls.
+
+- Updated `OpenAIRealtimeBetaLLMService` to accept `language` in the
+  `InputAudioTranscription` class for all models.
+
+- Updated the default model for `OpenAIRealtimeBetaLLMService` to
+  `gpt-4o-realtime-preview-2025-06-03`.
+
+- The `PipelineParams` arg `allow_interruptions` now defaults to `True`.
+
+- `TavusTransport` and `TavusVideoService` now send audio to Tavus using WebRTC
+  audio tracks instead of `app-messages` over WebSocket. This should improve the
+  overall audio quality.
+
+- Upgraded `daily-python` to 0.19.3.
+
+### Fixed
+
+- Fixed an issue that would cause heartbeat frames to be sent before processors
+  were started.
+
+- Fixed an event loop blocking issue when using `SentryMetrics`.
+
+- Fixed an issue in `FastAPIWebsocketClient` to ensure proper disconnection
+  when the websocket is already closed.
+
+- Fixed an issue where the `UserStoppedSpeakingFrame` was not received if the
+  transport was not receiving new audio frames.
+
+- Fixed an edge case where if the user interrupted the bot but no new aggregation
+  was received, the bot would not resume speaking.
+
+- Fixed an issue with `TelnyxFrameSerializer` where it would throw an exception
+  when the user hung up the call.
+
+- Fixed an issue with `ElevenLabsTTSService` where the context was not being
+  closed.
+
+- Fixed function calling in `AWSNovaSonicLLMService`.
+
+- Fixed an issue that would cause multiple `PipelineTask.on_idle_timeout`
+  events to be triggered repeatedly.
+
+- Fixed an issue that was causing user and bot speech to not be synchronized
+  during recordings.
+
+- Fixed an issue where voice settings weren't applied to ElevenLabsTTSService.
+
+- Fixed an issue with `GroqTTSService` where it was not properly parsing the
+  WAV file header.
+
+- Fixed an issue with `GoogleSTTService` where it was constantly reconnecting
+  before starting to receive audio from the user.
+
+- Fixed an issue where `GoogleLLMService`'s TTFB value was incorrect.
+
+### Deprecated
+
+- `AudioBufferProcessor` parameter `user_continuos_stream` is deprecated.
+
+### Other
+
+- Rename `14e-function-calling-gemini.py` to `14e-function-calling-google.py`.
+
+## [0.0.71] - 2025-06-10
+
+### Added
+
+- Adds a parameter called `additional_span_attributes` to PipelineTask that
+  lets you add any additional attributes you'd like to the conversation span.
+
+### Fixed
+
+- Fixed an issue with `CartesiaSTTService` initialization.
+
+## [0.0.70] - 2025-06-10
+
+### Added
+
+- Added `ExotelFrameSerializer` to handle telephony calls via Exotel.
+
+- Added the option `informal` to `TranslationConfig` on Gladia config.
+  Allowing to force informal language forms when available.
+
+- Added `CartesiaSTTService` which is a websocket based implementation to
+  transcribe audio. Added a foundational example in
+  `13f-cartesia-transcription.py`
+
+- Added an `websocket` example, showing how to use the new Pipecat client
+  `WebsocketTransport` to connect with Pipecat `FastAPIWebsocketTransport` or
+  `WebsocketServerTransport`.
+
+- Added language support to `RimeHttpTTSService`. Extended languages to include
+  German and French for both `RimeTTSService` and `RimeHttpTTSService`.
+
+### Changed
+
+- Upgraded `daily-python` to 0.19.2.
+
+- Make `PipelineTask.add_observer()` synchronous. This allows callers to call it
+  before doing the work of running the `PipelineTask` (i.e. without invoking
+  `PipelineTask.set_event_loop()` first).
+
+- Pipecat 0.0.69 forced `uvloop` event loop on Linux on macOS. Unfortunately,
+  this is causing issue in some systems. So, `uvloop` is not enabled by default
+  anymore. If you want to use `uvloop` you can just set the `asyncio` event
+  policy before starting your agent with:
+
+```python
+asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+```
+
+### Fixed
+
+- Fixed an issue with various TTS services that would cause audio glitches at
+  the start of every bot turn.
+
+- Fixed an `ElevenLabsTTSService` issue where a context warning was printed
+  when pushing a `TTSSpeakFrame`.
+
+- Fixed an `AssemblyAISTTService` issue that could cause unexpected behavior
+  when yielding empty `Frame()`s.
+
+- Fixed an issue where `OutputAudioRawFrame.transport_destination` was being
+  reset to `None` instead of retaining its intended value before sending the
+  audio frame to `write_audio_frame`.
+
+- Fixed a typo in Livekit transport that prevented initialization.
+
+## [0.0.69] - 2025-06-02 "AI Engineer World's Fair release" ✨
+
+### Added
+
+- Added a new frame `FunctionCallsStartedFrame`. This frame is pushed both
+  upstream and downstream from the LLM service to indicate that one or more
+  function calls are going to be executed.
+
+- Added LLM services `on_function_calls_started` event. This event will be
+  triggered when the LLM service receives function calls from the model and is
+  going to start executing them.
+
+- Function calls can now be executed sequentially (in the order received in the
+  completion) by passing `run_in_parallel=False` when creating your LLM
+  service. By default, if the LLM completion returns 2 or more function calls
+  they run concurrently. In both cases, concurrently and sequentially, a new LLM
+  completion will run when the last function call finishes.
+
+- Added OpenTelemetry tracing for `GeminiMultimodalLiveLLMService` and
+  `OpenAIRealtimeBetaLLMService`.
+
+- Added initial support for interruption strategies, which determine if the user
+  should interrupt the bot while the bot is speaking. Interruption strategies
+  can be based on factors such as audio volume or the number of words spoken by
+  the user. These can be specified via the new `interruption_strategies` field
+  in `PipelineParams`. A new `MinWordsInterruptionStrategy` strategy has been
+  introduced which triggers an interruption if the user has spoken a minimum
+  number of words. If no interruption strategies are specified, the normal
+  interruption behavior applies. If multiple strategies are provided, the first
+  one that evaluates to true will trigger the interruption.
+
+- `BaseInputTransport` now handles `StopFrame`. When a `StopFrame` is received
+  the transport will pause sending frames downstream until a new `StartFrame` is
+  received. This allows the transport to be reused (keeping the same connection)
+  in a different pipeline.
+
+- Updated AssemblyAI STT service to support their latest streaming
+  speech-to-text model with improved transcription latency and endpointing.
+
+- You can now access STT service results through the new
+  `TranscriptionFrame.result` and `InterimTranscriptionFrame.result` field. This
+  is useful in case you use some specific settings for the STT and you want to
+  access the STT results.
+
+- The examples runner is now public from the `pipecat.examples` package. This
+  allows everyone to build their own examples and run them easily.
+
+- It is now possible to push `OutputDTMFFrame` or `OutputDTMFUrgentFrame` with
+  `DailyTransport`. This will be sent properly if a Daily dial-out connection
+  has been established.
+
+- Added `OutputDTMFUrgentFrame` to send a DTMF keypress quickly. The previous
+  `OutputDTMFFrame` queues the keypress with the rest of data frames.
+
+- Added `DTMFAggregator`, which aggregates keypad presses into
+  `TranscriptionFrame`s. Aggregation occurs after a timeout, termination key
+  press, or user interruption. You can specify the prefix of the
+  `TranscriptionFrame`.
+
+- Added new functions `DailyTransport.start_transcription()` and
+  `DailyTransport.stop_transcription()` to be able to start and stop Daily
+  transcription dynamically (maybe with different settings).
+
+### Changed
+
+- Reverted the default model for `GeminiMultimodalLiveLLMService` back to
+  `models/gemini-2.0-flash-live-001`.
+  `gemini-2.5-flash-preview-native-audio-dialog` has inconsistent performance.
+  You can opt in to using this model by setting the `model` arg.
+
+- Function calls are now cancelled by default if there's an interruption. To
+  disable this behavior you can set `cancel_on_interruption=False` when
+  registering the function call. Since function calls are executed as tasks you
+  can tell if a function call has been cancelled by catching the
+  `asyncio.CancelledError` exception (and don't forget to raise it again!).
+
+- Updated OpenTelemetry tracing attribute `metrics.ttfb_ms` to `metrics.ttfb`.
+  The attribute reports TTFB in seconds.
+
+### Deprecated
+
+- `DailyTransport.send_dtmf()` is deprecated, push an `OutputDTMFFrame` or an
+  `OutputDTMFUrgentFrame` instead.
+
+### Fixed
+
+- Fixed an issue with `ElevenLabsTTSService` where long responses would
+  continue generating output even after an interruption.
+
+- Fixed an issue with the `OpenAILLMContext` where non-Roman characters were
+  being incorrectly encoded as Unicode escape sequences. This was a logging
+  issue and did not impact the actual conversation.
+
+- In `AWSBedrockLLMService`, worked around a possible bug in AWS Bedrock where
+  a `toolConfig` is required if there has been previous tool use in the
+  messages array. This workaround includes a no_op factory function call is
+  used to satisfy the requirement.
+
+- Fixed `WebsocketClientTransport` to use `FrameProcessorSetup.task_manager`
+  instead of `StartFrame.task_manager`.
+
+### Performance
+
+- Use `uvloop` as the new event loop on Linux and macOS systems.
+
+## [0.0.68] - 2025-05-28
+
+### Added
+
+- Added `GoogleHttpTTSService` which uses Google's HTTP TTS API.
+
+- Added `TavusTransport`, a new transport implementation compatible with any
+  Pipecat pipeline. When using the `TavusTransport`the Pipecat bot will
+  connect in the same room as the Tavus Avatar and the user.
+
+- Added `PlivoFrameSerializer` to support Plivo calls. A full running example
+  has also been added to `examples/plivo-chatbot`.
+
+- Added `UserBotLatencyLogObserver`. This is an observer that logs the latency
+  between when the user stops speaking and when the bot starts speaking. This
+  gives you an initial idea on how quickly the AI services respond.
+
+- Added `SarvamTTSService`, which implements Sarvam AI's TTS API:
+  https://docs.sarvam.ai/api-reference-docs/text-to-speech/convert.
+
+- Added `PipelineTask.add_observer()` and `PipelineTask.remove_observer()` to
+  allow mangaging observers at runtime. This is useful for cases where the task
+  is passed around to other code components that might want to observe the
+  pipeline dynamically.
+
+- Added `user_id` field to `TranscriptionMessage`. This allows identifying the
+  user in a multi-user scenario. Note that this requires that
+  `TranscriptionFrame` has the `user_id` properly set.
+
+- Added new `PipelineTask` event handlers `on_pipeline_started`,
+  `on_pipeline_stopped`, `on_pipeline_ended` and `on_pipeline_cancelled`, which
+  correspond to the `StartFrame`, `StopFrame`, `EndFrame` and `CancelFrame`
+  respectively.
+
+- Added additional languages to `LmntTTSService`. Languages include: `hi`,
+  `id`, `it`, `ja`, `nl`, `pl`, `ru`, `sv`, `th`, `tr`, `uk`, `vi`.
+
+- Added a `model` parameter to the `LmntTTSService` constructor, allowing
+  switching between LMNT models.
+
+- Added `MiniMaxHttpTTSService`, which implements MiniMax's T2A API for TTS.
+  Learn more: https://www.minimax.io/platform_overview
+
+- A new function `FrameProcessor.setup()` has been added to allow setting up
+  frame processors before receiving a `StartFrame`. This is what's happening
+  internally: `FrameProcessor.setup()` is called, `StartFrame` is pushed from
+  the beginning of the pipeline, your regular pipeline operations, `EndFrame`
+  or `CancelFrame` are pushed from the beginning of the pipeline and finally
+  `FrameProcessor.cleanup()` is called.
+
+- Added support for OpenTelemetry tracing in Pipecat. This initial
+  implementation includes:
+
+  - A `setup_tracing` method where you can specify your OpenTelemetry exporter
+  - Service decorators for STT (`@traced_stt`), LLM (`@traced_llm`), and TTS
+    (`@traced_tts`) which trace the execution and collect properties and
+    metrics (TTFB, token usage, character counts, etc.)
+  - Class decorators that provide execution tracking; these are generic and can
+    be used for service tracking as needed
+  - Spans that help track traces on a per conversations and turn basis:
+
+  ```
+  conversation-uuid
+  ├── turn-1
+  │   ├── stt_deepgramsttservice
+  │   ├── llm_openaillmservice
+  │   └── tts_cartesiattsservice
+  ...
+  └── turn-n
+      └── ...
+  ```
+
+  By default, Pipecat has implemented service decorators to trace execution of
+  STT, LLM, and TTS services. You can enable tracing by setting
+  `enable_tracing` to `True` in the PipelineTask.
+
+- Added `TurnTrackingObserver`, which tracks the start and end of a user/bot
+  turn pair and emits events `on_turn_started` and `on_turn_stopped`
+  corresponding to the start and end of a turn, respectively.
+
+- Allow passing observers to `run_test()` while running unit tests.
+
+### Changed
+
+- Upgraded `daily-python` to 0.19.1.
+
+- ⚠️ Updated `SmallWebRTCTransport` to align with how other transports handle
+  `on_client_disconnected`. Now, when the connection is closed and no reconnection
+  is attempted, `on_client_disconnected` is called instead of `on_client_close`. The
+  `on_client_close` callback is no longer used, use `on_client_disconnected` instead.
+
+- Check if `PipelineTask` has already been cancelled.
+
+- Don't raise an exception if event handler is not registered.
+
+- Upgraded `deepgram-sdk` to 4.1.0.
+
+- Updated `GoogleTTSService` to use Google's streaming TTS API. The default
+  voice also updated to `en-US-Chirp3-HD-Charon`.
+
+- ⚠️ Refactored the `TavusVideoService`, so it acts like a proxy, sending audio
+  to Tavus and receiving both audio and video. This will make
+  `TavusVideoService` usable with any Pipecat pipeline and with any transport.
+  This is a **breaking change**, check the
+  `examples/foundational/21a-tavus-layer-small-webrtc.py` to see how to use it.
+
+- `DailyTransport` now uses custom microphone audio tracks instead of virtual
+  microphones. Now, multiple Daily transports can be used in the same process.
+
+- `DailyTransport` now captures audio from individual participants instead of
+  the whole room. This allows identifying audio frames per participant.
+
+- Updated the default model for `AnthropicLLMService` to
+  `claude-sonnet-4-20250514`.
+
+- Updated the default model for `GeminiMultimodalLiveLLMService` to
+  `models/gemini-2.5-flash-preview-native-audio-dialog`.
+
+- `BaseTextFilter` methods `filter()`, `update_settings()`,
+  `handle_interruption()` and `reset_interruption()` are now async.
+
+- `BaseTextAggregator` methods `aggregate()`, `handle_interruption()` and
+  `reset()` are now async.
+
+- The API version for `CartesiaTTSService` and `CartesiaHttpTTSService` has
+  been updated. Also, the `cartesia` dependency has been updated to 2.x.
+
+- `CartesiaTTSService` and `CartesiaHttpTTSService` now support Cartesia's new
+  `speed` parameter which accepts values of `slow`, `normal`, and `fast`.
+
+- `GeminiMultimodalLiveLLMService` now uses the user transcription and usage
+  metrics provided by Gemini Live.
+
+- `GoogleLLMService` has been updated to use `google-genai` instead of the
+  deprecated `google-generativeai`.
+
+### Deprecated
+
+- In `CartesiaTTSService` and `CartesiaHttpTTSService`, `emotion` has been
+  deprecated by Cartesia. Pipecat is following suit and deprecating `emotion`
+  as well.
+
+### Removed
+
+- Since `GeminiMultimodalLiveLLMService` now transcribes it's own audio, the
+  `transcribe_user_audio` arg has been removed. Audio is now transcribed
+  automatically.
+
+- Removed `SileroVAD` frame processor, just use `SileroVADAnalyzer`
+  instead. Also removed, `07a-interruptible-vad.py` example.
+
+### Fixed
+
+- Fixed a `DailyTransport` issue that was not allow capturing video frames if
+  framerate was greater than zero.
+
+- Fixed a `DeegramSTTService` connection issue when the user provided their own
+  `LiveOptions`.
+
+- Fixed a `DailyTransport` issue that would cause images needing resize to block
+  the event loop.
+
+- Fixed an issue with `ElevenLabsTTSService` where changing the model or voice
+  while the service is running wasn't working.
+
+- Fixed an issue that would cause multiple instances of the same class to behave
+  incorrectly if any of the given constructor arguments defaulted to a mutable
+  value (e.g. lists, dictionaries, objects).
+
+- Fixed an issue with `CartesiaTTSService` where `TTSTextFrame` messages weren't
+  being emitted when the model was set to `sonic`. This resulted in the
+  assistant context not being updated with assistant messages.
+
+### Performance
+
+- `DailyTransport`: process audio, video and events in separate tasks.
+
+- Don't create event handler tasks if no user event handlers have been
+  registered.
+
+### Other
+
+- It is now possible to run all (or most) foundational example with multiple
+  transports. By default, they run with P2P (Peer-To-Peer) WebRTC so you can try
+  everything locally. You can also run them with Daily or even with a Twilio
+  phone number.
+
+- Added foundation examples `07y-interruptible-minimax.py` and
+  `07z-interruptible-sarvam.py`to show how to use the `MiniMaxHttpTTSService`
+  and `SarvamTTSService`, respectively.
+
+- Added an `open-telemetry-tracing` example, showing how to setup tracing. The
+  example also includes Jaeger as an open source OpenTelemetry client to review
+  traces from the example runs.
+
+- Added foundational example `29-turn-tracking-observer.py` to show how to use
+  the `TurnTrackingObserver`.
+
+## [0.0.67] - 2025-05-07
+
+### Added
+
+- Added `DebugLogObserver` for detailed frame logging with configurable
+  filtering by frame type and endpoint. This observer automatically extracts
+  and formats all frame data fields for debug logging.
+
+- `UserImageRequestFrame.video_source` field has been added to request an image
+  from the desired video source.
+
+- Added support for the AWS Nova Sonic speech-to-speech model with the new
+  `AWSNovaSonicLLMService`.
+  See https://docs.aws.amazon.com/nova/latest/userguide/speech.html.
+  Note that it requires Python >= 3.12 and `pip install pipecat-ai[aws-nova-sonic]`.
+
+- Added new AWS services `AWSBedrockLLMService` and `AWSTranscribeSTTService`.
+
+- Added `on_active_speaker_changed` event handler to the `DailyTransport` class.
+
+- Added `enable_ssml_parsing` and `enable_logging` to `InputParams` in
+  `ElevenLabsTTSService`.
+
+- Added support to `RimeHttpTTSService` for the `arcana` model.
+
+### Changed
+
+- Updated `ElevenLabsTTSService` to use the beta websocket API
+  (multi-stream-input). This new API supports context_ids and cancelling those
+  contexts, which greatly improves interruption handling.
+
+- Observers `on_push_frame()` now take a single argument `FramePushed` instead
+  of multiple arguments.
+
+- Updated the default voice for `DeepgramTTSService` to `aura-2-helena-en`.
+
+### Deprecated
+
+- `PollyTTSService` is now deprecated, use `AWSPollyTTSService` instead.
+
+- Observer `on_push_frame(src, dst, frame, direction, timestamp)` is now
+  deprecated, use `on_push_frame(data: FramePushed)` instead.
+
+### Fixed
+
+- Fixed a `DailyTransport` issue that was causing issues when multiple audio or
+  video sources where being captured.
+
+- Fixed a `UltravoxSTTService` issue that would cause the service to generate
+  all tokens as one word.
+
+- Fixed a `PipelineTask` issue that would cause tasks to not be cancelled if
+  task was cancelled from outside of Pipecat.
+
+- Fixed a `TaskManager` that was causing dangling tasks to be reported.
+
+- Fixed an issue that could cause data to be sent to the transports when they
+  were still not ready.
+
+- Remove custom audio tracks from `DailyTransport` before leaving.
+
+### Removed
+
+- Removed `CanonicalMetricsService` as it's no longer maintained.
+
+## [0.0.66] - 2025-05-02
+
+### Added
+
+- Added two new input parameters to `RimeTTSService`: `pause_between_brackets`
+  and `phonemize_between_brackets`.
+
+- Added support for cross-platform local smart turn detection. You can use
+  `LocalSmartTurnAnalyzer` for on-device inference using Torch.
+
+- `BaseOutputTransport` now allows multiple destinations if the transport
+  implementation supports it (e.g. Daily's custom tracks). With multiple
+  destinations it is possible to send different audio or video tracks with a
+  single transport simultaneously. To do that, you need to set the new
+  `Frame.transport_destination` field with your desired transport destination
+  (e.g. custom track name), tell the transport you want a new destination with
+  `TransportParams.audio_out_destinations` or
+  `TransportParams.video_out_destinations` and the transport should take care of
+  the rest.
+
+- Similar to the new `Frame.transport_destination`, there's a new
+  `Frame.transport_source` field which is set by the `BaseInputTransport` if the
+  incoming data comes from a non-default source (e.g. custom tracks).
+
+- `TTSService` has a new `transport_destination` constructor parameter. This
+  parameter will be used to update the `Frame.transport_destination` field for
+  each generated `TTSAudioRawFrame`. This allows sending multiple bots' audio to
+  multiple destinations in the same pipeline.
+
+- Added `DailyTransportParams.camera_out_enabled` and
+  `DailyTransportParams.microphone_out_enabled` which allows you to
+  enable/disable the main output camera or microphone tracks. This is useful if
+  you only want to use custom tracks and not send the main tracks. Note that you
+  still need `audio_out_enabled=True` or `video_out_enabled`.
+
+- Added `DailyTransport.capture_participant_audio()` which allows you to capture
+  an audio source (e.g. "microphone", "screenAudio" or a custom track name) from
+  a remote participant.
+
+- Added `DailyTransport.update_publishing()` which allows you to update the call
+  video and audio publishing settings (e.g. audio and video quality).
+
+- Added `RTVIObserverParams` which allows you to configure what RTVI messages
+  are sent to the clients.
+
+- Added a `context_window_compression` InputParam to
+  `GeminiMultimodalLiveLLMService` which allows you to enable a sliding context
+  window for the session as well as set the token limit of the sliding window.
+
+- Updated `SmallWebRTCConnection` to support `ice_servers` with credentials.
+
+- Added `VADUserStartedSpeakingFrame` and `VADUserStoppedSpeakingFrame`,
+  indicating when the VAD detected the user to start and stop speaking. These
+  events are helpful when using smart turn detection, as the user's stop time
+  can differ from when their turn ends (signified by UserStoppedSpeakingFrame).
+
+- Added `TranslationFrame`, a new frame type that contains a translated
+  transcription.
+
+- Added `TransportParams.audio_in_passthrough`. If set (the default), incoming
+  audio will be pushed downstream.
+
+- Added `MCPClient`; a way to connect to MCP servers and use the MCP servers'
+  tools.
+
+- Added `Mem0 OSS`, along with Mem0 cloud support now the OSS version is also
+  available.
+
+### Changed
+
+- `TransportParams.audio_mixer` now supports a string and also a dictionary to
+  provide a mixer per destination. For example:
+
+```python
+  audio_out_mixer={
+      "track-1": SoundfileMixer(...),
+      "track-2": SoundfileMixer(...),
+      "track-N": SoundfileMixer(...),
+  },
+```
+
+- The `STTMuteFilter` now mutes `InterimTranscriptionFrame` and
+  `TranscriptionFrame` which allows the `STTMuteFilter` to be used in
+  conjunction with transports that generate transcripts, e.g. `DailyTransport`.
+
+- Function calls now receive a single parameter `FunctionCallParams` instead of
+  `(function_name, tool_call_id, args, llm, context, result_callback)` which is
+  now deprecated.
+
+- Changed the user aggregator timeout for late transcriptions from 1.0s to 0.5s
+  (`LLMUserAggregatorParams.aggregation_timeout`). Sometimes, the STT services
+  might give us more than one transcription which could come after the user
+  stopped speaking. We still want to include these additional transcriptions
+  with the first one because it's part of the user turn. This is what this
+  timeout is helpful with.
+
+- Short utterances not detected by VAD while the bot is speaking are now
+  ignored. This reduces the amount of bot interruptions significantly providing
+  a more natural conversation experience.
+
+- Updated `GladiaSTTService` to output a `TranslationFrame` when specifying a
+  `translation` and `translation_config`.
+
+- STT services now passthrough audio frames by default. This allows you to add
+  audio recording without worrying about what's wrong in your pipeline when it
+  doesn't work the first time.
+
+- Input transports now always push audio downstream unless disabled with
+  `TransportParams.audio_in_passthrough`. After many Pipecat releases, we
+  realized this is the common use case. There are use cases where the input
+  transport already provides STT and you also don't want recordings, in which
+  case there's no need to push audio to the rest of the pipeline, but this is
+  not a very common case.
+
+- Added `RivaSegmentedSTTService`, which allows Riva offline/batch models, such
+  as to be "canary-1b-asr" used in Pipecat.
+
+### Deprecated
+
+- Function calls with parameters
+  `(function_name, tool_call_id, args, llm, context, result_callback)` are
+  deprectated, use a single `FunctionCallParams` parameter instead.
+
+- `TransportParams.camera_*` parameters are now deprecated, use
+  `TransportParams.video_*` instead.
+
+- `TransportParams.vad_enabled` parameter is now deprecated, use
+  `TransportParams.audio_in_enabled` and `TransportParams.vad_analyzer` instead.
+
+- `TransportParams.vad_audio_passthrough` parameter is now deprecated, use
+  `TransportParams.audio_in_passthrough` instead.
+
+- `ParakeetSTTService` is now deprecated, use `RivaSTTService` instead, which uses
+  the model "parakeet-ctc-1.1b-asr" by default.
+
+- `FastPitchTTSService` is now deprecated, use `RivaTTSService` instead, which uses
+  the model "magpie-tts-multilingual" by default.
+
+### Fixed
+
+- Fixed an issue with `SimliVideoService` where the bot was continuously outputting
+  audio, which prevents the `BotStoppedSpeakingFrame` from being emitted.
+
+- Fixed an issue where `OpenAIRealtimeBetaLLMService` would add two assistant
+  messages to the context.
+
+- Fixed an issue with `GeminiMultimodalLiveLLMService` where the context
+  contained tokens instead of words.
+
+- Fixed an issue with HTTP Smart Turn handling, where the service returns a 500
+  error. Previously, this would cause an unhandled exception. Now, a 500 error
+  is treated as an incomplete response.
+
+- Fixed a TTS services issue that could cause assistant output not to be
+  aggregated to the context when also using `TTSSpeakFrame`s.
+
+- Fixed an issue where the `SmartTurnMetricsData` was reporting 0ms for
+  inference and processing time when using the `FalSmartTurnAnalyzer`.
+
+### Other
+
+- Added `examples/daily-custom-tracks` to show how to send and receive Daily
+  custom tracks.
+
+- Added `examples/daily-multi-translation` to showcase how to send multiple
+  simulataneous translations with the same transport.
+
+- Added 04 foundational examples for client/server transports. Also, renamed
+  `29-livekit-audio-chat.py` to `04b-transports-livekit.py`.
+
+- Added foundational example `13c-gladia-translation.py` showing how to use
+  `TranscriptionFrame` and `TranslationFrame`.
+
+## [0.0.65] - 2025-04-23 "Sant Jordi's release" 🌹📕
+
+https://en.wikipedia.org/wiki/Saint_George%27s_Day_in_Catalonia
+
+### Added
+
+- Added automatic hangup logic to the Telnyx serializer. This feature hangs up
+  the Telnyx call when an `EndFrame` or `CancelFrame` is received. It is
+  enabled by default and is configurable via the `auto_hang_up` `InputParam`.
+
+- Added a keepalive task to `GladiaSTTService` to prevent the websocket from
+  disconnecting after 30 seconds of no audio input.
+
+### Changed
+
+- The `InputParams` for `ElevenLabsTTSService` and `ElevenLabsHttpTTSService`
+  no longer require that `stability` and `similarity_boost` be set. You can
+  individually set each param.
+
+- In `TwilioFrameSerializer`, `call_sid` is Optional so as to avoid a breaking
+  changed. `call_sid` is required to automatically hang up.
+
+### Fixed
+
+- Fixed an issue where `TwilioFrameSerializer` would send two hang up commands:
+  one for the `EndFrame` and one for the `CancelFrame`.
+
+## [0.0.64] - 2025-04-22
+
+### Added
+
+- Added automatic hangup logic to the Twilio serializer. This feature hangs up
+  the Twilio call when an `EndFrame` or `CancelFrame` is received. It is
+  enabled by default and is configurable via the `auto_hang_up` `InputParam`.
+
+- Added `SmartTurnMetricsData`, which contains end-of-turn prediction metrics,
+  to the `MetricsFrame`. Using `MetricsFrame`, you can now retrieve prediction
+  confidence scores and processing time metrics from the smart turn analyzers.
+
+- Added support for Application Default Credentials in Google services,
+  `GoogleSTTService`, `GoogleTTSService`, and `GoogleVertexLLMService`.
+
+- Added support for Smart Turn Detection via the `turn_analyzer` transport
+  parameter. You can now choose between `HttpSmartTurnAnalyzer()` or
+  `FalSmartTurnAnalyzer()` for remote inference or
+  `LocalCoreMLSmartTurnAnalyzer()` for on-device inference using Core ML.
+
+- `DeepgramTTSService` accepts `base_url` argument again, allowing you to
+  connect to an on-prem service.
+
+- Added `LLMUserAggregatorParams` and `LLMAssistantAggregatorParams` which allow
+  you to control aggregator settings. You can now pass these arguments when
+  creating aggregator pairs with `create_context_aggregator()`.
+
+- Added `previous_text` context support to ElevenLabsHttpTTSService, improving
+  speech consistency across sentences within an LLM response.
+
+- Added word/timestamp pairs to `ElevenLabsHttpTTSService`.
+
+- It is now possible to disable `SoundfileMixer` when created. You can then use
+  `MixerEnableFrame` to dynamically enable it when necessary.
+
+- Added `on_client_connected` and `on_client_disconnected` event handlers to
+  the `DailyTransport` class. These handlers map to the same underlying Daily
+  events as `on_participant_joined` and `on_participant_left`, respectively.
+  This makes it easier to write a single bot pipeline that can also use other
+  transports like `SmallWebRTCTransport` and `FastAPIWebsocketTransport`.
+
+### Changed
+
+- `GrokLLMService` now uses `grok-3-beta` as its default model.
+
+- Daily's REST helpers now include an `eject_at_token_exp` param, which ejects
+  the user when their token expires. This new parameter defaults to False.
+  Also, the default value for `enable_prejoin_ui` changed to False and
+  `eject_at_room_exp` changed to False.
+
+- `OpenAILLMService` and `OpenPipeLLMService` now use `gpt-4.1` as their
+  default model.
+
+- `SoundfileMixer` constructor arguments need to be keywords.
+
+### Deprecated
+
+- `DeepgramSTTService` parameter `url` is now deprecated, use `base_url`
+  instead.
+
+### Removed
+
+- Parameters `user_kwargs` and `assistant_kwargs` when creating a context
+  aggregator pair using `create_context_aggregator()` have been removed. Use
+  `user_params` and `assistant_params` instead.
+
+### Fixed
+
+- Fixed an issue that would cause TTS websocket-based services to not cleanup
+  resources properly when disconnecting.
+
+- Fixed a `TavusVideoService` issue that was causing audio choppiness.
+
+- Fixed an issue in `SmallWebRTCTransport` where an error was thrown if the
+  client did not create a video transceiver.
+
+- Fixed an issue where LLM input parameters were not working and applied
+  correctly in `GoogleVertexLLMService`, causing unexpected behavior during
+  inference.
+
+### Other
+
+- Updated the `twilio-chatbot` example to use the auto-hangup feature.
+
+## [0.0.63] - 2025-04-11
+
+### Added
+
+- Added media resolution control to `GeminiMultimodalLiveLLMService` with
+  `GeminiMediaResolution` enum, allowing configuration of token usage for
+  image processing (LOW: 64 tokens, MEDIUM: 256 tokens, HIGH: zoomed reframing
+  with 256 tokens).
+
+- Added Gemini's Voice Activity Detection (VAD) configuration to
+  `GeminiMultimodalLiveLLMService` with `GeminiVADParams`, allowing fine
+  control over speech detection sensitivity and timing, including:
+
+  - Start sensitivity (how quickly speech is detected)
+  - End sensitivity (how quickly turns end after pauses)
+  - Prefix padding (milliseconds of audio to keep before speech is detected)
+  - Silence duration (milliseconds of silence required to end a turn)
+
+- Added comprehensive language support to `GeminiMultimodalLiveLLMService`,
+  supporting over 30 languages via the `language` parameter, with proper
+  mapping between Pipecat's `Language` enum and Gemini's language codes.
+
+- Added support in `SmallWebRTCTransport` to detect when remote tracks are
+  muted.
+
+- Added support for image capture from a video stream to the
+  `SmallWebRTCTransport`.
+
+- Added a new iOS client option to the `SmallWebRTCTransport`
+  **video-transform** example.
 
 - Added new processors `ProducerProcessor` and `ConsumerProcessor`. The
   producer processor processes frames from the pipeline and decides whether the
@@ -25,8 +977,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
     type was incorrectly handled as a codec retransmission.
   - Avoid initial video delays.
 
+### Changed
+
+- In `GeminiMultimodalLiveLLMService`, removed the `transcribe_model_audio`
+  parameter in favor of Gemini Live's native output transcription support. Now
+  text transcriptions are produced directly by the model. No configuration is
+  required.
+
+- Updated `GeminiMultimodalLiveLLMService`’s default `model` to
+  `models/gemini-2.0-flash-live-001` and `base_url` to the `v1beta` websocket
+  URL.
+
 ### Fixed
 
+- Updated `daily-python` to 0.17.0 to fix an issue that was preventing to run on
+  older platforms.
+
+- Fixed an issue where `CartesiaTTSService`'s spell feature would result in
+  the spelled word in the context appearing as "F,O,O,B,A,R" instead of
+  "FOOBAR".
+
 - Fixed an issue in the Azure TTS services where the language was being set
   incorrectly.
 
@@ -34,6 +1004,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   `TransportParams.audio_out_10ms_chunks`. Previously, it only worked with 20ms
   chunks.
 
+- Fixed an issue with `GeminiMultimodalLiveLLMService` where the assistant
+  context messages had no space between words.
+
 - Fixed an issue where `LLMAssistantContextAggregator` would prevent a
   `BotStoppedSpeakingFrame` from moving through the pipeline.
 

CONTRIBUTING.md

@@ -41,36 +41,150 @@ We use Ruff for code linting and formatting. Please ensure your code passes all
 
 We follow Google-style docstrings with these specific conventions:
 
-- Class docstrings should fully document all parameters used in `__init__`
-- We don't require separate docstrings for `__init__` methods when parameters are documented in the class docstring
-- Property methods should have docstrings explaining their purpose and return value
+**Regular Classes:**
 
-Example of correctly documented class:
+- Class docstring describes the class purpose and key functionality
+- `__init__` method has its own docstring with complete `Args:` section documenting all parameters
+- All public methods must have docstrings with `Args:` and `Returns:` sections as appropriate
+
+**Dataclasses:**
+
+- Class docstring describes the purpose and documents all fields in a `Parameters:` section
+- No `__init__` docstring (auto-generated)
+
+**Properties:**
+
+- Must have docstrings with `Returns:` section
+
+**Abstract Methods:**
+
+- Must have docstrings explaining what subclasses should implement
+
+**`__init__.py` Files:**
+
+- **Skip docstrings** for pure import/re-export modules
+- **Add brief docstrings** for top-level packages or those with initialization logic
+
+**Enums:**
+
+- Class docstring describes the enumeration purpose
+- Use `Parameters:` section to document each enum value and its meaning
+- No `__init__` docstring (Enums don't have custom constructors)
+
+**Code Examples in Docstrings:**
+
+- Use `Examples:` as a section header for multiple examples
+- Use descriptive text followed by double colons (`::`) for each example
+- **Always include a blank line after the `::"`**
+- Indent all code consistently within each block
+- Separate multiple examples with blank lines for readability
+
+**Lists and Bullets in Docstrings:**
+
+- Use dashes (`-`) for bullet points, not asterisks (`*`)
+- **Add a blank line before bullet lists** when they follow a colon
+- Use section headers like "Supported features:" or "Behavior:" before lists
+- For complex nested information, consider using paragraph format instead
+
+**Deprecations:**
+
+- Use `warnings.warn()` in code for runtime deprecation warnings
+- Add `.. deprecated::` directive in docstrings for documentation visibility
+- Include version information and describe current status
+- Describe parameters in present tense, use directive to indicate deprecation status
+
+#### Examples:
 
 ```python
-class MyClass:
-    """Class description.
+# Regular class
+class MyService(BaseService):
+    """Description of what the service does.
 
-    Additional details about the class.
+    Provides detailed explanation of the service's functionality,
+    key features, and usage patterns.
 
-    Args:
-        param1: Description of first parameter.
-        param2: Description of second parameter.
+    Supported features:
+
+    - Feature one with detailed explanation
+    - Feature two with additional context
+    - Feature three for advanced use cases
     """
 
-    def __init__(self, param1, param2):
-        # No docstring required here as parameters are documented above
-        self.param1 = param1
-        self.param2 = param2
+    def __init__(self, param1: str, old_param: str = None, **kwargs):
+        """Initialize the service.
+
+        Args:
+            param1: Description of param1.
+            old_param: Controls legacy behavior.
+
+                .. deprecated:: 1.2.0
+                    This parameter no longer has any effect and will be removed in version 2.0.
+
+            **kwargs: Additional arguments passed to parent.
+        """
+        if old_param is not None:
+            import warnings
+            warnings.warn(
+                "Parameter 'old_param' is deprecated and will be removed in version 2.0.",
+                DeprecationWarning,
+            )
+        super().__init__(**kwargs)
 
     @property
-    def some_property(self) -> str:
-        """Get the formatted property value.
+    def sample_rate(self) -> int:
+        """Get the current sample rate.
 
         Returns:
-            A string representation of the property.
+            The sample rate in Hz.
         """
-        return f"Property: {self.param1}"
+        return self._sample_rate
+
+    async def process_data(self, data: str) -> bool:
+        """Process the provided data.
+
+        Args:
+            data: The data to process.
+
+        Returns:
+            True if processing succeeded.
+        """
+        pass
+
+# Dataclass with code examples
+@dataclass
+class MessageFrame:
+    """Frame containing messages in OpenAI format.
+
+    Supports both simple and content list message formats.
+
+    Example::
+
+        [
+            {"role": "user", "content": "Hello"},
+            {"role": "assistant", "content": "Hi there!"}
+        ]
+
+    Parameters:
+        messages: List of messages in OpenAI format.
+    """
+
+    messages: List[dict]
+
+# Enum class
+class Status(Enum):
+    """Status codes for processing operations.
+
+    Parameters:
+        PENDING: Operation is queued but not started.
+        RUNNING: Operation is currently in progress.
+        COMPLETED: Operation finished successfully.
+        FAILED: Operation encountered an error.
+    """
+
+    PENDING = "pending"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
 ```
 
 # Contributor Covenant Code of Conduct

MANIFEST.in

@@ -0,0 +1,4 @@
+prune docs
+prune examples
+prune scripts
+prune tests

README.md

@@ -1,43 +1,75 @@
 <h1><div align="center">
- <img alt="pipecat" width="300px" height="auto" src="https://raw.githubusercontent.com/pipecat-ai/pipecat/main/pipecat.png">
+ <img alt="pipecat" width="300px" height="auto" src="https://raw.githubusercontent.com/pipecat-ai/pipecat/main/pipecat.png">
 </div></h1>
 
 [![PyPI](https://img.shields.io/pypi/v/pipecat-ai)](https://pypi.org/project/pipecat-ai) ![Tests](https://github.com/pipecat-ai/pipecat/actions/workflows/tests.yaml/badge.svg) [![codecov](https://codecov.io/gh/pipecat-ai/pipecat/graph/badge.svg?token=LNVUIVO4Y9)](https://codecov.io/gh/pipecat-ai/pipecat) [![Docs](https://img.shields.io/badge/Documentation-blue)](https://docs.pipecat.ai) [![Discord](https://img.shields.io/discord/1239284677165056021)](https://discord.gg/pipecat)
 
-Pipecat is an open source Python framework for building voice and multimodal conversational agents. It handles the complex orchestration of AI services, network transport, audio processing, and multimodal interactions, letting you focus on creating engaging experiences.
+# 🎙️ Pipecat: Real-Time Voice & Multimodal AI Agents
 
-## What you can build
+**Pipecat** is an open-source Python framework for building real-time voice and multimodal conversational agents. Orchestrate audio and video, AI services, different transports, and conversation pipelines effortlessly—so you can focus on what makes your agent unique.
 
-- **Voice Assistants**: [Natural, real-time conversations with AI](https://demo.dailybots.ai/)
-- **Interactive Agents**: Personal coaches and meeting assistants
-- **Multimodal Apps**: Combine voice, video, images, and text
-- **Creative Tools**: [Story-telling experiences](https://storytelling-chatbot.fly.dev/) and social companions
-- **Business Solutions**: [Customer intake flows](https://www.youtube.com/watch?v=lDevgsp9vn0) and support bots
-- **Complex conversational flows**: [Refer to Pipecat Flows](https://github.com/pipecat-ai/pipecat-flows) to learn more
+> Want to dive right in? [Install Pipecat](https://docs.pipecat.ai/getting-started/installation) then try the [quickstart](https://docs.pipecat.ai/getting-started/quickstart).
 
-## See it in action
+## 🚀 What You Can Build
+
+- **Voice Assistants** – natural, streaming conversations with AI
+- **AI Companions** – coaches, meeting assistants, characters
+- **Multimodal Interfaces** – voice, video, images, and more
+- **Interactive Storytelling** – creative tools with generative media
+- **Business Agents** – customer intake, support bots, guided flows
+- **Complex Dialog Systems** – design logic with structured conversations
+
+🧭 Looking to build structured conversations? Check out [Pipecat Flows](https://github.com/pipecat-ai/pipecat-flows) for managing complex conversational states and transitions.
+
+## 🧠 Why Pipecat?
+
+- **Voice-first**: Integrates speech recognition, text-to-speech, and conversation handling
+- **Pluggable**: Supports many AI services and tools
+- **Composable Pipelines**: Build complex behavior from modular components
+- **Real-Time**: Ultra-low latency interaction with different transports (e.g. WebSockets or WebRTC)
+
+## 🎬 See it in action
 
 <p float="left">
-    <a href="https://github.com/pipecat-ai/pipecat/tree/main/examples/simple-chatbot"><img src="https://raw.githubusercontent.com/pipecat-ai/pipecat/main/examples/simple-chatbot/image.png" width="280" /></a>&nbsp;
-    <a href="https://github.com/pipecat-ai/pipecat/tree/main/examples/storytelling-chatbot"><img src="https://raw.githubusercontent.com/pipecat-ai/pipecat/main/examples/storytelling-chatbot/image.png" width="280" /></a>
+    <a href="https://github.com/pipecat-ai/pipecat/tree/main/examples/simple-chatbot"><img src="https://raw.githubusercontent.com/pipecat-ai/pipecat/main/examples/simple-chatbot/image.png" width="400" /></a>&nbsp;
+    <a href="https://github.com/pipecat-ai/pipecat/tree/main/examples/storytelling-chatbot"><img src="https://raw.githubusercontent.com/pipecat-ai/pipecat/main/examples/storytelling-chatbot/image.png" width="400" /></a>
     <br/>
-    <a href="https://github.com/pipecat-ai/pipecat/tree/main/examples/translation-chatbot"><img src="https://raw.githubusercontent.com/pipecat-ai/pipecat/main/examples/translation-chatbot/image.png" width="280" /></a>&nbsp;
-    <a href="https://github.com/pipecat-ai/pipecat/tree/main/examples/moondream-chatbot"><img src="https://raw.githubusercontent.com/pipecat-ai/pipecat/main/examples/moondream-chatbot/image.png" width="280" /></a>
+    <a href="https://github.com/pipecat-ai/pipecat/tree/main/examples/translation-chatbot"><img src="https://raw.githubusercontent.com/pipecat-ai/pipecat/main/examples/translation-chatbot/image.png" width="400" /></a>&nbsp;
+    <a href="https://github.com/pipecat-ai/pipecat/tree/main/examples/moondream-chatbot"><img src="https://raw.githubusercontent.com/pipecat-ai/pipecat/main/examples/moondream-chatbot/image.png" width="400" /></a>
 </p>
 
-## Key features
+## 📱 Client SDKs
 
-- **Voice-first Design**: Built-in speech recognition, TTS, and conversation handling
-- **Flexible Integration**: Works with popular AI services (OpenAI, ElevenLabs, etc.)
-- **Pipeline Architecture**: Build complex apps from simple, reusable components
-- **Real-time Processing**: Frame-based pipeline architecture for fluid interactions
-- **Production Ready**: Enterprise-grade WebRTC and Websocket support
+You can connect to Pipecat from any platform using our official SDKs:
 
-💡 Looking to build structured conversations? Check out [Pipecat Flows](https://github.com/pipecat-ai/pipecat-flows) for managing complex conversational states and transitions.
+| Platform | SDK Repo                                                                       | Description                      |
+| -------- | ------------------------------------------------------------------------------ | -------------------------------- |
+| Web      | [pipecat-client-web](https://github.com/pipecat-ai/pipecat-client-web)         | JavaScript and React client SDKs |
+| iOS      | [pipecat-client-ios](https://github.com/pipecat-ai/pipecat-client-ios)         | Swift SDK for iOS                |
+| Android  | [pipecat-client-android](https://github.com/pipecat-ai/pipecat-client-android) | Kotlin SDK for Android           |
+| C++      | [pipecat-client-cxx](https://github.com/pipecat-ai/pipecat-client-cxx)         | C++ client SDK                   |
 
-## Getting started
+## 🧩 Available services
 
-You can get started with Pipecat running on your local machine, then move your agent processes to the cloud when you’re ready. You can also add a 📞 telephone number, 🖼️ image output, 📺 video input, use different LLMs, and more.
+| Category            | Services                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
+| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Speech-to-Text      | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Parakeet (NVIDIA)](https://docs.pipecat.ai/server/services/stt/parakeet), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova) [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper)                                                                                                                                                                                                                          |
+| LLMs                | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together)                                                 |
+| Text-to-Speech      | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [FastPitch (NVIDIA)](https://docs.pipecat.ai/server/services/tts/fastpitch), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) |
+| Speech-to-Speech    | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
+| Transport           | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
+| Serializers         | [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| Video               | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
+| Memory              | [mem0](https://docs.pipecat.ai/server/services/memory/mem0)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
+| Vision & Image      | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/fal), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| Audio Processing    | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [Noisereduce](https://docs.pipecat.ai/server/utilities/audio/noisereduce-filter)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
+| Analytics & Metrics | [OpenTelemetry](https://docs.pipecat.ai/server/utilities/opentelemetry), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
+
+📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services)
+
+## ⚡ Getting started
+
+You can get started with Pipecat running on your local machine, then move your agent processes to the cloud when you’re ready.
 
 ```shell
 # Install the module
@@ -53,155 +85,71 @@ To keep things lightweight, only the core framework is included by default. If y
 pip install "pipecat-ai[option,...]"
 ```
 
-### Available services
-
-| Category            | Services                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          | Install Command Example                 |
-| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- |
-| Speech-to-Text      | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Parakeet (NVIDIA)](https://docs.pipecat.ai/server/services/stt/parakeet), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper)                                                                                                                                                                                                                                            | `pip install "pipecat-ai[deepgram]"`    |
-| LLMs                | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"`      |
-| Text-to-Speech      | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [FastPitch (NVIDIA)](https://docs.pipecat.ai/server/services/tts/fastpitch), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts)                       | `pip install "pipecat-ai[cartesia]"`    |
-| Speech-to-Speech    | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               | `pip install "pipecat-ai[google]"`      |
-| Transport           | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        | `pip install "pipecat-ai[daily]"`       |
-| Video               | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        | `pip install "pipecat-ai[tavus,simli]"` |
-| Memory              | [mem0](https://docs.pipecat.ai/server/services/memory/mem0)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       | `pip install "pipecat-ai[mem0]"`        |
-| Vision & Image      | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/fal), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         | `pip install "pipecat-ai[moondream]"`   |
-| Audio Processing    | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [Noisereduce](https://docs.pipecat.ai/server/utilities/audio/noisereduce-filter)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    | `pip install "pipecat-ai[silero]"`      |
-| Analytics & Metrics | [Canonical AI](https://docs.pipecat.ai/server/services/analytics/canonical), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | `pip install "pipecat-ai[canonical]"`   |
-
-📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services)
-
-## Code examples
+## 🧪 Code examples
 
 - [Foundational](https://github.com/pipecat-ai/pipecat/tree/main/examples/foundational) — small snippets that build on each other, introducing one or two concepts at a time
 - [Example apps](https://github.com/pipecat-ai/pipecat/tree/main/examples/) — complete applications that you can use as starting points for development
 
-## A simple voice agent running locally
+## 🛠️ Hacking on the framework itself
 
-Here is a very basic Pipecat bot that greets a user when they join a real-time session. We'll use [Daily](https://daily.co) for real-time media transport, and [Cartesia](https://cartesia.ai/) for text-to-speech.
+1. Set up a virtual environment before following these instructions. From the root of the repo:
 
-```python
-import asyncio
+   ```shell
+   python3 -m venv venv
+   source venv/bin/activate
+   ```
 
-from pipecat.frames.frames import TextFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.task import PipelineTask
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.services.cartesia import CartesiaTTSService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
+2. Install the development dependencies:
 
-async def main():
-  # Use Daily as a real-time media transport (WebRTC)
-  transport = DailyTransport(
-    room_url=...,
-    token="", # leave empty. Note: token is _not_ your api key
-    bot_name="Bot Name",
-    params=DailyParams(audio_out_enabled=True))
+   ```shell
+   pip install -r dev-requirements.txt
+   ```
 
-  # Use Cartesia for Text-to-Speech
-  tts = CartesiaTTSService(
-    api_key=...,
-    voice_id=...
-  )
+3. Install the git pre-commit hooks (these help ensure your code follows project rules):
 
-  # Simple pipeline that will process text to speech and output the result
-  pipeline = Pipeline([tts, transport.output()])
+   ```shell
+   pre-commit install
+   ```
 
-  # Create Pipecat processor that can run one or more pipelines tasks
-  runner = PipelineRunner()
+4. Install the `pipecat-ai` package locally in editable mode:
 
-  # Assign the task callable to run the pipeline
-  task = PipelineTask(pipeline)
+   ```shell
+   pip install -e .
+   ```
 
-  # Register an event handler to play audio when a
-  # participant joins the transport WebRTC session
-  @transport.event_handler("on_first_participant_joined")
-  async def on_first_participant_joined(transport, participant):
-    participant_name = participant.get("info", {}).get("userName", "")
-    # Queue a TextFrame that will get spoken by the TTS service (Cartesia)
-    await task.queue_frame(TextFrame(f"Hello there, {participant_name}!"))
+   > The `-e` or `--editable` option allows you to modify the code without reinstalling.
 
-  # Register an event handler to exit the application when the user leaves.
-  @transport.event_handler("on_participant_left")
-  async def on_participant_left(transport, participant, reason):
-    await task.cancel()
+5. Include optional dependencies as needed. For example:
 
-  # Run the pipeline task
-  await runner.run(task)
+   ```shell
+   pip install -e ".[daily,deepgram,cartesia,openai,silero]"
+   ```
 
-if __name__ == "__main__":
-  asyncio.run(main())
-```
+6. (Optional) If you want to use this package from another directory:
 
-Run it with:
-
-```shell
-python app.py
-```
-
-Daily provides a prebuilt WebRTC user interface. While the app is running, you can visit at `https://<yourdomain>.daily.co/<room_url>` and listen to the bot say hello!
-
-## WebRTC for production use
-
-WebSockets are fine for server-to-server communication or for initial development. But for production use, you’ll need client-server audio to use a protocol designed for real-time media transport. (For an explanation of the difference between WebSockets and WebRTC, see [this post.](https://www.daily.co/blog/how-to-talk-to-an-llm-with-your-voice/#webrtc))
-
-One way to get up and running quickly with WebRTC is to sign up for a Daily developer account. Daily gives you SDKs and global infrastructure for audio (and video) routing. Every account gets 10,000 audio/video/transcription minutes free each month.
-
-Sign up [here](https://dashboard.daily.co/u/signup) and [create a room](https://docs.daily.co/reference/rest-api/rooms) in the developer Dashboard.
-
-## Hacking on the framework itself
-
-_Note: You may need to set up a virtual environment before following these instructions. From the root of the repo:_
-
-```shell
-python3 -m venv venv
-source venv/bin/activate
-```
-
-Install the development dependencies:
-
-```shell
-pip install -r dev-requirements.txt
-```
-
-Install the git pre-commit hooks (these help ensure your code follows project rules):
-
-```shell
-pre-commit install
-```
-
-Install the `pipecat-ai` package locally in editable mode:
-
-```shell
-pip install -e .
-```
-
-The `-e` or `--editable` option allows you to modify the code without reinstalling.
-
-To include optional dependencies, add them to the install command. For example:
-
-```shell
-pip install -e ".[daily,deepgram,cartesia,openai,silero]"     # Updated for the services you're using
-```
-
-If you want to use this package from another directory:
-
-```shell
-pip install "path_to_this_repo[option,...]"
-```
+   ```shell
+   pip install "path_to_this_repo[option,...]"
+   ```
 
 ### Running tests
 
+Install the test dependencies:
+
+```shell
+pip install -r test-requirements.txt
+```
+
 From the root directory, run:
 
 ```shell
 pytest
 ```
 
-## Setting up your editor
+### Setting up your editor
 
 This project uses strict [PEP 8](https://peps.python.org/pep-0008/) formatting via [Ruff](https://github.com/astral-sh/ruff).
 
-### Emacs
+#### Emacs
 
 You can use [use-package](https://github.com/jwiegley/use-package) to install [emacs-lazy-ruff](https://github.com/christophermadsen/emacs-lazy-ruff) package and configure `ruff` arguments:
 
@@ -223,7 +171,7 @@ You can use [use-package](https://github.com/jwiegley/use-package) to install [e
   :hook ((python-mode . pyvenv-auto-run)))
 ```
 
-### Visual Studio Code
+#### Visual Studio Code
 
 Install the
 [Ruff](https://marketplace.visualstudio.com/items?itemName=charliermarsh.ruff) extension. Then edit the user settings (_Ctrl-Shift-P_ `Open User Settings (JSON)`) and set it as the default Python formatter, and enable formatting on save:
@@ -235,7 +183,7 @@ Install the
 }
 ```
 
-### PyCharm
+#### PyCharm
 
 `ruff` was installed in the `venv` environment described before, now to enable autoformatting on save, go to `File` -> `Settings` -> `Tools` -> `File Watchers` and add a new watcher with the following settings:
 
@@ -245,7 +193,7 @@ Install the
 4. **Arguments**: `format $FilePath$`
 5. **Program**: `$PyInterpreterDirectory$/ruff`
 
-## Contributing
+## 🤝 Contributing
 
 We welcome contributions from the community! Whether you're fixing bugs, improving documentation, or adding new features, here's how you can help:
 
@@ -258,7 +206,7 @@ Before submitting a pull request, please check existing issues and PRs to avoid
 
 We aim to review all contributions promptly and provide constructive feedback to help get your changes merged.
 
-## Getting help
+## 🛟 Getting help
 
 ➡️ [Join our Discord](https://discord.gg/pipecat)
 

dev-requirements.txt

@@ -1,13 +1,13 @@
 build~=1.2.2
-coverage~=7.6.12
+coverage~=7.9.1
 grpcio-tools~=1.67.1
 pip-tools~=7.4.1
-pre-commit~=4.0.1
-pyright~=1.1.397
-pytest~=8.3.4
-pytest-asyncio~=0.25.3
+pre-commit~=4.2.0
+pyright~=1.1.402
+pytest~=8.4.1
+pytest-asyncio~=1.0.0
 pytest-aiohttp==1.1.0
-ruff~=0.11.1
-setuptools~=70.0.0
-setuptools_scm~=8.1.0
-python-dotenv~=1.0.1
+ruff~=0.12.1
+setuptools~=78.1.1
+setuptools_scm~=8.3.1
+python-dotenv~=1.1.1

docs/ISSUE_TEMPLATE.md

@@ -1,22 +0,0 @@
-# Description
-Is this reporting a bug or feature request?
-
-
-If reporting a bug, please fill out the following:
-
-### Environment
-- pipecat-ai version:
-- python version:
-- OS:
-
-### Issue description
-Provide a clear description of the issue.
-
-### Repro steps
-List the steps to reproduce the issue.
-
-### Expected behavior
-
-### Actual behavior
-
-### Logs

docs/api/conf.py

@@ -1,5 +1,6 @@
 import logging
 import sys
+from datetime import datetime
 from pathlib import Path
 
 # Configure logging
@@ -13,7 +14,8 @@ sys.path.insert(0, str(project_root / "src"))
 
 # Project information
 project = "pipecat-ai"
-copyright = "2024, Daily"
+current_year = datetime.now().year
+copyright = f"2024-{current_year}, Daily" if current_year > 2024 else "2024, Daily"
 author = "Daily"
 
 # General configuration
@@ -24,19 +26,20 @@ extensions = [
     "sphinx.ext.intersphinx",
 ]
 
+suppress_warnings = [
+    "autodoc.mocked_object",
+]
+
 # Napoleon settings
 napoleon_google_docstring = True
-napoleon_numpy_docstring = False
 napoleon_include_init_with_doc = True
 
 # AutoDoc settings
 autodoc_default_options = {
     "members": True,
     "member-order": "bysource",
-    "special-members": "__init__",
-    "undoc-members": True,
-    "exclude-members": "__weakref__",
-    "no-index": True,
+    "undoc-members": False,
+    "exclude-members": "__weakref__,model_config",
     "show-inheritance": True,
 }
 
@@ -50,7 +53,6 @@ autodoc_mock_imports = [
     "pyht.protos",
     "pyht.protos.api_pb2",
     "pipecat_ai_playht",  # PlayHT wrapper
-    "vllm",
     "aiortc",
     "aiortc.mediastreams",
     "cv2",
@@ -72,82 +74,145 @@ autodoc_mock_imports = [
     "langchain",
     "lmnt",
     "noisereduce",
-    "openai",
     "openpipe",
     "simli",
     "soundfile",
-    # Existing mocks
     "pipecat_ai_krisp",
     "pyaudio",
     "_tkinter",
     "tkinter",
     "daily",
     "daily_python",
-    "pydantic.BaseModel",
-    "pydantic.Field",
-    "pydantic._internal._model_construction",
-    "pydantic._internal._fields",
+    # Moondream dependencies
+    "torch",
+    "transformers",
+    "intel_extension_for_pytorch",
+    # Ultravox dependencies
+    "huggingface_hub",
+    "vllm",
+    "vllm.engine.arg_utils",
+    "transformers.AutoTokenizer",
+    # Langchain dependencies
+    "langchain_core",
+    "langchain_core.messages",
+    "langchain_core.runnables",
+    "langchain_core.messages.AIMessageChunk",
+    "langchain_core.runnables.Runnable",
+    # LiveKit dependencies
+    "livekit",
+    "livekit.rtc",
+    "livekit_api",
+    "livekit_protocol",
+    "tenacity",
+    "tenacity.retry",
+    "tenacity.stop_after_attempt",
+    "tenacity.wait_exponential",
+    "rtc",
+    "rtc.Room",
+    "rtc.RoomOptions",
+    "rtc.AudioSource",
+    "rtc.LocalAudioTrack",
+    "rtc.TrackPublishOptions",
+    "rtc.TrackSource",
+    "rtc.AudioStream",
+    "rtc.AudioFrameEvent",
+    "rtc.AudioFrame",
+    "rtc.Track",
+    "rtc.TrackKind",
+    "rtc.RemoteParticipant",
+    "rtc.RemoteTrackPublication",
+    "rtc.DataPacket",
+    # Riva dependencies
+    "riva",
+    "riva.client",
+    "riva.client.Auth",
+    "riva.client.ASRService",
+    "riva.client.StreamingRecognitionConfig",
+    "riva.client.RecognitionConfig",
+    "riva.client.AudioEncoding",
+    "riva.client.proto.riva_tts_pb2",
+    "riva.client.SpeechSynthesisService",
+    # Local CoreML Smart Turn dependencies
+    "coremltools",
+    "coremltools.models",
+    "coremltools.models.MLModel",
+    "torch",
+    "torch.nn",
+    "torch.nn.functional",
+    "transformers",
+    "transformers.AutoFeatureExtractor",
+    # Also add specific classes that are imported
+    "AutoFeatureExtractor",
+    # Sentry dependencies
+    "sentry_sdk",
+    # AWS Nova Sonic dependencies
+    "aws_sdk_bedrock_runtime",
+    "aws_sdk_bedrock_runtime.client",
+    "aws_sdk_bedrock_runtime.config",
+    "aws_sdk_bedrock_runtime.models",
+    "smithy_aws_core",
+    "smithy_aws_core.credentials_resolvers",
+    "smithy_aws_core.credentials_resolvers.static",
+    "smithy_aws_core.identity",
+    "smithy_core",
+    "smithy_core.aio",
+    "smithy_core.aio.eventstream",
+    # MCP dependencies (you may already have these)
+    "mcp",
+    "mcp.client",
+    "mcp.client.session_group",
+    "mcp.client.sse",
+    "mcp.client.stdio",
+    "mcp.ClientSession",
+    "mcp.StdioServerParameters",
+    # gstreamer
+    "gi",
+    "gi.require_version",
+    "gi.repository",
+    # Protobuf mocks
+    "pipecat.frames.protobufs.frames_pb2",
+    "pipecat.serializers.protobuf",
+    "google.protobuf",
+    "google.protobuf.descriptor",
+    "google.protobuf.descriptor_pool",
+    "google.protobuf.runtime_version",
+    "google.protobuf.symbol_database",
+    "google.protobuf.internal.builder",
 ]
 
 # HTML output settings
 html_theme = "sphinx_rtd_theme"
 html_static_path = ["_static"]
-autodoc_typehints = "description"
+autodoc_typehints = "signature"  # Show type hints in the signature only, not in the docstring
 html_show_sphinx = False
 
 
-def verify_modules():
-    """Verify that required modules are available."""
-    required_modules = {
-        "services": [
-            "assemblyai",
-            "aws",
-            "cartesia",
-            "deepgram",
-            "google",
-            "lmnt",
-            "riva",
-            "simli",
-        ],
-        "serializers": ["livekit"],
-        "vad": ["silero", "vad_analyzer"],
-        "transports": {
-            "services": ["daily", "livekit"],
-            "local": ["audio", "tk"],
-            "network": ["fastapi_websocket", "websocket_server"],
-        },
-    }
+def import_core_modules():
+    """Import core pipecat modules for autodoc to discover."""
+    core_modules = [
+        "pipecat",
+        "pipecat.frames",
+        "pipecat.pipeline",
+        "pipecat.processors",
+        "pipecat.services",
+        "pipecat.transports",
+        "pipecat.audio",
+        "pipecat.adapters",
+        "pipecat.clocks",
+        "pipecat.metrics",
+        "pipecat.observers",
+        "pipecat.serializers",
+        "pipecat.sync",
+        "pipecat.transcriptions",
+        "pipecat.utils",
+    ]
 
-    missing = []
-    for category, modules in required_modules.items():
-        if isinstance(modules, dict):
-            # Handle nested structure
-            for subcategory, submodules in modules.items():
-                for module in submodules:
-                    try:
-                        __import__(f"pipecat.{category}.{subcategory}.{module}")
-                        logger.info(
-                            f"Successfully imported pipecat.{category}.{subcategory}.{module}"
-                        )
-                    except (ImportError, TypeError, NameError) as e:
-                        missing.append(f"pipecat.{category}.{subcategory}.{module}")
-                        logger.warning(
-                            f"Optional module not available: pipecat.{category}.{subcategory}.{module} - {str(e)}"
-                        )
-        else:
-            # Handle flat structure
-            for module in modules:
-                try:
-                    __import__(f"pipecat.{category}.{module}")
-                    logger.info(f"Successfully imported pipecat.{category}.{module}")
-                except (ImportError, TypeError, NameError) as e:
-                    missing.append(f"pipecat.{category}.{module}")
-                    logger.warning(
-                        f"Optional module not available: pipecat.{category}.{module} - {str(e)}"
-                    )
-
-    if missing:
-        logger.warning(f"Some optional modules are not available: {missing}")
+    for module_name in core_modules:
+        try:
+            __import__(module_name)
+            logger.info(f"Successfully imported {module_name}")
+        except ImportError as e:
+            logger.warning(f"Failed to import {module_name}: {e}")
 
 
 def clean_title(title: str) -> str:
@@ -159,36 +224,7 @@ def clean_title(title: str) -> str:
     parts = title.split(".")
     title = parts[-1]
 
-    # Special cases for service names and common acronyms
-    special_cases = {
-        "ai": "AI",
-        "aws": "AWS",
-        "api": "API",
-        "vad": "VAD",
-        "assemblyai": "AssemblyAI",
-        "deepgram": "Deepgram",
-        "elevenlabs": "ElevenLabs",
-        "openai": "OpenAI",
-        "openpipe": "OpenPipe",
-        "playht": "PlayHT",
-        "xtts": "XTTS",
-        "lmnt": "LMNT",
-    }
-
-    # Check if the entire title is a special case
-    if title.lower() in special_cases:
-        return special_cases[title.lower()]
-
-    # Otherwise, capitalize each word
-    words = title.split("_")
-    cleaned_words = []
-    for word in words:
-        if word.lower() in special_cases:
-            cleaned_words.append(special_cases[word.lower()])
-        else:
-            cleaned_words.append(word.capitalize())
-
-    return " ".join(cleaned_words)
+    return title
 
 
 def setup(app):
@@ -213,9 +249,8 @@ def setup(app):
 
     excludes = [
         str(project_root / "src/pipecat/pipeline/to_be_updated"),
-        str(project_root / "src/pipecat/processors/gstreamer"),
-        str(project_root / "src/pipecat/services/to_be_updated"),
-        str(project_root / "src/pipecat/vad"),  # deprecated
+        str(project_root / "src/pipecat/examples"),
+        str(project_root / "src/pipecat/tests"),
         "**/test_*.py",
         "**/tests/*.py",
     ]
@@ -256,5 +291,4 @@ def setup(app):
         logger.error(f"Error generating API documentation: {e}", exc_info=True)
 
 
-# Run module verification
-verify_modules()
+import_core_modules()

docs/api/index.rst

@@ -1,57 +1,17 @@
-Pipecat API Reference Docs
-==========================
+Pipecat API Reference
+=====================
 
-Welcome to Pipecat's API reference documentation!
+Welcome to the Pipecat API reference.
 
-Pipecat is an open source framework for building voice and multimodal assistants.
-It provides a flexible pipeline architecture for connecting various AI services,
-audio processing, and transport layers.
+Use the navigation on the left to browse modules, or search using the search box.
+
+**New to Pipecat?** Check out the `main documentation <https://docs.pipecat.ai>`_ for tutorials, guides, and client SDK information.
 
 Quick Links
 -----------
 
 * `GitHub Repository <https://github.com/pipecat-ai/pipecat>`_
-* `Website <https://pipecat.ai>`_
-
-API Reference
--------------
-
-Core Components
-~~~~~~~~~~~~~~~
-
-* :mod:`Frames <pipecat.frames>`
-* :mod:`Processors <pipecat.processors>`
-* :mod:`Pipeline <pipecat.pipeline>`
-
-Audio Processing
-~~~~~~~~~~~~~~~~
-
-* :mod:`Audio <pipecat.audio>`
-
-Services
-~~~~~~~~
-
-* :mod:`Services <pipecat.services>`
-
-Transport & Serialization
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-* :mod:`Transports <pipecat.transports>`
-   * :mod:`Local <pipecat.transports.local>`
-   * :mod:`Network <pipecat.transports.network>`
-   * :mod:`Services <pipecat.transports.services>`
-* :mod:`Serializers <pipecat.serializers>`
-
-Utilities
-~~~~~~~~~
-
-* :mod:`Adapters <pipecat.adapters>`
-* :mod:`Clocks <pipecat.clocks>`
-* :mod:`Metrics <pipecat.metrics>`
-* :mod:`Observers <pipecat.observers>`
-* :mod:`Sync <pipecat.sync>`
-* :mod:`Transcriptions <pipecat.transcriptions>`
-* :mod:`Utils <pipecat.utils>`
+* `Join our Community <https://discord.gg/pipecat>`_
 
 .. toctree::
    :maxdepth: 3
@@ -71,11 +31,4 @@ Utilities
    Sync <api/pipecat.sync>
    Transcriptions <api/pipecat.transcriptions>
    Transports <api/pipecat.transports>
-   Utils <api/pipecat.utils>
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
+   Utils <api/pipecat.utils>

docs/api/requirements.txt

@@ -10,7 +10,6 @@ pipecat-ai[anthropic]
 pipecat-ai[assemblyai]
 pipecat-ai[aws]
 pipecat-ai[azure]
-pipecat-ai[canonical]
 pipecat-ai[cartesia]
 pipecat-ai[cerebras]
 pipecat-ai[deepseek]
@@ -26,20 +25,24 @@ pipecat-ai[grok]
 pipecat-ai[groq]
 # pipecat-ai[krisp] # Mocked
 pipecat-ai[koala]
-pipecat-ai[langchain]
-pipecat-ai[livekit]
+# pipecat-ai[langchain] # Mocked
+# pipecat-ai[livekit] # Mocked
 pipecat-ai[lmnt]
 pipecat-ai[local]
+# pipecat-ai[local-smart-turn] # Mocked
 # pipecat-ai[mem0] # Mocked
 # pipecat-ai[mlx-whisper] # Mocked
-pipecat-ai[moondream]
+# pipecat-ai[moondream] # Mocked
 pipecat-ai[nim]
 # pipecat-ai[neuphonic] # Mocked
 pipecat-ai[noisereduce]
 pipecat-ai[openai]
 # pipecat-ai[openpipe]
 # pipecat-ai[playht] # Mocked due to grpcio conflict with riva
-pipecat-ai[riva]
+pipecat-ai[qwen]
+pipecat-ai[remote-smart-turn]
+# pipecat-ai[riva] # Mocked
+pipecat-ai[sambanova]
 pipecat-ai[silero]
 pipecat-ai[simli]
 pipecat-ai[soundfile]

dot-env.template

@@ -92,4 +92,25 @@ ASSEMBLYAI_API_KEY=...
 OPENROUTER_API_KEY=...
 
 # Piper
-PIPER_BASE_URL=...
+PIPER_BASE_URL=...
+
+# Smart turn
+LOCAL_SMART_TURN_MODEL_PATH=...
+FAL_SMART_TURN_API_KEY=...
+
+# Twilio
+TWILIO_ACCOUNT_SID=...
+TWILIO_AUTH_TOKEN=...
+
+# MiniMax
+MINIMAX_API_KEY=...
+MINIMAX_GROUP_ID=...
+
+# Sarvam AI
+SARVAM_API_KEY=...
+
+# SambaNova
+SAMBANOVA_API_KEY=...
+
+# Sentry
+SENTRY_DSN=...

examples/bot-ready-signalling/client/javascript/package.json

@@ -12,7 +12,7 @@
   "license": "ISC",
   "description": "",
   "devDependencies": {
-    "vite": "^6.0.9"
+    "vite": "^6.3.5"
   },
   "dependencies": {
     "@daily-co/daily-js": "0.74.0"

examples/canonical-metrics/README.md

@@ -1,66 +0,0 @@
-# Chatbot with canonical-metrics
-
-This project implements a chatbot using a pipeline architecture that integrates audio processing, transcription, and a language model for conversational interactions. The chatbot operates within a daily communication environment, utilizing various services for text-to-speech and language model responses.
-
-## Features
-
-- **Audio Input and Output**: Captures microphone input and plays back audio responses.
-- **Voice Activity Detection**: Utilizes Silero VAD to manage audio input intelligently.
-- **Text-to-Speech**: Integrates ElevenLabs TTS service to convert text responses into audio.
-- **Language Model Interaction**: Uses OpenAI's GPT-4 model to generate responses based on user input.
-- **Transcription Services**: Captures and transcribes participant speech for analytics.
-- **Metrics Collection**: Sends audio data for analysis via Canonical Metrics Service.
-
-## Requirements
-
-- Python 3.10+
-- `python-dotenv`
-- Additional libraries from the `pipecat` package.
-
-## Setup
-
-1. Clone the repository.
-2. Install the required packages.
-3. Set up environment variables for API keys:
-   - `OPENAI_API_KEY`
-   - `ELEVENLABS_API_KEY`
-   - `CANONICAL_API_KEY`
-   - `CANONICAL_API_URL`
-4. Run the script.
-
-## Usage
-
-The chatbot introduces itself and engages in conversations, providing brief and creative responses. Designed for flexibility, it can support multiple languages with appropriate configuration.
-
-## Events
-
-- Participants joining or leaving the call are handled dynamically, adjusting the chatbot's behavior accordingly.
-
-
-ℹ️ The first time, things might take extra time to get started since VAD (Voice Activity Detection) model needs to be downloaded.
-
-## Get started
-
-```python
-python3 -m venv venv
-source venv/bin/activate
-pip install -r requirements.txt
-
-cp env.example .env # and add your credentials
-
-```
-
-## Run the server
-
-```bash
-python server.py
-```
-
-Then, visit `http://localhost:7860/` in your browser to start a chatbot session.
-
-## Build and test the Docker image
-
-```
-docker build -t chatbot .
-docker run --env-file .env -p 7860:7860 chatbot
-```

examples/canonical-metrics/bot.py

@@ -1,148 +0,0 @@
-#
-# Copyright (c) 2024–2025, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import os
-import sys
-import uuid
-
-import aiohttp
-from dotenv import load_dotenv
-from loguru import logger
-from runner import configure
-
-from pipecat.audio.vad.silero import SileroVADAnalyzer
-from pipecat.frames.frames import EndFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
-from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor
-from pipecat.services.canonical.metrics import CanonicalMetricsService
-from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
-from pipecat.services.openai.llm import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def main():
-    async with aiohttp.ClientSession() as session:
-        (room_url, token) = await configure(session)
-
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Chatbot",
-            DailyParams(
-                audio_out_enabled=True,
-                audio_in_enabled=True,
-                camera_out_enabled=False,
-                vad_enabled=True,
-                vad_audio_passthrough=True,
-                vad_analyzer=SileroVADAnalyzer(),
-                transcription_enabled=True,
-                #
-                # Spanish
-                #
-                # transcription_settings=DailyTranscriptionSettings(
-                #     language="es",
-                #     tier="nova",
-                #     model="2-general"
-                # )
-            ),
-        )
-
-        tts = ElevenLabsTTSService(
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            #
-            # English
-            #
-            voice_id="cgSgspJ2msm6clMCkdW9",
-            #
-            # Spanish
-            #
-            # model="eleven_multilingual_v2",
-            # voice_id="gD1IexrzCvsXPHUuT0s3",
-        )
-
-        llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
-
-        messages = [
-            {
-                "role": "system",
-                #
-                # English
-                #
-                "content": "You are Chatbot, a friendly, helpful robot. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way, but keep your responses brief. Start by introducing yourself. Keep all your responses to 12 words or fewer.",
-                #
-                # Spanish
-                #
-                # "content": "Eres Chatbot, un amigable y útil robot. Tu objetivo es demostrar tus capacidades de una manera breve. Tus respuestas se convertiran a audio así que nunca no debes incluir caracteres especiales. Contesta a lo que el usuario pregunte de una manera creativa, útil y breve. Empieza por presentarte a ti mismo.",
-            },
-        ]
-
-        context = OpenAILLMContext(messages)
-        context_aggregator = llm.create_context_aggregator(context)
-
-        """
-        CanonicalMetrics uses AudioBufferProcessor under the hood to buffer the audio. On
-        call completion, CanonicalMetrics will send the audio buffer to Canonical for
-        analysis. Visit https://voice.canonical.chat to learn more.
-        """
-        audio_buffer_processor = AudioBufferProcessor(num_channels=2)
-        canonical = CanonicalMetricsService(
-            audio_buffer_processor=audio_buffer_processor,
-            aiohttp_session=session,
-            api_key=os.getenv("CANONICAL_API_KEY"),
-            call_id=str(uuid.uuid4()),
-            assistant="pipecat-chatbot",
-            assistant_speaks_first=True,
-            context=context,
-        )
-        pipeline = Pipeline(
-            [
-                transport.input(),  # microphone
-                context_aggregator.user(),
-                llm,
-                tts,
-                transport.output(),
-                canonical,  # uploads audio buffer to Canonical AI for metrics
-                audio_buffer_processor,  # captures audio into a buffer
-                context_aggregator.assistant(),
-            ]
-        )
-
-        task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            await audio_buffer_processor.start_recording()
-            await transport.capture_participant_transcription(participant["id"])
-            await task.queue_frames([context_aggregator.user().get_context_frame()])
-
-        @transport.event_handler("on_participant_left")
-        async def on_participant_left(transport, participant, reason):
-            print(f"Participant left: {participant}")
-            await task.cancel()
-
-        @transport.event_handler("on_call_state_updated")
-        async def on_call_state_updated(transport, state):
-            if state == "left":
-                # Here we don't want to cancel, we just want to finish sending
-                # whatever is queued, so we use an EndFrame().
-                await task.queue_frame(EndFrame())
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

examples/canonical-metrics/requirements.txt

@@ -1,5 +0,0 @@
-python-dotenv
-fastapi[all]
-uvicorn
-pipecat-ai[daily,openai,silero,elevenlabs,canonical]
-

examples/chatbot-audio-recording/bot.py

@@ -66,9 +66,7 @@ async def main():
             DailyParams(
                 audio_out_enabled=True,
                 audio_in_enabled=True,
-                camera_out_enabled=False,
-                vad_enabled=True,
-                vad_audio_passthrough=True,
+                video_out_enabled=False,
                 vad_analyzer=SileroVADAnalyzer(),
                 transcription_enabled=True,
                 #
@@ -95,7 +93,7 @@ async def main():
             # voice_id="gD1IexrzCvsXPHUuT0s3",
         )
 
-        llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
+        llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
 
         messages = [
             {
@@ -130,7 +128,15 @@ async def main():
             ]
         )
 
-        task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
+        task = PipelineTask(
+            pipeline,
+            params=PipelineParams(
+                audio_in_sample_rate=16000,
+                audio_out_sample_rate=16000,
+                enable_metrics=True,
+                enable_usage_metrics=True,
+            ),
+        )
 
         @audiobuffer.event_handler("on_audio_data")
         async def on_audio_data(buffer, audio, sample_rate, num_channels):

examples/chatbot-audio-recording/runner.py

@@ -53,4 +53,3 @@ async def configure(aiohttp_session: aiohttp.ClientSession):
     token = await daily_rest_helper.get_token(url, expiry_time)
 
     return (url, token)
-    return (url, token)

examples/daily-custom-tracks/README.md

@@ -0,0 +1,39 @@
+# Daily Custom Tracks
+
+This example shows how to send and receive Daily custom tracks. We will run a simple `daily-python` application to send an audio file with a custom track (named "pipecat") to a room. Then, the Pipecat bot will mirror that custom track into another custom track (named "pipecat-mirror") in the same room.
+
+## Get started
+
+```python
+python3 -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+```
+
+## Run the bot
+
+Start the bot by giving it a Daily room URL.
+
+```bash
+python bot.py -u ROOM_URL
+```
+
+The bot will wait for the first participant to join. Then, it will mirror a custom track named "pipecat" into a new custom track named "pipecat-mirror".
+
+## Run the sender
+
+Now, run the custom track sender. This is a simple `daily-python` application that opens and audio file and sends it as a custom track to the same Daily room.
+
+```bash
+python custom_track_sender.py -u ROOM_URL -i office-ambience-mono-16000.mp3
+```
+
+## Open client
+
+Finally, open the client so you can hear both custom tracks.
+
+```bash
+open index.html
+```
+
+Once the client is opened, copy the URL of the Daily room and join it. You should be able to select which custom track you want to hear.

examples/daily-custom-tracks/bot.py

@@ -0,0 +1,89 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import sys
+
+import aiohttp
+from loguru import logger
+from runner import configure
+
+from pipecat.frames.frames import Frame, InputAudioRawFrame, OutputAudioRawFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+class CustomTrackMirrorProcessor(FrameProcessor):
+    def __init__(self, transport_destination: str, **kwargs):
+        super().__init__(**kwargs)
+        self._transport_destination = transport_destination
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, InputAudioRawFrame) and frame.transport_source:
+            output_frame = OutputAudioRawFrame(
+                audio=frame.audio,
+                sample_rate=frame.sample_rate,
+                num_channels=frame.num_channels,
+            )
+            output_frame.transport_destination = self._transport_destination
+            await self.push_frame(output_frame)
+        else:
+            await self.push_frame(frame, direction)
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        (room_url, _) = await configure(session)
+
+        transport = DailyTransport(
+            room_url,
+            None,
+            "Custom tracks mirror",
+            DailyParams(
+                audio_in_enabled=True,
+                audio_out_enabled=True,
+                microphone_out_enabled=False,  # Disable since we just use custom tracks
+                audio_out_destinations=["pipecat-mirror"],
+            ),
+        )
+
+        pipeline = Pipeline(
+            [
+                transport.input(),  # Transport user input
+                CustomTrackMirrorProcessor("pipecat-mirror"),
+                transport.output(),  # Transport bot output
+            ]
+        )
+
+        task = PipelineTask(
+            pipeline,
+            params=PipelineParams(
+                audio_in_sample_rate=16000,
+                audio_out_sample_rate=16000,
+                enable_metrics=True,
+                enable_usage_metrics=True,
+            ),
+        )
+
+        @transport.event_handler("on_first_participant_joined")
+        async def on_first_participant_joined(transport, participant):
+            await transport.capture_participant_audio(participant["id"], audio_source="pipecat")
+
+        runner = PipelineRunner()
+
+        await runner.run(task)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/daily-custom-tracks/custom_track_sender.py

@@ -0,0 +1,74 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import argparse
+import time
+
+from daily import CallClient, CustomAudioSource, Daily
+from pydub import AudioSegment
+
+parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
+parser.add_argument("-u", "--url", type=str, required=True, help="URL of the Daily room to join")
+parser.add_argument(
+    "-i", "--input", type=str, required=True, help="Input audio file (needs 16000 sample rate)"
+)
+
+args, _ = parser.parse_known_args()
+
+audio = AudioSegment.from_mp3(args.input)
+
+raw_bytes = audio.raw_data
+sample_rate = audio.frame_rate
+channels = audio.channels
+
+print(f"Length: {len(raw_bytes)} bytes")
+print(f"Sample rate: {sample_rate}, Channels: {channels}")
+
+# Initialize the Daily context & create call client
+Daily.init()
+
+client = CallClient()
+
+# Join the room and indicate we have a custom track named "pipecat".
+client.join(
+    args.url,
+    client_settings={
+        "publishing": {
+            "camera": False,
+            "microphone": False,
+            "customAudio": {"pipecat": True},
+        },
+    },
+)
+
+# Just sleep for a couple of seconds. To do this well we should really use
+# completions.
+time.sleep(2)
+
+# Create the custom audio source. This is where we will write our audio.
+audio_source = CustomAudioSource(sample_rate, channels)
+
+# Create an audio track and assign it our audio source.
+client.add_custom_audio_track("pipecat", audio_source)
+
+# Just sleep for a second. To do this well we should really use completions.
+time.sleep(1)
+
+try:
+    # Just write one second of audio until we have read all the file.
+    chunk_size = sample_rate * channels * 2
+    while len(raw_bytes) > 0:
+        chunk = raw_bytes[:chunk_size]
+        raw_bytes = raw_bytes[chunk_size:]
+        audio_source.write_frames(chunk)
+
+except KeyboardInterrupt:
+    client.leave()
+
+# Just sleep for a second. To do this well we should really use completions.
+time.sleep(1)
+
+client.release()

examples/daily-custom-tracks/index.html

@@ -0,0 +1,173 @@
+<html>
+  <head>
+    <title>daily custom tracks</title>
+  </head>
+  <script crossorigin src="https://unpkg.com/@daily-co/daily-js"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/fomantic-ui/2.8.6/semantic.min.js"></script>
+  <link
+    rel="stylesheet"
+    type="text/css"
+    href="https://cdnjs.cloudflare.com/ajax/libs/fomantic-ui/2.8.6/semantic.min.css"
+    />
+  <script>
+    function enableButton(buttonId, enable) {
+        const button = document.getElementById(buttonId);
+        button.disabled = !enable;
+    }
+
+    function enableJoinButton(enable) {
+        enableButton("join-button", enable);
+    }
+
+    function enableLeaveButton(enable) {
+        enableButton("leave-button", enable);
+    }
+
+    function destroyPlayers(query) {
+        const items = document.querySelectorAll(query);
+        if (items) {
+            for (const item of items) {
+                item.remove();
+            }
+        }
+    }
+
+    function destroyParticipantPlayers(participantId) {
+        destroyPlayers(`audio[data-participant-id="${participantId}"]`);
+        destroyPlayers(`button[data-participant-id="${participantId}"]`);
+    }
+
+    async function startPlayer(player, track) {
+        player.muted = false;
+        player.autoplay = true;
+        if (track != null) {
+            player.srcObject = new MediaStream([track]);
+        }
+    }
+
+    async function buildAudioPlayer(track, participantId) {
+        const audioContainer = document.getElementById("audio-container");
+        const player = document.createElement("audio");
+        player.dataset.participantId = participantId;
+
+        // Create a new button for controlling audio
+        const audioControlButton = document.createElement("button");
+        audioControlButton.className = "ui primary green button"
+        audioControlButton.innerText = track._mediaTag == "cam-audio" ? "english" : track._mediaTag;
+        audioControlButton.dataset.participantId = participantId;
+        audioControlButton.onclick = () => {
+            if (player.paused) {
+
+                player.play();
+                audioControlButton.className = "ui primary red button"
+            } else {
+                player.pause();
+                audioControlButton.className = "ui primary green button"
+            }
+        };
+
+        audioContainer.appendChild(player);
+        audioContainer.appendChild(audioControlButton);
+
+        await startPlayer(player, track);
+        player.pause()
+
+        return player;
+    }
+
+    function subscribeToTracks(participantId) {
+        console.log(`subscribing to track`);
+
+        if (participantId === "local") {
+            return;
+        }
+
+        callObject.updateParticipant(participantId, {
+            setSubscribedTracks: {
+                audio: true,
+                video: false,
+                custom: true,
+            },
+        });
+    }
+
+    function startDaily() {
+        enableJoinButton(true);
+        enableLeaveButton(false);
+
+        window.callObject = window.DailyIframe.createCallObject({});
+
+        callObject.on("participant-joined", (e) => {
+            if (!e.participant.local) {
+                console.log("participant-joined", e.participant);
+               subscribeToTracks(e.participant.session_id);
+            }
+        });
+
+        callObject.on("participant-left", (e) => {
+            console.log("participant-left", e.participant.session_id);
+            destroyParticipantPlayers(e.participant.session_id);
+        });
+
+        callObject.on("track-started", async (e) => {
+            console.log("track-started", e.track);
+            if (e.track.kind === "audio") {
+                await buildAudioPlayer(e.track, e.participant.session_id);
+            }
+        });
+    }
+
+    async function joinRoom() {
+        enableJoinButton(false);
+        enableLeaveButton(true);
+
+        const meetingUrl = document.getElementById("meeting-url").value;
+
+        callObject.join({
+            url: meetingUrl,
+            startVideoOff: true,
+            startAudioOff: true,
+            subscribeToTracksAutomatically: false,
+            receiveSettings: {
+                base: { video: { layer: 0 } },
+            },
+        });
+    }
+
+    async function leaveRoom() {
+        enableJoinButton(true);
+        enableLeaveButton(false);
+
+        callObject.leave();
+
+        const audioContainer = document.getElementById("audio-container");
+        audioContainer.replaceChildren();
+    }
+  </script>
+
+  <body onload="startDaily()">
+    <div class="ui centered page grid" style="margin-top: 30px">
+      <div class="ten wide column">
+        <div class="ui form" style="margin-top: 30px">
+          <div class="field">
+            <label>Meeting URL</label>
+            <input id="meeting-url" value="" />
+          </div>
+        </div>
+      </div>
+    </div>
+    <div class="ui centered aligned header" style="margin-top: 30px">
+      <button id="join-button" class="ui primary button" onclick="joinRoom()">
+        Join
+      </button>
+      <button id="leave-button" class="ui button" onclick="leaveRoom()">
+        Leave
+      </button>
+    </div>
+    <div id="tile" class="ui container" style="margin-top: 30px">
+      <div id="tile" class="ui center aligned grid">
+        <div id="audio-container"></div><br/>
+      </div>
+    </div>
+  </body>
+</html>

examples/daily-custom-tracks/requirements.txt

@@ -0,0 +1,2 @@
+pydub
+pipecat-ai[daily]

examples/daily-multi-translation/Dockerfile

@@ -1,7 +1,12 @@
 FROM python:3.10-bullseye
+
 RUN mkdir /app
+RUN mkdir /app/assets
+RUN mkdir /app/utils
 COPY *.py /app/
 COPY requirements.txt /app/
+
+
 WORKDIR /app
 RUN pip3 install -r requirements.txt
 

examples/daily-multi-translation/README.md

@@ -0,0 +1,39 @@
+# Daily Multi Translation
+
+This example shows how to use Daily to stream multiple simultaneous translations using a single transport. Daily provides custom tracks and in this example we will simultaneously translate incoming audio in English to Spanish, French and German, each of them being sent to a custom track.
+
+## Get started
+
+```python
+python3 -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+
+cp env.example .env # and add your credentials
+
+```
+
+## Run the server
+
+```bash
+python server.py
+```
+
+Then, visit `http://localhost:7860/` in your browser. This will open a Daily Prebuilt room where you will speak in English (make sure you are not muted).
+
+## Open client
+
+Next, you need to open the client that will listen to the translations.
+
+```bash
+open index.html
+```
+
+Once the client is opened, copy the URL of the Daily room created above and join it. You should be able to select which translation you want to hear.
+
+## Build and test the Docker image
+
+```
+docker build -t daily-multi-translation .
+docker run --env-file .env -p 7860:7860 daily-multi-translation
+```

examples/daily-multi-translation/bot.py

@@ -0,0 +1,163 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import os
+import sys
+
+import aiohttp
+from dotenv import load_dotenv
+from loguru import logger
+from runner import configure
+
+from pipecat.audio.mixers.soundfile_mixer import SoundfileMixer
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.observers.loggers.transcription_log_observer import TranscriptionLogObserver
+from pipecat.pipeline.parallel_pipeline import ParallelPipeline
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.deepgram.stt import DeepgramSTTService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+BACKGROUND_SOUND_FILE = "office-ambience-mono-16000.mp3"
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        (room_url, token) = await configure(session)
+
+        transport = DailyTransport(
+            room_url,
+            token,
+            "Multi translation bot",
+            DailyParams(
+                audio_in_enabled=True,
+                audio_out_enabled=True,
+                audio_out_mixer={
+                    "spanish": SoundfileMixer(
+                        sound_files={"office": BACKGROUND_SOUND_FILE}, default_sound="office"
+                    ),
+                    "french": SoundfileMixer(
+                        sound_files={"office": BACKGROUND_SOUND_FILE}, default_sound="office"
+                    ),
+                    "german": SoundfileMixer(
+                        sound_files={"office": BACKGROUND_SOUND_FILE}, default_sound="office"
+                    ),
+                },
+                audio_out_destinations=["spanish", "french", "german"],
+                microphone_out_enabled=False,  # Disable since we just use custom tracks
+                vad_analyzer=SileroVADAnalyzer(),
+            ),
+        )
+
+        stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
+
+        tts_spanish = CartesiaTTSService(
+            api_key=os.getenv("CARTESIA_API_KEY"),
+            voice_id="cefcb124-080b-4655-b31f-932f3ee743de",
+            transport_destination="spanish",
+        )
+        tts_french = CartesiaTTSService(
+            api_key=os.getenv("CARTESIA_API_KEY"),
+            voice_id="8832a0b5-47b2-4751-bb22-6a8e2149303d",
+            transport_destination="french",
+        )
+        tts_german = CartesiaTTSService(
+            api_key=os.getenv("CARTESIA_API_KEY"),
+            voice_id="38aabb6a-f52b-4fb0-a3d1-988518f4dc06",
+            transport_destination="german",
+        )
+
+        messages_spanish = [
+            {
+                "role": "system",
+                "content": "You will be provided with a sentence in English, and your task is to only translate it into Spanish.",
+            },
+        ]
+        messages_french = [
+            {
+                "role": "system",
+                "content": "You will be provided with a sentence in English, and your task is to only translate it into French.",
+            },
+        ]
+        messages_german = [
+            {
+                "role": "system",
+                "content": "You will be provided with a sentence in English, and your task is to only translate it into German.",
+            },
+        ]
+
+        llm_spanish = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
+        llm_french = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
+        llm_german = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
+
+        context_spanish = OpenAILLMContext(messages_spanish)
+        context_aggregator_spanish = llm_spanish.create_context_aggregator(context_spanish)
+
+        context_french = OpenAILLMContext(messages_french)
+        context_aggregator_french = llm_french.create_context_aggregator(context_french)
+
+        context_german = OpenAILLMContext(messages_german)
+        context_aggregator_german = llm_german.create_context_aggregator(context_german)
+
+        pipeline = Pipeline(
+            [
+                transport.input(),  # Transport user input
+                stt,
+                ParallelPipeline(
+                    # Spanish pipeline.
+                    [
+                        context_aggregator_spanish.user(),
+                        llm_spanish,
+                        tts_spanish,
+                        context_aggregator_spanish.assistant(),
+                    ],
+                    # French pipeline.
+                    [
+                        context_aggregator_french.user(),
+                        llm_french,
+                        tts_french,
+                        context_aggregator_french.assistant(),
+                    ],
+                    # German pipeline.
+                    [
+                        context_aggregator_german.user(),
+                        llm_german,
+                        tts_german,
+                        context_aggregator_german.assistant(),
+                    ],
+                ),
+                transport.output(),  # Transport bot output
+            ]
+        )
+
+        task = PipelineTask(
+            pipeline,
+            params=PipelineParams(
+                audio_in_sample_rate=16000,
+                audio_out_sample_rate=16000,
+                enable_metrics=True,
+                enable_usage_metrics=True,
+            ),
+            observers=[TranscriptionLogObserver()],
+        )
+
+        runner = PipelineRunner()
+
+        await runner.run(task)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/daily-multi-translation/env.example

@@ -1,6 +1,5 @@
 DAILY_SAMPLE_ROOM_URL=https://yourdomain.daily.co/yourroom # (for joining the bot to the same room repeatedly for local dev)
 DAILY_API_KEY=7df...
 OPENAI_API_KEY=sk-PL...
-ELEVENLABS_API_KEY=aeb...
-CANONICAL_API_KEY=can...
-CANONICAL_API_URL=
+DEEPGRAM_API_KEY=efb...
+CARTESIA_API_KEY=aeb...

examples/daily-multi-translation/index.html

@@ -0,0 +1,202 @@
+<html>
+  <head>
+    <title>daily multi translation</title>
+  </head>
+  <script crossorigin src="https://unpkg.com/@daily-co/daily-js"></script>
+  <script
+    src="https://code.jquery.com/jquery-3.1.1.min.js"
+    integrity="sha256-hVVnYaiADRTO2PzUGmuLJr8BLUSjGIZsDYGmIJLv2b8="
+    crossorigin="anonymous"
+    ></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/fomantic-ui/2.8.6/semantic.min.js"></script>
+  <link
+    rel="stylesheet"
+    type="text/css"
+    href="https://cdnjs.cloudflare.com/ajax/libs/fomantic-ui/2.8.6/semantic.min.css"
+    />
+  <script>
+    function enableButton(buttonId, enable) {
+        const button = document.getElementById(buttonId);
+        button.disabled = !enable;
+    }
+
+    function enableJoinButton(enable) {
+        enableButton("join-button", enable);
+    }
+
+    function enableLeaveButton(enable) {
+        enableButton("leave-button", enable);
+    }
+
+    function destroyPlayers(query) {
+        const items = document.querySelectorAll(query);
+        if (items) {
+            for (const item of items) {
+                item.remove();
+            }
+        }
+    }
+
+    function destroyParticipantPlayers(participantId) {
+        destroyPlayers(`video[data-participant-id="${participantId}"]`);
+        destroyPlayers(`audio[data-participant-id="${participantId}"]`);
+        destroyPlayers(`button[data-participant-id="${participantId}"]`);
+    }
+
+    async function startPlayer(player, track) {
+        player.muted = false;
+        player.autoplay = true;
+        if (track != null) {
+            player.srcObject = new MediaStream([track]);
+        }
+    }
+
+    async function buildVideoPlayer(track, participantId) {
+        const videoContainer = document.getElementById("video-container");
+        const player = document.createElement("video");
+        player.dataset.participantId = participantId;
+
+        videoContainer.appendChild(player);
+
+        await startPlayer(player, track);
+        await player.play();
+
+        return player;
+    }
+
+    async function buildAudioPlayer(track, participantId) {
+        const audioContainer = document.getElementById("audio-container");
+        const player = document.createElement("audio");
+        player.dataset.participantId = participantId;
+
+        // Create a new button for controlling audio
+        const audioControlButton = document.createElement("button");
+        audioControlButton.className = "ui primary green button"
+        audioControlButton.innerText = track._mediaTag == "cam-audio" ? "english" : track._mediaTag;
+        audioControlButton.dataset.participantId = participantId;
+        audioControlButton.onclick = () => {
+            if (player.paused) {
+
+                player.play();
+                audioControlButton.className = "ui primary red button"
+            } else {
+                player.pause();
+                audioControlButton.className = "ui primary green button"
+            }
+        };
+
+        audioContainer.appendChild(player);
+        audioContainer.appendChild(audioControlButton);
+
+        await startPlayer(player, track);
+        player.pause()
+
+        return player;
+    }
+
+    function subscribeToTracks(participantId) {
+        console.log(`subscribing to track`);
+
+        if (participantId === "local") {
+            return;
+        }
+
+        callObject.updateParticipant(participantId, {
+            setSubscribedTracks: {
+                audio: true,
+                video: true,
+                custom: true,
+            },
+        });
+    }
+
+    function startDaily() {
+        enableJoinButton(true);
+        enableLeaveButton(false);
+
+        window.callObject = window.DailyIframe.createCallObject({});
+
+        callObject.on("participant-joined", (e) => {
+            if (!e.participant.local) {
+                console.log("participant-joined", e.participant);
+               subscribeToTracks(e.participant.session_id);
+            }
+        });
+
+        callObject.on("participant-left", (e) => {
+            console.log("participant-left", e.participant.session_id);
+            destroyParticipantPlayers(e.participant.session_id);
+        });
+
+        callObject.on("track-started", async (e) => {
+            console.log("track-started", e.track);
+            if (e.track.kind === "video") {
+                await buildVideoPlayer(e.track, e.participant.session_id);
+            } else if (e.track.kind === "audio") {
+                await buildAudioPlayer(e.track, e.participant.session_id);
+            }
+        });
+    }
+
+    async function joinRoom() {
+        enableJoinButton(false);
+        enableLeaveButton(true);
+
+        const meetingUrl = document.getElementById("meeting-url").value;
+
+        callObject.join({
+            url: meetingUrl,
+            startVideoOff: true,
+            startAudioOff: true,
+            subscribeToTracksAutomatically: false,
+            receiveSettings: {
+                base: { video: { layer: 0 } },
+            },
+        });
+    }
+
+    async function leaveRoom() {
+        enableJoinButton(true);
+        enableLeaveButton(false);
+
+        callObject.leave();
+
+        const videoContainer = document.getElementById("video-container");
+        videoContainer.replaceChildren();
+
+        const audioContainer = document.getElementById("audio-container");
+        audioContainer.replaceChildren();
+    }
+  </script>
+
+  <body onload="startDaily()">
+    <div class="ui centered page grid" style="margin-top: 30px">
+      <div class="ten wide column">
+        <div class="ui form" style="margin-top: 30px">
+          <div class="field">
+            <label>Meeting URL</label>
+            <input id="meeting-url" value="" />
+          </div>
+        </div>
+      </div>
+    </div>
+    <div class="ui centered aligned header" style="margin-top: 30px">
+      <button id="join-button" class="ui primary button" onclick="joinRoom()">
+        Join
+      </button>
+      <button id="leave-button" class="ui button" onclick="leaveRoom()">
+        Leave
+      </button>
+    </div>
+    <div id="tile" class="ui container" style="margin-top: 30px">
+      <div id="tile" class="ui center aligned grid">
+        <div id="audio-container"></div><br/>
+      </div>
+    </div>
+    <div id="tile" class="ui container" style="margin-top: 30px">
+      <div id="tile" class="ui center aligned grid">
+        <div id="video-container" class="ui segment"></div>
+      </div>
+    </div>
+  </body>
+</html>

examples/daily-multi-translation/requirements.txt

@@ -0,0 +1,5 @@
+aiofiles
+python-dotenv
+fastapi[all]
+uvicorn
+pipecat-ai[daily,deepgram,openai,silero,cartesia,soundfile]

examples/daily-multi-translation/runner.py

@@ -6,7 +6,6 @@
 
 import argparse
 import os
-from typing import Optional
 
 import aiohttp
 
@@ -14,15 +13,7 @@ from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper
 
 
 async def configure(aiohttp_session: aiohttp.ClientSession):
-    (url, token, _) = await configure_with_args(aiohttp_session)
-    return (url, token)
-
-
-async def configure_with_args(
-    aiohttp_session: aiohttp.ClientSession, parser: Optional[argparse.ArgumentParser] = None
-):
-    if not parser:
-        parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
+    parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
     parser.add_argument(
         "-u", "--url", type=str, required=False, help="URL of the Daily room to join"
     )
@@ -61,4 +52,4 @@ async def configure_with_args(
 
     token = await daily_rest_helper.get_token(url, expiry_time)
 
-    return (url, token, args)
+    return (url, token)

examples/deployment/flyio-example/bot.py

@@ -41,8 +41,7 @@ async def main(room_url: str, token: str):
             api_key=daily_api_key,
             audio_in_enabled=True,
             audio_out_enabled=True,
-            camera_out_enabled=False,
-            vad_enabled=True,
+            video_out_enabled=False,
             vad_analyzer=SileroVADAnalyzer(),
             transcription_enabled=True,
         ),
@@ -53,7 +52,7 @@ async def main(room_url: str, token: str):
         voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
     )
 
-    llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
+    llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
 
     messages = [
         {
@@ -76,7 +75,13 @@ async def main(room_url: str, token: str):
         ]
     )
 
-    task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
+    task = PipelineTask(
+        pipeline,
+        params=PipelineParams(
+            enable_metrics=True,
+            enable_usage_metrics=True,
+        ),
+    )
 
     @transport.event_handler("on_first_participant_joined")
     async def on_first_participant_joined(transport, participant):

examples/deployment/modal-example/.gitignore

@@ -1,3 +1,6 @@
+# Modal clone
+modal-examples
+
 # Python
 __pycache__/
 *.py[cod]

examples/deployment/modal-example/README.md

@@ -1,37 +1,91 @@
 # Deploying Pipecat to Modal.com
 
-Barebones deployment example for [modal.com](https://www.modal.com)
+Deployment example for [modal.com](https://www.modal.com). This example demonstrates how to deploy a FastAPI webapp to Modal with an RTVI compatible `/connect` endpoint that launches a Pipecat pipeline in a separate Modal container and returns a room/token for the client to join. This example also supports providing a parameter to the `/connect` endpoint for specifying which Pipecat pipeline to launch; openai, gemini, or vllm. The vllm pipeline points to a self-hosted OpenAI compatible LLM, using a llama model (neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16), deployed to Modal.
 
-1. Install dependencies
+![](diagram.jpg)
 
-```bash
-python -m venv venv
-source venv/bin/active # or OS equivalent
-pip install -r requirements.txt
-```
+# Running this Example
 
-2. Setup .env
+## Install the Modal CLI
 
-```bash
-cp env.example .env
-```
+Setup a Modal account and install it on your machine if you have not already, following their easy 3 steps in their [Getting Started Guide](https://modal.com/docs/guide#getting-started)
 
-Alternatively, you can configure your Modal app to use [secrets](https://modal.com/docs/guide/secrets)
+## Deploy a self-serve LLM
 
-3. Test the app locally
+1. Deploy Modal's OpenAI-compatible LLM service:
 
-```bash
-modal serve app.py
-```
+   ```bash
+   git clone https://github.com/modal-labs/modal-examples
+   cd modal-examples
+   modal deploy 06_gpu_and_ml/llm-serving/vllm_inference.py
+   ```
+
+   Refer to Modal's guide and example for [Deploying an OpenAI-compatible LLM service with vLLM](https://modal.com/docs/examples/vllm_inference) for more details.
+
+2. Take note of the endpoint URL from the previous step, which will look like:
+   ```
+   https://{your-workspace}--example-vllm-openai-compatible-serve.modal.run
+   ```
+   You'll need this for the `bot_vllm.py` file in the next section.
+
+    **Note:**  The default Modal LLM example uses Llama-3.1 and will shut down after 15 minutes of inactivity. Cold starts take 5-10 minutes. To prepare the service, we recommend visiting the `/docs` endpoint (`https://<Modal workspace>--example-vllm-openai-compatible-serve.modal.run/docs`) for your deployed LLM and wait for it to fully load before connecting your client.
+
+## Deploy FastAPI App and Pipecat pipeline to Modal 
+
+1. Setup environment variables
+
+   ```bash
+   cd server
+   cp env.example .env
+   # Modify .env to provide your service API Keys
+   ```
+
+   Alternatively, you can configure your Modal app to use [secrets](https://modal.com/docs/guide/secrets)
+
+2. Update the `modal_url` in `server/src/bot_vllm.py` to point to the url produced from the self-serve llm deploy, mentioned above.
+
+3. From within the `server` directory, test the app locally:
+
+   ```bash
+   modal serve app.py
+   ```
 
 4. Deploy to production
 
-```bash
-modal deploy app.py
-```
+   ```bash
+   modal deploy app.py
+   ```
 
-## Configuration options
+5. Note the endpoint URL produced from this deployment. It will look like:
 
-This app sets some sensible defaults for reducing cold starts, such as `minkeep_warm=1`, which will keep at least 1 warm instance ready for your bot function.
+   ```bash
+   https://{your-workspace}--pipecat-modal-fastapi-app.modal.run
+   ```
 
-It has been configured to only allow a concurrency of 1 (`max_inputs=1`) as each user will require their own running function.
+   You'll need this URL for the client's `app.js` configuration mentioned in its README.
+
+## Launch your bots on Modal
+
+### Option 1: Direct Link
+
+Simply click on the url displayed after running the server or deploy step to launch an agent and be redirected to a Daily room to talk with the launched bot. This will use the OpenAI pipeline.
+
+### Option 2: Connect via an RTVI Client
+
+Follow the instructions provided in the [client folder's README](client/javascript/README.md) for building and running a custom client that connects to your Modal endpoint. The provided client provides a dropdown for choosing which bot pipeline to run.
+
+# Navigating your llm, server, and Pipecat logs
+
+In your [Modal dashboard](https://modal.com/apps), you should have two Apps listed under Live Apps:
+
+1. `example-vllm-openai-compatible`: This App contains the containers and logs used to run your self-hosted LLM. There will be just one App Function listed: `serve`. Click on this function to view logs for your LLM.
+2. `pipecat-modal`: This App contains the containers and logs used to run your `connect` endpoints and Pipecat pipelines. It will list two App Functions:
+    1. `fastapi_app`: This function is running the endpoints that your client will interact with and initiate starting a new pipeline (`/`, `/connect`, `/status`). Click on this function to see logs for each endpoint hit.
+    2. `bot_runner`: This function handles launching and running a bot pipeline. Click on this function to get a list of all pipeline runs and access each run's logs.
+
+# Modal + Pipecat Tips
+
+- In most other Pipecat examples, we use `Popen` to launch the pipeline process from the `/connect` endpoint. In this example, we use a Modal function instead. This allows us to run the pipelines using a separately defined Modal image as well as run each pipeline in an isolated container.
+- For the FastAPI and most common Pipecat Pipeline containers, a default `debian_slim` CPU-only should be all that's required to run. GPU containers are needed for self-hosted services.
+- To minimize cold starts of the pipeline and reduce latency for users, set `min_containers=1` on the Modal Function that launches the pipeline to ensure at least one warm instance of your function is always available.
+- For next steps on running a self-hosted llm and reducing latency, check out all of [Modal's LLM examples](https://modal.com/docs/examples/vllm_inference).

examples/deployment/modal-example/app.py

@@ -1,80 +0,0 @@
-#
-# Copyright (c) 2024–2025, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import os
-
-import aiohttp
-import modal
-from bot import _voice_bot_process
-from fastapi import HTTPException
-from fastapi.responses import JSONResponse
-from loguru import logger
-
-MAX_SESSION_TIME = 15 * 60  # 15 minutes
-
-app = modal.App("pipecat-modal")
-
-
-image = modal.Image.debian_slim(python_version="3.12").pip_install_from_requirements(
-    "requirements.txt"
-)
-
-
-@app.function(
-    image=image,
-    cpu=1.0,
-    secrets=[modal.Secret.from_dotenv()],
-    keep_warm=1,
-    enable_memory_snapshot=True,
-    max_inputs=1,  # Do not reuse instances across requests
-    retries=0,
-)
-def launch_bot_process(room_url: str, token: str):
-    _voice_bot_process(room_url, token)
-
-
-@app.function(
-    image=image,
-    secrets=[modal.Secret.from_dotenv()],
-)
-@modal.web_endpoint(method="POST")
-async def start():
-    from pipecat.transports.services.helpers.daily_rest import (
-        DailyRESTHelper,
-        DailyRoomParams,
-    )
-
-    logger.info("Request received")
-
-    async with aiohttp.ClientSession() as session:
-        daily_rest_helper = DailyRESTHelper(
-            daily_api_key=os.getenv("DAILY_API_KEY", ""),
-            daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"),
-            aiohttp_session=session,
-        )
-
-        # Create new Daily room
-        room = await daily_rest_helper.create_room(DailyRoomParams())
-        if not room.url:
-            raise HTTPException(
-                status_code=500,
-                detail="Unable to create room",
-            )
-        logger.info(f"Created room: {room.url}")
-
-        # Create bot token for room
-        token = await daily_rest_helper.get_token(room.url, MAX_SESSION_TIME)
-        if not token:
-            raise HTTPException(status_code=500, detail=f"Failed to get token for room: {room.url}")
-
-        logger.info(f"Bot token created: {token}")
-
-        # Spawn a new bot process
-        launch_bot_process.spawn(room_url=room.url, token=token)
-
-        # Return room URL to the user to join
-        # Note: in production, you would want to return a token to the user
-        return JSONResponse(content={"room_url": room.url, token: token})

examples/deployment/modal-example/bot.py

@@ -1,95 +0,0 @@
-#
-# Copyright (c) 2024–2025, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import os
-import sys
-
-from dotenv import load_dotenv
-from loguru import logger
-
-from pipecat.audio.vad.silero import SileroVADAnalyzer
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
-from pipecat.services.cartesia.tts import CartesiaTTSService
-from pipecat.services.openai.llm import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def main(room_url: str, token: str):
-    transport = DailyTransport(
-        room_url,
-        token,
-        "bot",
-        DailyParams(
-            audio_out_enabled=True,
-            transcription_enabled=True,
-            vad_enabled=True,
-            vad_analyzer=SileroVADAnalyzer(),
-        ),
-    )
-
-    tts = CartesiaTTSService(
-        api_key=os.getenv("CARTESIA_API_KEY", ""), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121"
-    )
-
-    llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
-
-    messages = [
-        {
-            "role": "system",
-            "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
-        },
-    ]
-
-    context = OpenAILLMContext(messages)
-    context_aggregator = llm.create_context_aggregator(context)
-
-    pipeline = Pipeline(
-        [
-            transport.input(),
-            context_aggregator.user(),
-            llm,
-            tts,
-            transport.output(),
-            context_aggregator.assistant(),
-        ]
-    )
-
-    task = PipelineTask(
-        pipeline,
-        params=PipelineParams(
-            allow_interruptions=True,
-            enable_metrics=True,
-            enable_usage_metrics=True,
-            report_only_initial_ttfb=True,
-        ),
-    )
-
-    @transport.event_handler("on_first_participant_joined")
-    async def on_first_participant_joined(transport, participant):
-        await transport.capture_participant_transcription(participant["id"])
-        messages.append({"role": "system", "content": "Please introduce yourself to the user."})
-        await task.queue_frames([context_aggregator.user().get_context_frame()])
-
-    @transport.event_handler("on_participant_left")
-    async def on_participant_left(transport, participant, reason):
-        await task.cancel()
-
-    runner = PipelineRunner()
-
-    await runner.run(task)
-
-
-def _voice_bot_process(room_url: str, token: str):
-    asyncio.run(main(room_url, token))

examples/deployment/modal-example/client/javascript/.gitignore

@@ -0,0 +1 @@
+node_modules

examples/deployment/modal-example/client/javascript/README.md

@@ -0,0 +1,29 @@
+# JavaScript Implementation
+
+Basic implementation using the [Pipecat JavaScript SDK](https://docs.pipecat.ai/client/js/introduction).
+
+## Setup
+
+1. Deploy the Modal server. See the main [README](../../README).
+
+2. Navigate to the `client/javascript` directory:
+
+```bash
+cd client/javascript
+```
+
+3. Modify the baseUrl in src/app.js to point to your deployed Modal endpoint
+
+4. Install dependencies:
+
+```bash
+npm install
+```
+
+5. Run the client app:
+
+```
+npm run dev
+```
+
+6. Visit http://localhost:5173 in your browser.

examples/deployment/modal-example/client/javascript/index.html

@@ -0,0 +1,49 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>AI Chatbot</title>
+  </head>
+
+  <body>
+    <div class="container">
+      <div class="status-bar">
+        <div class="status">
+          Status: <span id="connection-status">Disconnected</span>
+        </div>
+        <div class="controls">
+          <select id="bot-selector">
+            <option value="openai">OpenAI</option>
+            <option value="gemini">Gemini</option>
+            <option value="vllm">Llama</option>
+          </select>
+          <button id="connect-btn">Connect</button>
+          <button id="disconnect-btn" disabled>Disconnect</button>
+        </div>
+      </div>
+
+      <div class="main-content">
+        <div class="bot-container">
+          <div id="bot-video-container"></div>
+          <audio id="bot-audio" autoplay></audio>
+        </div>
+      </div>
+
+      <div class="device-bar">
+        <div class="device-controls">
+          <select id="device-selector"></select>
+          <button id="mic-toggle-btn">Mute Mic</button>
+        </div>
+      </div>
+
+      <div class="debug-panel">
+        <h3>Debug Info</h3>
+        <div id="debug-log"></div>
+      </div>
+    </div>
+
+    <script type="module" src="/src/app.js"></script>
+    <link rel="stylesheet" href="/src/style.css" />
+  </body>
+</html>

examples/deployment/modal-example/client/javascript/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "client",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "preview": "vite preview"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "description": "",
+  "devDependencies": {
+    "vite": "^6.3.5"
+  },
+  "dependencies": {
+    "@pipecat-ai/client-js": "^0.3.5",
+    "@pipecat-ai/daily-transport": "^0.3.10"
+  }
+}

examples/deployment/modal-example/client/javascript/src/app.js

@@ -0,0 +1,381 @@
+/**
+ * Copyright (c) 2024–2025, Daily
+ *
+ * SPDX-License-Identifier: BSD 2-Clause License
+ */
+
+/**
+ * RTVI Client Implementation
+ *
+ * This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
+ * It handles audio/video streaming and manages the connection lifecycle.
+ *
+ * Requirements:
+ * - A running RTVI bot server (defaults to http://localhost:7860)
+ * - The server must implement the /connect endpoint that returns Daily.co room credentials
+ * - Browser with WebRTC support
+ */
+
+import { RTVIClient, RTVIEvent } from '@pipecat-ai/client-js';
+import { DailyTransport } from '@pipecat-ai/daily-transport';
+
+/**
+ * ChatbotClient handles the connection and media management for a real-time
+ * voice and video interaction with an AI bot.
+ */
+class ChatbotClient {
+  constructor() {
+    // Initialize client state
+    this.rtviClient = null;
+    this.setupDOMElements();
+    this.initializeClientAndTransport();
+    this.setupEventListeners();
+  }
+
+  /**
+   * Set up references to DOM elements and create necessary media elements
+   */
+  setupDOMElements() {
+    // Get references to UI control elements
+    this.connectBtn = document.getElementById('connect-btn');
+    this.disconnectBtn = document.getElementById('disconnect-btn');
+    this.statusSpan = document.getElementById('connection-status');
+    this.debugLog = document.getElementById('debug-log');
+    this.botVideoContainer = document.getElementById('bot-video-container');
+    this.deviceSelector = document.getElementById('device-selector');
+
+    // Create an audio element for bot's voice output
+    this.botAudio = document.createElement('audio');
+    this.botAudio.autoplay = true;
+    this.botAudio.playsInline = true;
+    document.body.appendChild(this.botAudio);
+  }
+
+  /**
+   * Set up event listeners for connect/disconnect buttons
+   */
+  setupEventListeners() {
+    this.connectBtn.addEventListener('click', () => this.connect());
+    this.disconnectBtn.addEventListener('click', () => this.disconnect());
+
+    // Populate device selector
+    this.rtviClient.getAllMics().then((mics) => {
+      console.log('Available mics:', mics);
+      mics.forEach((device) => {
+        const option = document.createElement('option');
+        option.value = device.deviceId;
+        option.textContent = device.label || `Microphone ${device.deviceId}`;
+        this.deviceSelector.appendChild(option);
+      });
+    });
+    this.deviceSelector.addEventListener('change', (event) => {
+      const selectedDeviceId = event.target.value;
+      console.log('Selected device ID:', selectedDeviceId);
+      this.rtviClient.updateMic(selectedDeviceId);
+    });
+
+    // Handle mic mute/unmute toggle
+    const micToggleBtn = document.getElementById('mic-toggle-btn');
+
+    micToggleBtn.addEventListener('click', () => {
+      let micEnabled = this.rtviClient.isMicEnabled;
+      micToggleBtn.textContent = micEnabled ? 'Unmute Mic' : 'Mute Mic';
+      this.rtviClient.enableMic(!micEnabled);
+      // Add logic to mute/unmute the mic
+      if (micEnabled) {
+        console.log('Mic muted');
+        // Add code to mute the mic
+      } else {
+        console.log('Mic unmuted');
+        // Add code to unmute the mic
+      }
+    });
+  }
+
+  /**
+   * Set up the RTVI client and Daily transport
+   */
+  async initializeClientAndTransport() {
+    // Initialize the RTVI client with a DailyTransport and our configuration
+    this.rtviClient = new RTVIClient({
+      transport: new DailyTransport(),
+      params: {
+        // REPLACE WITH YOUR MODAL URL ENDPOINT
+        baseUrl:
+          'https://<Modal workspace>--pipecat-modal-bot-launcher.modal.run',
+        endpoints: {
+          connect: '/connect',
+        },
+        requestData: {
+          bot_name: 'openai',
+        },
+      },
+      enableMic: true, // Enable microphone for user input
+      enableCam: false,
+      callbacks: {
+        // Handle connection state changes
+        onConnected: () => {
+          this.updateStatus('Connected');
+          this.connectBtn.disabled = true;
+          this.disconnectBtn.disabled = false;
+          this.log('Client connected');
+        },
+        onDisconnected: () => {
+          this.updateStatus('Disconnected');
+          this.connectBtn.disabled = false;
+          this.disconnectBtn.disabled = true;
+          this.log('Client disconnected');
+        },
+        // Handle transport state changes
+        onTransportStateChanged: (state) => {
+          this.updateStatus(`Transport: ${state}`);
+          this.log(`Transport state changed: ${state}`);
+          if (state === 'connecting') {
+            window.startTime = Date.now();
+          }
+          if (state === 'ready') {
+            this.setupMediaTracks();
+            console.warn('TIME TO BOT READY:', Date.now() - window.startTime);
+          }
+        },
+        // Handle bot connection events
+        onBotConnected: (participant) => {
+          this.log(`Bot connected: ${JSON.stringify(participant)}`);
+        },
+        onBotDisconnected: (participant) => {
+          this.log(`Bot disconnected: ${JSON.stringify(participant)}`);
+        },
+        onBotReady: (data) => {
+          this.log(`Bot ready: ${JSON.stringify(data)}`);
+          this.setupMediaTracks();
+        },
+        // Transcript events
+        onUserTranscript: (data) => {
+          // Only log final transcripts
+          if (data.final) {
+            this.log(`User: ${data.text}`);
+          }
+        },
+        onBotTranscript: (data) => {
+          this.log(`Bot: ${data.text}`);
+        },
+        // Error handling
+        onMessageError: (error) => {
+          console.log('Message error:', error);
+        },
+        onMicUpdated: (data) => {
+          console.log('Mic updated:', data);
+          this.deviceSelector.value = data.deviceId;
+        },
+        onError: (error) => {
+          console.log('Error:', JSON.stringify(error));
+        },
+      },
+    });
+
+    // Set up listeners for media track events
+    this.setupTrackListeners();
+
+    await this.rtviClient.initDevices();
+    window.client = this.rtviClient;
+  }
+
+  /**
+   * Add a timestamped message to the debug log
+   */
+  log(message) {
+    const entry = document.createElement('div');
+    entry.textContent = `${new Date().toISOString()} - ${message}`;
+
+    // Add styling based on message type
+    if (message.startsWith('User: ')) {
+      entry.style.color = '#2196F3'; // blue for user
+    } else if (message.startsWith('Bot: ')) {
+      entry.style.color = '#4CAF50'; // green for bot
+    }
+
+    this.debugLog.appendChild(entry);
+    this.debugLog.scrollTop = this.debugLog.scrollHeight;
+    console.log(message);
+  }
+
+  /**
+   * Update the connection status display
+   */
+  updateStatus(status) {
+    this.statusSpan.textContent = status;
+    this.log(`Status: ${status}`);
+  }
+
+  /**
+   * Check for available media tracks and set them up if present
+   * This is called when the bot is ready or when the transport state changes to ready
+   */
+  setupMediaTracks() {
+    if (!this.rtviClient) return;
+
+    // Get current tracks from the client
+    const tracks = this.rtviClient.tracks();
+
+    // Set up any available bot tracks
+    if (tracks.bot?.audio) {
+      this.setupAudioTrack(tracks.bot.audio);
+    }
+    if (tracks.bot?.video) {
+      this.setupVideoTrack(tracks.bot.video);
+    }
+  }
+
+  /**
+   * Set up listeners for track events (start/stop)
+   * This handles new tracks being added during the session
+   */
+  setupTrackListeners() {
+    if (!this.rtviClient) return;
+
+    // Listen for new tracks starting
+    this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
+      // Only handle non-local (bot) tracks
+      if (!participant?.local) {
+        if (track.kind === 'audio') {
+          this.setupAudioTrack(track);
+        } else if (track.kind === 'video') {
+          this.setupVideoTrack(track);
+        }
+        this.log(
+          `Track started event: ${track.kind} from ${
+            participant?.name || 'unknown'
+          }`
+        );
+      } else {
+        this.log('Local mic unmuted');
+      }
+    });
+
+    // Listen for tracks stopping
+    this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
+      if (participant.local) {
+        this.log('Local mic muted');
+        return;
+      }
+      this.log(
+        `Track stopped event: ${track.kind} from ${
+          participant?.name || 'unknown'
+        }`
+      );
+    });
+  }
+
+  /**
+   * Set up an audio track for playback
+   * Handles both initial setup and track updates
+   */
+  setupAudioTrack(track) {
+    this.log('Setting up audio track');
+    // Check if we're already playing this track
+    if (this.botAudio.srcObject) {
+      const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
+      if (oldTrack?.id === track.id) return;
+    }
+    // Create a new MediaStream with the track and set it as the audio source
+    this.botAudio.srcObject = new MediaStream([track]);
+  }
+
+  /**
+   * Set up a video track for display
+   * Handles both initial setup and track updates
+   */
+  setupVideoTrack(track) {
+    this.log('Setting up video track');
+    const videoEl = document.createElement('video');
+    videoEl.autoplay = true;
+    videoEl.playsInline = true;
+    videoEl.muted = true;
+    videoEl.style.width = '100%';
+    videoEl.style.height = '100%';
+    videoEl.style.objectFit = 'cover';
+
+    // Check if we're already displaying this track
+    if (this.botVideoContainer.querySelector('video')?.srcObject) {
+      const oldTrack = this.botVideoContainer
+        .querySelector('video')
+        .srcObject.getVideoTracks()[0];
+      if (oldTrack?.id === track.id) return;
+    }
+
+    // Create a new MediaStream with the track and set it as the video source
+    videoEl.srcObject = new MediaStream([track]);
+    this.botVideoContainer.innerHTML = '';
+    this.botVideoContainer.appendChild(videoEl);
+  }
+
+  /**
+   * Initialize and connect to the bot
+   * This sets up the RTVI client, initializes devices, and establishes the connection
+   */
+  async connect() {
+    try {
+      const botSelector = document.getElementById('bot-selector');
+      const selectedBot = botSelector.value;
+      this.rtviClient.params.requestData.bot_name = selectedBot;
+
+      // Initialize audio/video devices
+      this.log('Initializing devices...');
+      await this.rtviClient.initDevices();
+
+      // Connect to the bot
+      this.log(`Connecting to bot: ${selectedBot}`);
+      await this.rtviClient.connect();
+
+      this.log('Connection complete');
+    } catch (error) {
+      // Handle any errors during connection
+      console.error('Connection error:', error);
+      this.log(`Error connecting: ${JSON.stringify(error.message)}`);
+      this.log(`Error stack: ${error.stack}`);
+      this.updateStatus('Error');
+
+      // Clean up if there's an error
+      if (this.rtviClient) {
+        try {
+          await this.rtviClient.disconnect();
+        } catch (disconnectError) {
+          this.log(`Error during disconnect: ${disconnectError.message}`);
+        }
+      }
+    }
+  }
+
+  /**
+   * Disconnect from the bot and clean up media resources
+   */
+  async disconnect() {
+    if (this.rtviClient) {
+      try {
+        // Disconnect the RTVI client
+        await this.rtviClient.disconnect();
+
+        // Clean up audio
+        if (this.botAudio.srcObject) {
+          this.botAudio.srcObject.getTracks().forEach((track) => track.stop());
+          this.botAudio.srcObject = null;
+        }
+
+        // Clean up video
+        if (this.botVideoContainer.querySelector('video')?.srcObject) {
+          const video = this.botVideoContainer.querySelector('video');
+          video.srcObject.getTracks().forEach((track) => track.stop());
+          video.srcObject = null;
+        }
+        this.botVideoContainer.innerHTML = '';
+      } catch (error) {
+        this.log(`Error disconnecting: ${error.message}`);
+      }
+    }
+  }
+}
+
+// Initialize the client when the page loads
+window.addEventListener('DOMContentLoaded', () => {
+  new ChatbotClient();
+});

examples/deployment/modal-example/client/javascript/src/style.css

@@ -0,0 +1,135 @@
+body {
+  margin: 0;
+  padding: 20px;
+  font-family: Arial, sans-serif;
+  background-color: #f0f0f0;
+}
+
+.container {
+  max-width: 1200px;
+  margin: 0 auto;
+}
+
+.status-bar,
+.device-bar {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  padding: 10px;
+  background-color: #fff;
+  border-radius: 8px;
+  margin-bottom: 20px;
+}
+
+.controls,
+.device-controls {
+  display: flex;
+  align-items: center;
+  gap: 10px; /* Adds spacing between elements */
+}
+
+.device-controls {
+  margin-left: auto;
+}
+
+.controls button,
+.device-controls button {
+  padding: 8px 16px;
+  margin-left: 10px;
+  border: none;
+  border-radius: 4px;
+  cursor: pointer;
+}
+
+#bot-selector,
+#device-selector {
+  padding: 8px 16px;
+  padding-right: 40px;
+  border: none;
+  border-radius: 4px;
+  background-color: #6c757d; /* Gray background */
+  color: white; /* White text */
+  cursor: pointer;
+  appearance: none; /* Removes default browser styling for dropdowns */
+  background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='white'%3E%3Cpath d='M7 10l5 5 5-5z'/%3E%3C/svg%3E"); /* Custom arrow */
+  background-repeat: no-repeat;
+  background-position: right 8px center; /* Position the arrow */
+}
+
+#bot-selector:focus,
+#device-selector:focus {
+  outline: none;
+  box-shadow: 0 0 4px rgba(0, 0, 0, 0.3); /* Add a subtle focus effect */
+}
+
+#connect-btn {
+  background-color: #4caf50;
+  color: white;
+}
+
+#disconnect-btn {
+  background-color: #f44336;
+  color: white;
+}
+
+#mic-toggle-btn {
+}
+
+button:disabled {
+  opacity: 0.5;
+  cursor: not-allowed;
+}
+
+.main-content {
+  background-color: #fff;
+  border-radius: 8px;
+  padding: 20px;
+  margin-bottom: 20px;
+}
+
+.bot-container {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+}
+
+#bot-video-container {
+  width: 640px;
+  height: 360px;
+  background-color: #e0e0e0;
+  border-radius: 8px;
+  margin: 20px auto;
+  overflow: hidden;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+#bot-video-container video {
+  width: 100%;
+  height: 100%;
+  object-fit: cover;
+}
+
+.debug-panel {
+  background-color: #fff;
+  border-radius: 8px;
+  padding: 20px;
+}
+
+.debug-panel h3 {
+  margin: 0 0 10px 0;
+  font-size: 16px;
+  font-weight: bold;
+}
+
+#debug-log {
+  height: 200px;
+  overflow-y: auto;
+  background-color: #f8f8f8;
+  padding: 10px;
+  border-radius: 4px;
+  font-family: monospace;
+  font-size: 12px;
+  line-height: 1.4;
+}

examples/deployment/modal-example/env.example

@@ -1,3 +0,0 @@
-DAILY_API_KEY=
-OPENAI_API_KEY=
-CARTESIA_API_KEY=

examples/deployment/modal-example/requirements.txt

@@ -1,5 +0,0 @@
-python-dotenv==1.0.1
-modal==0.71.3
-pipecat-ai[daily,silero,cartesia,openai]==0.0.52
-fastapi==0.115.6
-aiohttp==3.11.11

examples/deployment/modal-example/server/app.py

@@ -0,0 +1,307 @@
+"""modal_example.
+
+This module shows a simple example of how to deploy a bot using Modal and FastAPI.
+
+It includes:
+- FastAPI endpoints for starting agents and checking bot statuses.
+- Dynamic loading of bot implementations.
+- Use of a Daily transport for bot communication.
+"""
+
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import importlib
+import os
+from contextlib import asynccontextmanager
+from typing import Any, Dict, Literal
+
+import aiohttp
+import modal
+from fastapi import APIRouter, FastAPI, HTTPException
+from fastapi.responses import JSONResponse, RedirectResponse
+from pydantic import BaseModel
+
+# container specifications for the FastAPI web server
+web_image = (
+    modal.Image.debian_slim(python_version="3.13")
+    .pip_install_from_requirements("requirements.txt")
+    .pip_install("pipecat-ai[daily]")
+    .add_local_dir("src", remote_path="/root/src")
+)
+
+# container specifications for the Pipecat pipeline
+bot_image = (
+    modal.Image.debian_slim(python_version="3.13")
+    .apt_install("ffmpeg")
+    .pip_install_from_requirements("requirements.txt")
+    .pip_install("pipecat-ai[daily,elevenlabs,openai,silero,google]")
+    .add_local_dir("src", remote_path="/root/src")
+)
+
+app = modal.App("pipecat-modal", secrets=[modal.Secret.from_dotenv()])
+
+router = APIRouter()
+
+bot_jobs = {}
+daily_helpers = {}
+
+# Names of all supported bot implementations
+# These correspond to the bot files in the src directory
+BotName = Literal["openai", "gemini", "vllm"]
+
+
+def cleanup():
+    """Cleanup function to terminate all bot processes.
+
+    Called during server shutdown.
+    """
+    for entry in bot_jobs.values():
+        func = modal.FunctionCall.from_id(entry[0])
+        if func:
+            func.cancel()
+
+
+def get_bot_file(bot_name: BotName) -> str:
+    """Retrieve the bot file name corresponding to the provided bot_name.
+
+    Args:
+        bot_name (BotName): The name of the bot (e.g., 'openai', 'gemini', 'vllm').
+
+    Returns:
+        str: The file name corresponding to the bot implementation.
+
+    Raises:
+        ValueError: If the bot name is invalid or not supported.
+    """
+    # bot_implementation = os.getenv("BOT_IMPLEMENTATION", "openai").lower().strip()
+    bot_implementation = bot_name.lower().strip()
+    if not bot_implementation:
+        bot_implementation = "openai"
+    if bot_implementation not in ["openai", "gemini", "vllm"]:
+        raise ValueError(
+            f"Invalid BOT_IMPLEMENTATION: {bot_implementation}. Must be 'openai' or 'gemini' or 'vllm'"
+        )
+
+    return f"bot_{bot_implementation}"
+
+
+def get_runner(path: str, bot_file: str) -> callable:
+    """Dynamically import the run_bot function based on the bot name.
+
+    Args:
+        path (str): The path to the bot files (e.g., 'src').
+        bot_file (str): The file name of the bot implementation (e.g., 'openai', 'gemini', 'vllm').
+
+    Returns:
+        function: The run_bot function from the specified bot module.
+
+    Raises:
+        ImportError: If the specified bot module or run_bot function is not found.
+    """
+    try:
+        # Dynamically construct the module name
+        module_name = f"{path}.{bot_file}"
+        # Import the module
+        module = importlib.import_module(module_name)
+        # Get the run_bot function from the module
+        return getattr(module, "run_bot")
+    except (ImportError, AttributeError) as e:
+        raise ImportError(f"Failed to import run_bot from {module_name}: {e}")
+
+
+async def create_room_and_token() -> tuple[str, str]:
+    """Create a Daily room and generate an authentication token.
+
+    This function checks for existing room URL and token in the environment variables.
+    If not found, it creates a new room using the Daily API and generates a token for it.
+
+    Returns:
+        tuple[str, str]: A tuple containing the room URL and the authentication token.
+
+    Raises:
+        HTTPException: If room creation or token generation fails.
+    """
+    from pipecat.transports.services.helpers.daily_rest import DailyRoomParams
+
+    room_url = os.getenv("DAILY_SAMPLE_ROOM_URL", None)
+    token = os.getenv("DAILY_SAMPLE_ROOM_TOKEN", None)
+    if not room_url:
+        room = await daily_helpers["rest"].create_room(DailyRoomParams())
+        if not room.url:
+            raise HTTPException(status_code=500, detail="Failed to create room")
+        room_url = room.url
+
+        token = await daily_helpers["rest"].get_token(room_url)
+        if not token:
+            raise HTTPException(status_code=500, detail=f"Failed to get token for room: {room_url}")
+
+    return room_url, token
+
+
+@app.function(image=bot_image, min_containers=1)
+async def bot_runner(room_url, token, bot_name: BotName = "openai"):
+    """Launch the provided bot process, providing the given room URL and token for the bot to join.
+
+    Args:
+        room_url (str): The URL of the Daily room where the bot and client will communicate.
+        token (str): The authentication token for the room.
+        bot_name (BotName): The name of the bot implementation to use. Defaults to "openai".
+
+    Raises:
+        HTTPException: If the bot pipeline fails to start.
+    """
+    try:
+        path = "src"
+        bot_file = get_bot_file(bot_name)
+        run_bot = get_runner(path, bot_file)
+
+        print(f"Starting bot process: {bot_file} -u {room_url} -t {token}")
+        await run_bot(room_url, token)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to start bot pipeline: {e}")
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """FastAPI lifespan manager that handles startup and shutdown tasks.
+
+    - Creates aiohttp session
+    - Initializes Daily API helper
+    - Cleans up resources on shutdown
+    """
+    from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper
+
+    aiohttp_session = aiohttp.ClientSession()
+    daily_helpers["rest"] = DailyRESTHelper(
+        daily_api_key=os.getenv("DAILY_API_KEY", ""),
+        daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"),
+        aiohttp_session=aiohttp_session,
+    )
+    yield
+    await aiohttp_session.close()
+    cleanup()
+
+
+class ConnectData(BaseModel):
+    """Data provided by client to specify the bot pipeline.
+
+    Attributes:
+        bot_name (BotName): The name of the bot to connect to. Defaults to "openai".
+    """
+
+    bot_name: BotName = "openai"
+
+
+async def start(data: ConnectData):
+    """Internal method to start a bot agent and return the room URL and token.
+
+    Args:
+        data (ConnectData): The data containing the bot name to use.
+
+    Returns:
+        tuple[str, str]: A tuple containing the room URL and token.
+    """
+    room_url, token = await create_room_and_token()
+    launch_bot_func = modal.Function.from_name("pipecat-modal", "bot_runner")
+    function_id = launch_bot_func.spawn(room_url, token, data.bot_name)
+    bot_jobs[function_id] = (function_id, room_url)
+
+    return room_url, token
+
+
+@router.get("/")
+async def start_agent():
+    """A user endpoint for launching a bot agent and redirecting to the created room URL.
+
+    This function retrieves the bot implementation from the environment,
+    starts the bot agent, and redirects the user to the room URL to
+    interact with the bot through a Daily Prebuilt Interface.
+
+    Returns:
+        RedirectResponse: A response that redirects to the room URL.
+    """
+    bot_name = os.getenv("BOT_IMPLEMENTATION", "openai").lower().strip()
+    print(f"Starting bot: {bot_name}")
+    room_url, token = await start(ConnectData(bot_name=bot_name))
+
+    return RedirectResponse(room_url)
+
+
+@router.post("/connect")
+async def rtvi_connect(data: ConnectData) -> Dict[Any, Any]:
+    """A user endpoint for launching a bot agent and retrieving the room/token credentials.
+
+    This function retrieves the bot implementation from the request, if provided,
+    starts the bot agent, and returns the room URL and token for the bot. This allows the
+    client to then connect to the bot using their own RTVI interface.
+
+    Args:
+        data (ConnectData): Optional. The data containing the bot name to use.
+
+    Returns:
+        Dict[Any, Any]: A dictionary containing the room URL and token.
+    """
+    print(f"Starting bot: {data.bot_name}")
+    if data is None or not data.bot_name:
+        data.bot_name = os.getenv("BOT_IMPLEMENTATION", "openai").lower().strip()
+    room_url, token = await start(data)
+
+    return {"room_url": room_url, "token": token}
+
+
+@router.get("/status/{fid}")
+def get_status(fid: str):
+    """Retrieve the status of a bot process by its function ID.
+
+    Args:
+        fid (str): The function ID of the bot process.
+
+    Returns:
+        JSONResponse: A JSON response containing the bot's status and result code.
+
+    Raises:
+        HTTPException: If the bot process with the given ID is not found.
+    """
+    func = modal.FunctionCall.from_id(fid)
+    if not func:
+        raise HTTPException(status_code=404, detail=f"Bot with process id: {fid} not found")
+
+    try:
+        result = func.get(timeout=0)
+        return JSONResponse({"bot_id": fid, "status": "finished", "code": result})
+    except modal.exception.OutputExpiredError:
+        return JSONResponse({"bot_id": fid, "status": "finished", "code": 404})
+    except TimeoutError:
+        return JSONResponse({"bot_id": fid, "status": "running", "code": 202})
+
+
+@app.function(image=web_image, min_containers=1)
+@modal.concurrent(max_inputs=1)
+@modal.asgi_app()
+def fastapi_app():
+    """Create and configure the FastAPI application.
+
+    This function initializes the FastAPI app with middleware, routes, and lifespan management.
+    It is decorated to be used as a Modal ASGI app.
+    """
+    from fastapi.middleware.cors import CORSMiddleware
+
+    # Initialize FastAPI app
+    web_app = FastAPI(lifespan=lifespan)
+
+    web_app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    # Include the endpoints from endpoints.py
+    web_app.include_router(router)
+
+    return web_app

examples/deployment/modal-example/server/env.example

@@ -0,0 +1,14 @@
+DAILY_API_KEY=
+
+# determines which bot file to default to: 'openai', 'gemini', or 'vllm'
+BOT_IMPLEMENTATION=openai
+
+# needed for the openai bot pipeline
+OPENAI_API_KEY=
+ELEVENLABS_API_KEY=
+
+# needed for the gemini live bot pipeline
+GOOGLE_API_KEY=
+
+# needed if you modified the API Key for your self-hosted LLM
+VLLM_API_KEY=

examples/deployment/modal-example/server/requirements.txt

@@ -0,0 +1,2 @@
+python-dotenv==1.0.1
+modal==0.71.3

examples/deployment/modal-example/server/src/bot_gemini.py

@@ -0,0 +1,197 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Gemini Bot Implementation.
+
+This module implements a chatbot using Google's Gemini Multimodal Live model.
+It includes:
+- Real-time audio/video interaction through Daily
+- Animated robot avatar
+- Speech-to-speech model
+
+The bot runs as part of a pipeline that processes audio/video frames and manages
+the conversation flow using Gemini's streaming capabilities.
+"""
+
+import os
+import sys
+
+from dotenv import load_dotenv
+from loguru import logger
+from PIL import Image
+
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.frames.frames import (
+    BotStartedSpeakingFrame,
+    BotStoppedSpeakingFrame,
+    Frame,
+    OutputImageRawFrame,
+    SpriteFrame,
+)
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor
+from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+
+load_dotenv(override=True)
+
+try:
+    logger.remove(0)
+    logger.add(sys.stderr, level="DEBUG")
+except ValueError:
+    # Handle the case where logger is already initialized
+    pass
+
+sprites = []
+script_dir = os.path.dirname(__file__)
+
+for i in range(1, 26):
+    # Build the full path to the image file
+    full_path = os.path.join(script_dir, f"assets/robot0{i}.png")
+    # Get the filename without the extension to use as the dictionary key
+    # Open the image and convert it to bytes
+    with Image.open(full_path) as img:
+        sprites.append(OutputImageRawFrame(image=img.tobytes(), size=img.size, format=img.format))
+
+# Create a smooth animation by adding reversed frames
+flipped = sprites[::-1]
+sprites.extend(flipped)
+
+# Define static and animated states
+quiet_frame = sprites[0]  # Static frame for when bot is listening
+talking_frame = SpriteFrame(images=sprites)  # Animation sequence for when bot is talking
+
+
+class TalkingAnimation(FrameProcessor):
+    """Manages the bot's visual animation states.
+
+    Switches between static (listening) and animated (talking) states based on
+    the bot's current speaking status.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._is_talking = False
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process incoming frames and update animation state.
+
+        Args:
+            frame: The incoming frame to process
+            direction: The direction of frame flow in the pipeline
+        """
+        await super().process_frame(frame, direction)
+
+        # Switch to talking animation when bot starts speaking
+        if isinstance(frame, BotStartedSpeakingFrame):
+            if not self._is_talking:
+                await self.push_frame(talking_frame)
+                self._is_talking = True
+        # Return to static frame when bot stops speaking
+        elif isinstance(frame, BotStoppedSpeakingFrame):
+            await self.push_frame(quiet_frame)
+            self._is_talking = False
+
+        await self.push_frame(frame, direction)
+
+
+async def run_bot(room_url: str, token: str):
+    """Main bot execution function.
+
+    Sets up and runs the bot pipeline including:
+    - Daily video transport with specific audio parameters
+    - Gemini Live multimodal model integration
+    - Voice activity detection
+    - Animation processing
+    - RTVI event handling
+    """
+    # Set up Daily transport with specific audio/video parameters for Gemini
+    transport = DailyTransport(
+        room_url,
+        token,
+        "Chatbot",
+        DailyParams(
+            audio_out_enabled=True,
+            camera_out_enabled=True,
+            camera_out_width=1024,
+            camera_out_height=576,
+            vad_enabled=True,
+            vad_audio_passthrough=True,
+            vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
+        ),
+    )
+
+    # Initialize the Gemini Multimodal Live model
+    llm = GeminiMultimodalLiveLLMService(
+        api_key=os.getenv("GOOGLE_API_KEY"),
+        voice_id="Puck",  # Aoede, Charon, Fenrir, Kore, Puck
+        transcribe_user_audio=True,
+    )
+
+    messages = [
+        {
+            "role": "user",
+            "content": "You are Chatbot, a friendly, helpful robot. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way, but keep your responses brief. Start by introducing yourself.",
+        },
+    ]
+
+    # Set up conversation context and management
+    # The context_aggregator will automatically collect conversation context
+    context = OpenAILLMContext(messages)
+    context_aggregator = llm.create_context_aggregator(context)
+
+    ta = TalkingAnimation()
+
+    #
+    # RTVI events for Pipecat client UI
+    #
+    rtvi = RTVIProcessor(config=RTVIConfig(config=[]))
+
+    pipeline = Pipeline(
+        [
+            transport.input(),
+            rtvi,
+            context_aggregator.user(),
+            llm,
+            ta,
+            transport.output(),
+            context_aggregator.assistant(),
+        ]
+    )
+
+    task = PipelineTask(
+        pipeline,
+        params=PipelineParams(
+            enable_metrics=True,
+            enable_usage_metrics=True,
+        ),
+        observers=[RTVIObserver(rtvi)],
+    )
+    await task.queue_frame(quiet_frame)
+
+    @rtvi.event_handler("on_client_ready")
+    async def on_client_ready(rtvi):
+        await rtvi.set_bot_ready()
+        # Kick off the conversation
+        await task.queue_frames([context_aggregator.user().get_context_frame()])
+
+    @transport.event_handler("on_first_participant_joined")
+    async def on_first_participant_joined(transport, participant):
+        await transport.capture_participant_transcription(participant["id"])
+
+    @transport.event_handler("on_participant_left")
+    async def on_participant_left(transport, participant, reason):
+        print(f"Participant left: {participant}")
+        await task.cancel()
+
+    runner = PipelineRunner()
+
+    await runner.run(task)

examples/deployment/modal-example/server/src/bot_openai.py

@@ -0,0 +1,225 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""OpenAI Bot Implementation.
+
+This module implements a chatbot using OpenAI's GPT-4 model for natural language
+processing. It includes:
+- Real-time audio/video interaction through Daily
+- Animated robot avatar
+- Text-to-speech using ElevenLabs
+- Support for both English and Spanish
+
+The bot runs as part of a pipeline that processes audio/video frames and manages
+the conversation flow.
+"""
+
+import os
+import sys
+
+from dotenv import load_dotenv
+from loguru import logger
+from PIL import Image
+
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.frames.frames import (
+    BotStartedSpeakingFrame,
+    BotStoppedSpeakingFrame,
+    Frame,
+    OutputImageRawFrame,
+    SpriteFrame,
+)
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor
+from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+
+load_dotenv(override=True)
+
+try:
+    logger.remove(0)
+    logger.add(sys.stderr, level="DEBUG")
+except ValueError:
+    # Handle the case where logger is already initialized
+    pass
+
+sprites = []
+script_dir = os.path.dirname(__file__)
+
+# Load sequential animation frames
+for i in range(1, 26):
+    # Build the full path to the image file
+    full_path = os.path.join(script_dir, f"assets/robot0{i}.png")
+    # Get the filename without the extension to use as the dictionary key
+    # Open the image and convert it to bytes
+    with Image.open(full_path) as img:
+        sprites.append(OutputImageRawFrame(image=img.tobytes(), size=img.size, format=img.format))
+
+# Create a smooth animation by adding reversed frames
+flipped = sprites[::-1]
+sprites.extend(flipped)
+
+# Define static and animated states
+quiet_frame = sprites[0]  # Static frame for when bot is listening
+talking_frame = SpriteFrame(images=sprites)  # Animation sequence for when bot is talking
+
+
+class TalkingAnimation(FrameProcessor):
+    """Manages the bot's visual animation states.
+
+    Switches between static (listening) and animated (talking) states based on
+    the bot's current speaking status.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._is_talking = False
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process incoming frames and update animation state.
+
+        Args:
+            frame: The incoming frame to process
+            direction: The direction of frame flow in the pipeline
+        """
+        await super().process_frame(frame, direction)
+
+        # Switch to talking animation when bot starts speaking
+        if isinstance(frame, BotStartedSpeakingFrame):
+            if not self._is_talking:
+                await self.push_frame(talking_frame)
+                self._is_talking = True
+        # Return to static frame when bot stops speaking
+        elif isinstance(frame, BotStoppedSpeakingFrame):
+            await self.push_frame(quiet_frame)
+            self._is_talking = False
+
+        await self.push_frame(frame, direction)
+
+
+async def run_bot(room_url: str, token: str):
+    """Main bot execution function.
+
+    Sets up and runs the bot pipeline including:
+    - Daily video transport
+    - Speech-to-text and text-to-speech services
+    - Language model integration
+    - Animation processing
+    - RTVI event handling
+    """
+    # Set up Daily transport with video/audio parameters
+    transport = DailyTransport(
+        room_url,
+        token,
+        "Chatbot",
+        DailyParams(
+            audio_out_enabled=True,
+            camera_out_enabled=True,
+            camera_out_width=1024,
+            camera_out_height=576,
+            vad_enabled=True,
+            vad_analyzer=SileroVADAnalyzer(),
+            transcription_enabled=True,
+            #
+            # Spanish
+            #
+            # transcription_settings=DailyTranscriptionSettings(
+            #     language="es",
+            #     tier="nova",
+            #     model="2-general"
+            # )
+        ),
+    )
+
+    # Initialize text-to-speech service
+    tts = ElevenLabsTTSService(
+        api_key=os.getenv("ELEVENLABS_API_KEY"),
+        #
+        # English
+        #
+        voice_id="SAz9YHcvj6GT2YYXdXww",
+        #
+        # Spanish
+        #
+        # model="eleven_multilingual_v2",
+        # voice_id="gD1IexrzCvsXPHUuT0s3",
+    )
+
+    # Initialize LLM service
+    llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
+
+    messages = [
+        {
+            "role": "system",
+            #
+            # English
+            #
+            "content": "You are an incessant one-upper. Start by asking the user how their day is going.",
+            #
+            # Spanish
+            #
+            # "content": "Eres Chatbot, un amigable y útil robot. Tu objetivo es demostrar tus capacidades de una manera breve. Tus respuestas se convertiran a audio así que nunca no debes incluir caracteres especiales. Contesta a lo que el usuario pregunte de una manera creativa, útil y breve. Empieza por presentarte a ti mismo.",
+        },
+    ]
+
+    # Set up conversation context and management
+    # The context_aggregator will automatically collect conversation context
+    context = OpenAILLMContext(messages)
+    context_aggregator = llm.create_context_aggregator(context)
+
+    ta = TalkingAnimation()
+
+    #
+    # RTVI events for Pipecat client UI
+    #
+    rtvi = RTVIProcessor(config=RTVIConfig(config=[]))
+
+    pipeline = Pipeline(
+        [
+            transport.input(),
+            rtvi,
+            context_aggregator.user(),
+            llm,
+            tts,
+            ta,
+            transport.output(),
+            context_aggregator.assistant(),
+        ]
+    )
+
+    task = PipelineTask(
+        pipeline,
+        params=PipelineParams(
+            enable_metrics=True,
+            enable_usage_metrics=True,
+        ),
+        observers=[RTVIObserver(rtvi)],
+    )
+    await task.queue_frame(quiet_frame)
+
+    @rtvi.event_handler("on_client_ready")
+    async def on_client_ready(rtvi):
+        await rtvi.set_bot_ready()
+        # Kick off the conversation
+        await task.queue_frames([context_aggregator.user().get_context_frame()])
+
+    @transport.event_handler("on_first_participant_joined")
+    async def on_first_participant_joined(transport, participant):
+        await transport.capture_participant_transcription(participant["id"])
+
+    @transport.event_handler("on_participant_left")
+    async def on_participant_left(transport, participant, reason):
+        print(f"Participant left: {participant}")
+        await task.cancel()
+
+    runner = PipelineRunner()
+
+    await runner.run(task)

examples/deployment/modal-example/server/src/bot_vllm.py

@@ -0,0 +1,238 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""OpenAI Bot Implementation.
+
+This module implements a chatbot using OpenAI's GPT-4 model for natural language
+processing. It includes:
+- Real-time audio/video interaction through Daily
+- Animated robot avatar
+- Text-to-speech using ElevenLabs
+- Support for both English and Spanish
+
+The bot runs as part of a pipeline that processes audio/video frames and manages
+the conversation flow.
+"""
+
+import os
+import sys
+from typing import List
+
+from dotenv import load_dotenv
+from loguru import logger
+from openai.types.chat import ChatCompletionMessageParam
+from PIL import Image
+
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.frames.frames import (
+    BotStartedSpeakingFrame,
+    BotStoppedSpeakingFrame,
+    Frame,
+    OutputImageRawFrame,
+    SpriteFrame,
+)
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor
+from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+
+load_dotenv(override=True)
+
+try:
+    logger.remove(0)
+    logger.add(sys.stderr, level="DEBUG")
+except ValueError:
+    # Handle the case where logger is already initialized
+    pass
+
+# REPLACE WITH YOUR MODAL URL ENDPOINT
+modal_url = "https://<Modal workspace>--example-vllm-openai-compatible-serve.modal.run"
+api_key = os.getenv("VLLM_API_KEY", "super-secret-key")
+
+
+sprites = []
+script_dir = os.path.dirname(__file__)
+
+# Load sequential animation frames
+for i in range(1, 26):
+    # Build the full path to the image file
+    full_path = os.path.join(script_dir, f"assets/robot0{i}.png")
+    # Get the filename without the extension to use as the dictionary key
+    # Open the image and convert it to bytes
+    with Image.open(full_path) as img:
+        sprites.append(OutputImageRawFrame(image=img.tobytes(), size=img.size, format=img.format))
+
+# Create a smooth animation by adding reversed frames
+flipped = sprites[::-1]
+sprites.extend(flipped)
+
+# Define static and animated states
+quiet_frame = sprites[0]  # Static frame for when bot is listening
+talking_frame = SpriteFrame(images=sprites)  # Animation sequence for when bot is talking
+
+
+class TalkingAnimation(FrameProcessor):
+    """Manages the bot's visual animation states.
+
+    Switches between static (listening) and animated (talking) states based on
+    the bot's current speaking status.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._is_talking = False
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process incoming frames and update animation state.
+
+        Args:
+            frame: The incoming frame to process
+            direction: The direction of frame flow in the pipeline
+        """
+        await super().process_frame(frame, direction)
+
+        # Switch to talking animation when bot starts speaking
+        if isinstance(frame, BotStartedSpeakingFrame):
+            if not self._is_talking:
+                await self.push_frame(talking_frame)
+                self._is_talking = True
+        # Return to static frame when bot stops speaking
+        elif isinstance(frame, BotStoppedSpeakingFrame):
+            await self.push_frame(quiet_frame)
+            self._is_talking = False
+
+        await self.push_frame(frame, direction)
+
+
+async def run_bot(room_url: str, token: str):
+    """Main bot execution function.
+
+    Sets up and runs the bot pipeline including:
+    - Daily video transport
+    - Speech-to-text and text-to-speech services
+    - Language model integration
+    - Animation processing
+    - RTVI event handling
+    """
+    # Set up Daily transport with video/audio parameters
+    transport = DailyTransport(
+        room_url,
+        token,
+        "Chatbot",
+        DailyParams(
+            audio_out_enabled=True,
+            camera_out_enabled=True,
+            camera_out_width=1024,
+            camera_out_height=576,
+            vad_enabled=True,
+            vad_analyzer=SileroVADAnalyzer(),
+            transcription_enabled=True,
+            #
+            # Spanish
+            #
+            # transcription_settings=DailyTranscriptionSettings(
+            #     language="es",
+            #     tier="nova",
+            #     model="2-general"
+            # )
+        ),
+    )
+
+    # Initialize text-to-speech service
+    tts = ElevenLabsTTSService(
+        api_key=os.getenv("ELEVENLABS_API_KEY"),
+        #
+        # English
+        #
+        voice_id="D38z5RcWu1voky8WS1ja",
+        #
+        # Spanish
+        #
+        # model="eleven_multilingual_v2",
+        # voice_id="gD1IexrzCvsXPHUuT0s3",
+    )
+
+    # Initialize LLM service
+    llm = OpenAILLMService(
+        # To use OpenAI
+        api_key=api_key,
+        # Or, to use a local vLLM (or similar) api server
+        model="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16",
+        base_url=f"{modal_url}/v1",
+    )
+
+    messages = [
+        {
+            "role": "system",
+            #
+            # English
+            #
+            "content": "You are a salesman for Modal, the cloud-native serverless Python computing platform.",
+            #
+            # Spanish
+            #
+            # "content": "Eres Chatbot, un amigable y útil robot. Tu objetivo es demostrar tus capacidades de una manera breve. Tus respuestas se convertiran a audio así que nunca no debes incluir caracteres especiales. Contesta a lo que el usuario pregunte de una manera creativa, útil y breve. Empieza por presentarte a ti mismo.",
+        },
+    ]
+
+    # Set up conversation context and management
+    # The context_aggregator will automatically collect conversation context
+    context = OpenAILLMContext(messages)
+    context_aggregator = llm.create_context_aggregator(context)
+
+    ta = TalkingAnimation()
+
+    #
+    # RTVI events for Pipecat client UI
+    #
+    rtvi = RTVIProcessor(config=RTVIConfig(config=[]))
+
+    pipeline = Pipeline(
+        [
+            transport.input(),
+            rtvi,
+            context_aggregator.user(),
+            llm,
+            tts,
+            ta,
+            transport.output(),
+            context_aggregator.assistant(),
+        ]
+    )
+
+    task = PipelineTask(
+        pipeline,
+        params=PipelineParams(
+            enable_metrics=True,
+            enable_usage_metrics=True,
+        ),
+        observers=[RTVIObserver(rtvi)],
+    )
+    await task.queue_frame(quiet_frame)
+
+    @rtvi.event_handler("on_client_ready")
+    async def on_client_ready(rtvi):
+        await rtvi.set_bot_ready()
+        # Kick off the conversation
+        await task.queue_frames([context_aggregator.user().get_context_frame()])
+
+    @transport.event_handler("on_first_participant_joined")
+    async def on_first_participant_joined(transport, participant):
+        await transport.capture_participant_transcription(participant["id"])
+
+    @transport.event_handler("on_participant_left")
+    async def on_participant_left(transport, participant, reason):
+        print(f"Participant left: {participant}")
+        await task.cancel()
+
+    runner = PipelineRunner()
+
+    await runner.run(task)

examples/deployment/modal-example/server/src/runner.py

@@ -0,0 +1,84 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import argparse
+import asyncio
+import importlib
+import os
+
+
+def get_bot_file(arg_bot: str | None) -> str:
+    bot_implementation = arg_bot or os.getenv("BOT_IMPLEMENTATION", "openai").lower().strip()
+    if not bot_implementation:
+        bot_implementation = "openai"
+    if bot_implementation not in ["openai", "gemini", "vllm"]:
+        raise ValueError(
+            f"Invalid BOT_IMPLEMENTATION: {bot_implementation}. Must be 'openai' or 'gemini'"
+        )
+    return f"bot_{bot_implementation}"
+
+
+def get_runner(bot_file: str):
+    """Dynamically import the run_bot function based on the bot name.
+
+    Args:
+        bot_name (str): The name of the bot implementation (e.g., 'openai', 'gemini').
+
+    Returns:
+        function: The run_bot function from the specified bot module.
+
+    Raises:
+        ImportError: If the specified bot module or run_bot function is not found.
+    """
+    try:
+        # Dynamically construct the module name
+        module_name = f"{bot_file}"
+        # Import the module
+        module = importlib.import_module(module_name)
+        # Get the run_bot function from the module
+        return getattr(module, "run_bot")
+    except (ImportError, AttributeError) as e:
+        raise ImportError(f"Failed to import run_bot from {module_name}: {e}")
+
+
+def main():
+    """Parse the args to launch the appropriate bot using the given room/token."""
+    parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
+    parser.add_argument(
+        "-u", "--url", type=str, required=False, help="URL of the Daily room to join"
+    )
+    parser.add_argument(
+        "-t",
+        "--token",
+        type=str,
+        required=False,
+        help="Daily room token",
+    )
+    parser.add_argument(
+        "-b",
+        "--bot",
+        type=str,
+        required=False,
+        help="Bot runner to use (e.g., openai, gemini)",
+    )
+
+    args, unknown = parser.parse_known_args()
+
+    url = args.url or os.getenv("DAILY_SAMPLE_ROOM_URL")
+    token = args.token or os.getenv("DAILY_SAMPLE_ROOM_TOKEN")
+    bot_file = get_bot_file(args.bot)
+
+    if not url:
+        raise Exception(
+            "No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL."
+        )
+
+    run_bot = get_runner(bot_file)
+    asyncio.run(run_bot(url, token))
+
+
+if __name__ == "__main__":
+    main()

examples/deployment/pipecat-cloud-daily-pstn-server/README.md

@@ -100,7 +100,28 @@ phone numbers with valid values for your use case.
 
 ### Dialin Request
 
-The server will receive a request when a call is received from Daily.
+The server will receive a request when a call is received from Daily. 
+The payload that the webhook received is as follows:
+```json
+{
+  // for dial-in from webhook
+  "To": "+14152251493",
+  "From": "+14158483432",
+  "callId": "string-contains-uuid",
+  "callDomain": "string-contains-uuid",
+  "sipHeaders": {
+    "X-My-Custom-Header": "value",
+    "x-caller": "+1234567890",
+    "x-called": "+1987654321", 
+   },
+}
+```
+The `To`, `From`, `callId`, `callDomain` fields are converted to 
+`snake_case` and mapped to `dialin_settings`. In addition, `sipHeader` 
+contains any custom SIP headers received by Daily on the SIP 
+interconnect address (`sip_uri`). These are headers sent from 
+Twilio or other external SIP platforms, for example, to send the 
+caller's phone number.
 
 ### Dialout Request
 
@@ -158,6 +179,7 @@ curl -X POST http://localhost:3000/api/dial \
     "From": "+1987654321",
     "callId": "call-uuid-123",
     "callDomain": "domain-uuid-456",
+    "sipHeader": {},
     "dialout_settings": [
       {
         "phoneNumber": "+1234567890",

examples/deployment/pipecat-cloud-daily-pstn-server/fastapi-webhook-server/server.py

@@ -39,6 +39,11 @@ class RoomRequest(BaseModel):
         None, description="A flag to perform voicemail or answeing-machine detection"
     )
     call_transfer: Optional[Dict[str, Any]] = Field(None, description="to initiate a call transfer")
+    sipHeaders: Optional[Dict[str, Any]] = Field(
+        None,
+        alias="sip_headers",
+        description="Custom SIP headers received from the external SIP provider",
+    )
 
     class Config:
         populate_by_name = True
@@ -57,6 +62,14 @@ class RoomRequest(BaseModel):
     "callDomain": "string-contains-uuid"
     These need to be remapped to dialin_settings
 
+    In addition, we may receive in the body that can be 
+    sent to the bot as a custom field, sip_headers
+    "sipHeaders": {
+        "X-My-Custom-Header": "value",
+        "x-caller": "+14158483432",
+        "x-called": "+14152251493",
+    },
+
     "dialout_settings": [
         {"phoneNumber": "+14158483432", "callerId": "+14152251493"}, 
         {"sipUri": "sip:username@sip.hostname"}
@@ -141,6 +154,7 @@ async def dial(request: RoomRequest, raw_request: Request):
             "display_name": request.From,
             "sip_mode": "dial-in",
             "num_endpoints": 2 if request.call_transfer is not None else 1,
+            "codecs": {"audio": ["OPUS"]},
         }
         daily_room_properties["sip"] = sip_config
 
@@ -156,6 +170,7 @@ async def dial(request: RoomRequest, raw_request: Request):
             "dialout_settings": request.dialout_settings,
             "voicemail_detection": request.voicemail_detection,
             "call_transfer": request.call_transfer,
+            "sip_headers": request.sipHeaders,  # passing the SIP headers to the bot
         },
     }
 

examples/deployment/pipecat-cloud-daily-pstn-server/nextjs-webhook-server/package-lock.json

@@ -215,10 +215,9 @@
       }
     },
     "node_modules/@next/env": {
-      "version": "14.2.26",
-      "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.26.tgz",
-      "integrity": "sha512-vO//GJ/YBco+H7xdQhzJxF7ub3SUwft76jwaeOyVVQFHCi5DCnkP16WHB+JBylo4vOKPoZBlR94Z8xBxNBdNJA==",
-      "license": "MIT"
+      "version": "14.2.30",
+      "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.30.tgz",
+      "integrity": "sha512-KBiBKrDY6kxTQWGzKjQB7QirL3PiiOkV7KW98leHFjtVRKtft76Ra5qSA/SL75xT44dp6hOcqiiJ6iievLOYug=="
     },
     "node_modules/@next/eslint-plugin-next": {
       "version": "14.2.25",
@@ -231,13 +230,12 @@
       }
     },
     "node_modules/@next/swc-darwin-arm64": {
-      "version": "14.2.26",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.26.tgz",
-      "integrity": "sha512-zDJY8gsKEseGAxG+C2hTMT0w9Nk9N1Sk1qV7vXYz9MEiyRoF5ogQX2+vplyUMIfygnjn9/A04I6yrUTRTuRiyQ==",
+      "version": "14.2.30",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.30.tgz",
+      "integrity": "sha512-EAqfOTb3bTGh9+ewpO/jC59uACadRHM6TSA9DdxJB/6gxOpyV+zrbqeXiFTDy9uV6bmipFDkfpAskeaDcO+7/g==",
       "cpu": [
         "arm64"
       ],
-      "license": "MIT",
       "optional": true,
       "os": [
         "darwin"
@@ -247,13 +245,12 @@
       }
     },
     "node_modules/@next/swc-darwin-x64": {
-      "version": "14.2.26",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.26.tgz",
-      "integrity": "sha512-U0adH5ryLfmTDkahLwG9sUQG2L0a9rYux8crQeC92rPhi3jGQEY47nByQHrVrt3prZigadwj/2HZ1LUUimuSbg==",
+      "version": "14.2.30",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.30.tgz",
+      "integrity": "sha512-TyO7Wz1IKE2kGv8dwQ0bmPL3s44EKVencOqwIY69myoS3rdpO1NPg5xPM5ymKu7nfX4oYJrpMxv8G9iqLsnL4A==",
       "cpu": [
         "x64"
       ],
-      "license": "MIT",
       "optional": true,
       "os": [
         "darwin"
@@ -263,13 +260,12 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-gnu": {
-      "version": "14.2.26",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.26.tgz",
-      "integrity": "sha512-SINMl1I7UhfHGM7SoRiw0AbwnLEMUnJ/3XXVmhyptzriHbWvPPbbm0OEVG24uUKhuS1t0nvN/DBvm5kz6ZIqpg==",
+      "version": "14.2.30",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.30.tgz",
+      "integrity": "sha512-I5lg1fgPJ7I5dk6mr3qCH1hJYKJu1FsfKSiTKoYwcuUf53HWTrEkwmMI0t5ojFKeA6Vu+SfT2zVy5NS0QLXV4Q==",
       "cpu": [
         "arm64"
       ],
-      "license": "MIT",
       "optional": true,
       "os": [
         "linux"
@@ -279,13 +275,12 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-musl": {
-      "version": "14.2.26",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.26.tgz",
-      "integrity": "sha512-s6JaezoyJK2DxrwHWxLWtJKlqKqTdi/zaYigDXUJ/gmx/72CrzdVZfMvUc6VqnZ7YEvRijvYo+0o4Z9DencduA==",
+      "version": "14.2.30",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.30.tgz",
+      "integrity": "sha512-8GkNA+sLclQyxgzCDs2/2GSwBc92QLMrmYAmoP2xehe5MUKBLB2cgo34Yu242L1siSkwQkiV4YLdCnjwc/Micw==",
       "cpu": [
         "arm64"
       ],
-      "license": "MIT",
       "optional": true,
       "os": [
         "linux"
@@ -295,13 +290,12 @@
       }
     },
     "node_modules/@next/swc-linux-x64-gnu": {
-      "version": "14.2.26",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.26.tgz",
-      "integrity": "sha512-FEXeUQi8/pLr/XI0hKbe0tgbLmHFRhgXOUiPScz2hk0hSmbGiU8aUqVslj/6C6KA38RzXnWoJXo4FMo6aBxjzg==",
+      "version": "14.2.30",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.30.tgz",
+      "integrity": "sha512-8Ly7okjssLuBoe8qaRCcjGtcMsv79hwzn/63wNeIkzJVFVX06h5S737XNr7DZwlsbTBDOyI6qbL2BJB5n6TV/w==",
       "cpu": [
         "x64"
       ],
-      "license": "MIT",
       "optional": true,
       "os": [
         "linux"
@@ -311,13 +305,12 @@
       }
     },
     "node_modules/@next/swc-linux-x64-musl": {
-      "version": "14.2.26",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.26.tgz",
-      "integrity": "sha512-BUsomaO4d2DuXhXhgQCVt2jjX4B4/Thts8nDoIruEJkhE5ifeQFtvW5c9JkdOtYvE5p2G0hcwQ0UbRaQmQwaVg==",
+      "version": "14.2.30",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.30.tgz",
+      "integrity": "sha512-dBmV1lLNeX4mR7uI7KNVHsGQU+OgTG5RGFPi3tBJpsKPvOPtg9poyav/BYWrB3GPQL4dW5YGGgalwZ79WukbKQ==",
       "cpu": [
         "x64"
       ],
-      "license": "MIT",
       "optional": true,
       "os": [
         "linux"
@@ -327,13 +320,12 @@
       }
     },
     "node_modules/@next/swc-win32-arm64-msvc": {
-      "version": "14.2.26",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.26.tgz",
-      "integrity": "sha512-5auwsMVzT7wbB2CZXQxDctpWbdEnEW/e66DyXO1DcgHxIyhP06awu+rHKshZE+lPLIGiwtjo7bsyeuubewwxMw==",
+      "version": "14.2.30",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.30.tgz",
+      "integrity": "sha512-6MMHi2Qc1Gkq+4YLXAgbYslE1f9zMGBikKMdmQRHXjkGPot1JY3n5/Qrbg40Uvbi8//wYnydPnyvNhI1DMUW1g==",
       "cpu": [
         "arm64"
       ],
-      "license": "MIT",
       "optional": true,
       "os": [
         "win32"
@@ -343,13 +335,12 @@
       }
     },
     "node_modules/@next/swc-win32-ia32-msvc": {
-      "version": "14.2.26",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.26.tgz",
-      "integrity": "sha512-GQWg/Vbz9zUGi9X80lOeGsz1rMH/MtFO/XqigDznhhhTfDlDoynCM6982mPCbSlxJ/aveZcKtTlwfAjwhyxDpg==",
+      "version": "14.2.30",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.30.tgz",
+      "integrity": "sha512-pVZMnFok5qEX4RT59mK2hEVtJX+XFfak+/rjHpyFh7juiT52r177bfFKhnlafm0UOSldhXjj32b+LZIOdswGTg==",
       "cpu": [
         "ia32"
       ],
-      "license": "MIT",
       "optional": true,
       "os": [
         "win32"
@@ -359,13 +350,12 @@
       }
     },
     "node_modules/@next/swc-win32-x64-msvc": {
-      "version": "14.2.26",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.26.tgz",
-      "integrity": "sha512-2rdB3T1/Gp7bv1eQTTm9d1Y1sv9UuJ2LAwOE0Pe2prHKe32UNscj7YS13fRB37d0GAiGNR+Y7ZcW8YjDI8Ns0w==",
+      "version": "14.2.30",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.30.tgz",
+      "integrity": "sha512-4KCo8hMZXMjpTzs3HOqOGYYwAXymXIy7PEPAXNEcEOyKqkjiDlECumrWziy+JEF0Oi4ILHGxzgQ3YiMGG2t/Lg==",
       "cpu": [
         "x64"
       ],
-      "license": "MIT",
       "optional": true,
       "os": [
         "win32"
@@ -620,11 +610,10 @@
       }
     },
     "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
-      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
+      "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
         "balanced-match": "^1.0.0"
       }
@@ -1224,11 +1213,10 @@
       "license": "MIT"
     },
     "node_modules/brace-expansion": {
-      "version": "1.1.11",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
-      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "version": "1.1.12",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
+      "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
         "balanced-match": "^1.0.0",
         "concat-map": "0.0.1"
@@ -2614,11 +2602,10 @@
       }
     },
     "node_modules/glob/node_modules/brace-expansion": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
-      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
+      "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==",
       "dev": true,
-      "license": "MIT",
       "dependencies": {
         "balanced-match": "^1.0.0"
       }
@@ -3613,12 +3600,11 @@
       "license": "MIT"
     },
     "node_modules/next": {
-      "version": "14.2.26",
-      "resolved": "https://registry.npmjs.org/next/-/next-14.2.26.tgz",
-      "integrity": "sha512-b81XSLihMwCfwiUVRRja3LphLo4uBBMZEzBBWMaISbKTwOmq3wPknIETy/8000tr7Gq4WmbuFYPS7jOYIf+ZJw==",
-      "license": "MIT",
+      "version": "14.2.30",
+      "resolved": "https://registry.npmjs.org/next/-/next-14.2.30.tgz",
+      "integrity": "sha512-+COdu6HQrHHFQ1S/8BBsCag61jZacmvbuL2avHvQFbWa2Ox7bE+d8FyNgxRLjXQ5wtPyQwEmk85js/AuaG2Sbg==",
       "dependencies": {
-        "@next/env": "14.2.26",
+        "@next/env": "14.2.30",
         "@swc/helpers": "0.5.5",
         "busboy": "1.6.0",
         "caniuse-lite": "^1.0.30001579",
@@ -3633,15 +3619,15 @@
         "node": ">=18.17.0"
       },
       "optionalDependencies": {
-        "@next/swc-darwin-arm64": "14.2.26",
-        "@next/swc-darwin-x64": "14.2.26",
-        "@next/swc-linux-arm64-gnu": "14.2.26",
-        "@next/swc-linux-arm64-musl": "14.2.26",
-        "@next/swc-linux-x64-gnu": "14.2.26",
-        "@next/swc-linux-x64-musl": "14.2.26",
-        "@next/swc-win32-arm64-msvc": "14.2.26",
-        "@next/swc-win32-ia32-msvc": "14.2.26",
-        "@next/swc-win32-x64-msvc": "14.2.26"
+        "@next/swc-darwin-arm64": "14.2.30",
+        "@next/swc-darwin-x64": "14.2.30",
+        "@next/swc-linux-arm64-gnu": "14.2.30",
+        "@next/swc-linux-arm64-musl": "14.2.30",
+        "@next/swc-linux-x64-gnu": "14.2.30",
+        "@next/swc-linux-x64-musl": "14.2.30",
+        "@next/swc-win32-arm64-msvc": "14.2.30",
+        "@next/swc-win32-ia32-msvc": "14.2.30",
+        "@next/swc-win32-x64-msvc": "14.2.30"
       },
       "peerDependencies": {
         "@opentelemetry/api": "^1.1.0",

examples/deployment/pipecat-cloud-daily-pstn-server/nextjs-webhook-server/pages/api/dial.js

@@ -65,6 +65,7 @@ export default async function handler(req, res) {
       From,
       callId,
       callDomain,
+      sipHeaders,
       dialout_settings,
       voicemail_detection,
       call_transfer
@@ -103,6 +104,7 @@ export default async function handler(req, res) {
         display_name: From,
         sip_mode: 'dial-in',
         num_endpoints: call_transfer !== null ? 2 : 1,
+        codecs: {"audio": ["OPUS"]},
       };
       daily_room_properties.sip = sip_config;
     }
@@ -116,6 +118,7 @@ export default async function handler(req, res) {
         dialout_settings,
         voicemail_detection,
         call_transfer,
+        sip_headers: sipHeaders,
       },
     };
 
@@ -172,4 +175,4 @@ export const config = {
       sizeLimit: '1mb',
     },
   },
-};
+};

examples/deployment/pipecat-cloud-example/bot.py

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+import asyncio
 import os
 
 import aiohttp
@@ -21,47 +22,26 @@ from pipecat.services.cartesia.tts import CartesiaTTSService
 from pipecat.services.openai.llm import OpenAILLMService
 from pipecat.transports.services.daily import DailyParams, DailyTransport
 
-# Check if we're in local development mode
-LOCAL_RUN = os.getenv("LOCAL_RUN")
-if LOCAL_RUN:
-    import asyncio
-    import webbrowser
-
-    try:
-        from local_runner import configure
-    except ImportError:
-        logger.error("Could not import local_runner module. Local development mode may not work.")
-
 # Load environment variables
 load_dotenv(override=True)
 
+# Check if we're in local development mode
+LOCAL_RUN = os.getenv("LOCAL_RUN")
 
-async def main(room_url: str, token: str):
+
+async def main(transport: DailyTransport):
     """Main pipeline setup and execution function.
 
     Args:
-        room_url: The Daily room URL
-        token: The Daily room token
+        transport: The DailyTransport object for the bot
     """
-    logger.debug("Starting bot in room: {}", room_url)
-
-    transport = DailyTransport(
-        room_url,
-        token,
-        "bot",
-        DailyParams(
-            audio_out_enabled=True,
-            transcription_enabled=True,
-            vad_enabled=True,
-            vad_analyzer=SileroVADAnalyzer(),
-        ),
-    )
+    logger.debug("Starting bot")
 
     tts = CartesiaTTSService(
-        api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22"
+        api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121"
     )
 
-    llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
+    llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
 
     messages = [
         {
@@ -87,10 +67,8 @@ async def main(room_url: str, token: str):
     task = PipelineTask(
         pipeline,
         params=PipelineParams(
-            allow_interruptions=True,
             enable_metrics=True,
             enable_usage_metrics=True,
-            report_only_initial_ttfb=True,
         ),
     )
 
@@ -126,10 +104,25 @@ async def bot(args: DailySessionArguments):
         body: The configuration object from the request body
         session_id: The session ID for logging
     """
+    from pipecat.audio.filters.krisp_filter import KrispFilter
+
     logger.info(f"Bot process initialized {args.room_url} {args.token}")
 
+    transport = DailyTransport(
+        args.room_url,
+        args.token,
+        "Pipecat Bot",
+        DailyParams(
+            audio_in_enabled=True,
+            audio_in_filter=None if LOCAL_RUN else KrispFilter(),
+            audio_out_enabled=True,
+            transcription_enabled=True,
+            vad_analyzer=SileroVADAnalyzer(),
+        ),
+    )
+
     try:
-        await main(args.room_url, args.token)
+        await main(transport)
         logger.info("Bot process completed")
     except Exception as e:
         logger.exception(f"Error in bot process: {str(e)}")
@@ -137,18 +130,27 @@ async def bot(args: DailySessionArguments):
 
 
 # Local development functions
-async def local_main():
+async def local_daily():
     """Function for local development testing."""
+    from local_runner import configure
+
     try:
         async with aiohttp.ClientSession() as session:
             (room_url, token) = await configure(session)
-            logger.warning("_")
-            logger.warning("_")
-            logger.warning(f"Talk to your voice agent here: {room_url}")
-            logger.warning("_")
-            logger.warning("_")
-            webbrowser.open(room_url)
-            await main(room_url, token)
+            transport = DailyTransport(
+                room_url,
+                token,
+                "Pipecat Bot",
+                DailyParams(
+                    audio_in_enabled=True,
+                    audio_out_enabled=True,
+                    transcription_enabled=True,
+                    vad_analyzer=SileroVADAnalyzer(),
+                ),
+            )
+
+            await main(transport)
+
     except Exception as e:
         logger.exception(f"Error in local development mode: {e}")
 
@@ -156,6 +158,6 @@ async def local_main():
 # Local development entry point
 if LOCAL_RUN and __name__ == "__main__":
     try:
-        asyncio.run(local_main())
+        asyncio.run(local_daily())
     except Exception as e:
         logger.exception(f"Failed to run in local mode: {e}")