translation use case working

.dockerignore

@@ -1,30 +0,0 @@
-# flyctl launch added from .gitignore
-**/.vscode
-**/env
-**/__pycache__
-**/*~
-**/venv
-#*#
-
-# Distribution / packaging
-**/.Python
-**/build
-**/develop-eggs
-**/dist
-**/downloads
-**/eggs
-**/.eggs
-**/lib
-**/lib64
-**/parts
-**/sdist
-**/var
-**/wheels
-**/share/python-wheels
-**/*.egg-info
-**/.installed.cfg
-**/*.egg
-**/MANIFEST
-**/.DS_Store
-**/.env
-fly.toml

.github/workflows/build.yaml

@@ -1,44 +0,0 @@
-name: build
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - "**"
-    paths-ignore:
-      - "docs/**"
-
-concurrency:
-  group: build-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    name: "Build and Install"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python
-        id: setup_python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-      - name: Setup virtual environment
-        run: |
-          python -m venv .venv
-      - name: Install basic Python dependencies
-        run: |
-          source .venv/bin/activate
-          python -m pip install --upgrade pip
-          pip install -r dev-requirements.txt
-      - name: Build project
-        run: |
-          source .venv/bin/activate
-          python -m build
-      - name: Install project and other Python dependencies
-        run: |
-          source .venv/bin/activate
-          pip install --editable .

.github/workflows/lint.yaml

@@ -29,11 +29,11 @@ jobs:
       - name: Setup virtual environment
         run: |
           python -m venv .venv
-      - name: Install development Python dependencies
+      - name: Install basic Python dependencies
         run: |
           source .venv/bin/activate
           python -m pip install --upgrade pip
-          pip install -r dev-requirements.txt
+          pip install -r requirements.txt
       - name: autopep8
         id: autopep8
         run: |

.github/workflows/publish.yaml

@@ -1,84 +0,0 @@
-name: publish
-
-on:
-  workflow_dispatch:
-    inputs:
-      gitref:
-        type: string
-        description: "what git ref to build"
-        required: true
-
-jobs:
-  build:
-    name: "Build and upload wheels"
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.inputs.gitref }}
-      - name: Set up Python
-        id: setup_python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-      - name: Setup virtual environment
-        run: |
-          python -m venv .venv
-      - name: Install basic Python dependencies
-        run: |
-          source .venv/bin/activate
-          python -m pip install --upgrade pip
-          pip install -r dev-requirements.txt
-      - name: Build project
-        run: |
-          source .venv/bin/activate
-          python -m build
-      - name: Upload wheels
-        uses: actions/upload-artifact@v4
-        with:
-          name: wheels
-          path: ./dist
-
-  publish-to-pypi:
-    name: "Publish to PyPI"
-    runs-on: ubuntu-latest
-    needs: [ build ]
-    environment:
-      name: pypi
-      url: https://pypi.org/p/pipecat-ai
-    permissions:
-      id-token: write
-    steps:
-      - name: Download wheels
-        uses: actions/download-artifact@v4
-        with:
-          name: wheels
-          path: ./dist
-      - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          verbose: true
-          print-hash: true
-
-  publish-to-test-pypi:
-    name: "Publish to Test PyPI"
-    runs-on: ubuntu-latest
-    needs: [ build ]
-    environment:
-      name: testpypi
-      url: https://pypi.org/p/pipecat-ai
-    permissions:
-      id-token: write
-    steps:
-      - name: Download wheels
-        uses: actions/download-artifact@v4
-        with:
-          name: wheels
-          path: ./dist
-      - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          verbose: true
-          print-hash: true
-          repository-url: https://test.pypi.org/legacy/

.github/workflows/publish_test.yaml

@@ -1,63 +0,0 @@
-name: publish-test
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-
-jobs:
-  build:
-    name: "Build and upload wheels"
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.inputs.gitref }}
-          fetch-tags: true
-          fetch-depth: 100
-      - name: Set up Python
-        id: setup_python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-      - name: Setup virtual environment
-        run: |
-          python -m venv .venv
-      - name: Install basic Python dependencies
-        run: |
-          source .venv/bin/activate
-          python -m pip install --upgrade pip
-          pip install -r dev-requirements.txt
-      - name: Build project
-        run: |
-          source .venv/bin/activate
-          python -m build
-      - name: Upload wheels
-        uses: actions/upload-artifact@v4
-        with:
-          name: wheels
-          path: ./dist
-
-  publish-to-test-pypi:
-    name: "Publish to Test PyPI"
-    runs-on: ubuntu-latest
-    needs: [ build ]
-    environment:
-      name: testpypi
-      url: https://pypi.org/p/pipecat-ai
-    permissions:
-      id-token: write
-    steps:
-      - name: Download wheels
-        uses: actions/download-artifact@v4
-        with:
-          name: wheels
-          path: ./dist
-      - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          verbose: true
-          print-hash: true
-          repository-url: https://test.pypi.org/legacy/

.github/workflows/tests.yaml

@@ -29,9 +29,9 @@ jobs:
       - name: Cache virtual environment
         uses: actions/cache@v3
         with:
-          # We are hashing requirements-dev.txt and requirements-extra.txt which
-          # contain all dependencies needed to run the tests and examples.
-          key: venv-${{ runner.os }}-${{ steps.setup_python.outputs.python-version}}-${{ hashFiles('linux-py3.10-requirements.txt') }}-${{ hashFiles('dev-requirements.txt') }}
+          # TODO: we are hashing requirements.txt but that doesn't contain all
+          # our dependencies pinned.
+          key: venv-${{ runner.os }}-${{ steps.setup_python.outputs.python-version}}-${{ hashFiles('requirements.txt') }}
           path: .venv
       - name: Install system packages
         run: sudo apt-get install -y portaudio19-dev
@@ -42,8 +42,17 @@ jobs:
         run: |
           source .venv/bin/activate
           python -m pip install --upgrade pip
-          pip install -r linux-py3.10-requirements.txt -r dev-requirements.txt
+          pip install -r requirements.txt
+      - name: Build project
+        run: |
+          source .venv/bin/activate
+          python -m build
+      - name: Install project and other Python dependencies
+        run: |
+          source .venv/bin/activate
+          pip install --editable .
       - name: Test with pytest
         run: |
           source .venv/bin/activate
+          pip install pytest
           pytest --doctest-modules --ignore-glob="*to_be_updated*" src tests

.gitignore

@@ -3,7 +3,6 @@ env/
 __pycache__/
 *~
 venv
-.venv
 #*#
 
 # Distribution / packaging
@@ -27,4 +26,3 @@ share/python-wheels/
 MANIFEST
 .DS_Store
 .env
-fly.toml

CHANGELOG.md

@@ -1,595 +0,0 @@
-# Changelog
-
-All notable changes to **pipecat** will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [0.0.32] - 2024-06-22
-
-### Added
-
-- Allow specifying a `DeepgramSTTService` url which allows using on-prem
-  Deepgram.
-
-- Added new `FastAPIWebsocketTransport`. This is a new websocket transport that
-  can be integrated with FastAPI websockets.
-
-- Added new `TwilioFrameSerializer`. This is a new serializer that knows how to
-  serialize and deserialize audio frames from Twilio.
-
-- Added Daily transport event: `on_dialout_answered`.  See
-  https://reference-python.daily.co/api_reference.html#daily.EventHandler
-
-- Added new `AzureSTTService`. This allows you to use Azure Speech-To-Text.
-
-### Performance
-
-- Convert `BaseOutputTransport` and `BaseOutputTransport` to fully use asyncio
-  and remove the use of threads.
-
-### Other
-
-- Added `twilio-chatbot`. This is an example that shows how to integrate Twilio
-  phone numbers with a Pipecat bot.
-
-- Updated `07f-interruptible-azure.py` to use `AzureLLMService`,
-  `AzureSTTService` and `AzureTTSService`.
-
-## [0.0.31] - 2024-06-13
-
-### Performance
-
-- Break long audio frames into 20ms chunks instead of 10ms.
-
-## [0.0.30] - 2024-06-13
-
-### Added
-
-- Added `report_only_initial_ttfb` to `PipelineParams`. This will make it so
-  only the initial TTFB metrics after the user stops talking are reported.
-
-- Added `OpenPipeLLMService`. This service will let you run OpenAI through
-  OpenPipe's SDK.
-
-- Allow specifying frame processors' name through a new `name` constructor
-  argument.
-
-- Added `DeepgramSTTService`. This service has an ongoing websocket
-  connection. To handle this, it subclasses `AIService` instead of
-  `STTService`. The output of this service will be pushed from the same task,
-  except system frames like `StartFrame`, `CancelFrame` or
-  `StartInterruptionFrame`.
-
-### Changed
-
-- `FrameSerializer.deserialize()` can now return `None` in case it is not
-  possible to desearialize the given data.
-
-- `daily_rest.DailyRoomProperties` now allows extra unknown parameters.
-
-### Fixed
-
-- Fixed an issue where `DailyRoomProperties.exp` always had the same old
-  timestamp unless set by the user.
-
-- Fixed a couple of issues with `WebsocketServerTransport`. It needed to use
-  `push_audio_frame()` and also VAD was not working properly.
-
-- Fixed an issue that would cause LLM aggregator to fail with small
-  `VADParams.stop_secs` values.
-
-- Fixed an issue where `BaseOutputTransport` would send longer audio frames
-  preventing interruptions.
-
-### Other
-
-- Added new `07h-interruptible-openpipe.py` example. This example shows how to
-  use OpenPipe to run OpenAI LLMs and get the logs stored in OpenPipe.
-
-- Added new `dialin-chatbot` example. This examples shows how to call the bot
-  using a phone number.
-
-## [0.0.29] - 2024-06-07
-
-### Added
-
-- Added a new `FunctionFilter`. This filter will let you filter frames based on
-  a given function, except system messages which should never be filtered.
-
-- Added `FrameProcessor.can_generate_metrics()` method to indicate if a
-  processor can generate metrics. In the future this might get an extra argument
-  to ask for a specific type of metric.
-
-- Added `BasePipeline`. All pipeline classes should be based on this class. All
-  subclasses should implement a `processors_with_metrics()` method that returns
-  a list of all `FrameProcessor`s in the pipeline that can generate metrics.
-
-- Added `enable_metrics` to `PipelineParams`.
-
-- Added `MetricsFrame`. The `MetricsFrame` will report different metrics in the
-  system. Right now, it can report TTFB (Time To First Byte) values for
-  different services, that is the time spent between the arrival of a `Frame` to
-  the processor/service until the first `DataFrame` is pushed downstream. If
-  metrics are enabled an intial `MetricsFrame` with all the services in the
-  pipeline will be sent.
-
-- Added TTFB metrics and debug logging for TTS services.
-
-### Changed
-
-- Moved `ParallelTask` to `pipecat.pipeline.parallel_task`.
-
-### Fixed
-
-- Fixed PlayHT TTS service to work properly async.
-
-## [0.0.28] - 2024-06-05
-
-### Fixed
-
-- Fixed an issue with `SileroVADAnalyzer` that would cause memory to keep
-  growing indefinitely.
-
-## [0.0.27] - 2024-06-05
-
-### Added
-
-- Added `DailyTransport.participants()` and `DailyTransport.participant_counts()`.
-
-## [0.0.26] - 2024-06-05
-
-### Added
-
-- Added `OpenAITTSService`.
-
-- Allow passing `output_format` and `model_id` to `CartesiaTTSService` to change
-  audio sample format and the model to use.
-
-- Added `DailyRESTHelper` which helps you create Daily rooms and tokens in an
-  easy way.
-
-- `PipelineTask` now has a `has_finished()` method to indicate if the task has
-  completed. If a task is never ran `has_finished()` will return False.
-
-- `PipelineRunner` now supports SIGTERM. If received, the runner will be
-  canceled.
-
-### Fixed
-
-- Fixed an issue where `BaseInputTransport` and `BaseOutputTransport` where
-  stopping push tasks before pushing `EndFrame` frames could cause the bots to
-  get stuck.
-
-- Fixed an error closing local audio transports.
-
-- Fixed an issue with Deepgram TTS that was introduced in the previous release.
-
-- Fixed `AnthropicLLMService` interruptions. If an interruption occurred, a
-  `user` message could be appended after the previous `user` message. Anthropic
-  does not allow that because it requires alternate `user` and `assistant`
-  messages.
-
-### Performance
-
-- The `BaseInputTransport` does not pull audio frames from sub-classes any
-  more. Instead, sub-classes now push audio frames into a queue in the base
-  class. Also, `DailyInputTransport` now pushes audio frames every 20ms instead
-  of 10ms.
-
-- Remove redundant camera input thread from `DailyInputTransport`. This should
-  improve performance a little bit when processing participant videos.
-
-- Load Cartesia voice on startup.
-
-## [0.0.25] - 2024-05-31
-
-### Added
-
-- Added WebsocketServerTransport. This will create a websocket server and will
-  read messages coming from a client. The messages are serialized/deserialized
-  with protobufs. See `examples/websocket-server` for a detailed example.
-
-- Added function calling (LLMService.register_function()). This will allow the
-  LLM to call functions you have registered when needed. For example, if you
-  register a function to get the weather in Los Angeles and ask the LLM about
-  the weather in Los Angeles, the LLM will call your function.
-  See https://platform.openai.com/docs/guides/function-calling
-
-- Added new `LangchainProcessor`.
-
-- Added Cartesia TTS support (https://cartesia.ai/)
-
-### Fixed
-
-- Fixed SileroVAD frame processor.
-
-- Fixed an issue where `camera_out_enabled` would cause the highg CPU usage if
-  no image was provided.
-
-### Performance
-
-- Removed unnecessary audio input tasks.
-
-## [0.0.24] - 2024-05-29
-
-### Added
-
-- Exposed `on_dialin_ready` for Daily transport SIP endpoint handling. This
-  notifies when the Daily room SIP endpoints are ready. This allows integrating
-  with third-party services like Twilio.
-
-- Exposed Daily transport `on_app_message` event.
-
-- Added Daily transport `on_call_state_updated` event.
-
-- Added Daily transport `start_recording()`, `stop_recording` and
-  `stop_dialout`.
-
-### Changed
-
-- Added `PipelineParams`. This replaces the `allow_interruptions` argument in
-  `PipelineTask` and will allow future parameters in the future.
-
-- Fixed Deepgram Aura TTS base_url and added ErrorFrame reporting.
-
-- GoogleLLMService `api_key` argument is now mandatory.
-
-### Fixed
-
-- Daily tranport `dialin-ready` doesn't not block anymore and it now handles
-  timeouts.
-
-- Fixed AzureLLMService.
-
-## [0.0.23] - 2024-05-23
-
-### Fixed
-
-- Fixed an issue handling Daily transport `dialin-ready` event.
-
-## [0.0.22] - 2024-05-23
-
-### Added
-
-- Added Daily transport `start_dialout()` to be able to make phone or SIP calls.
-  See https://reference-python.daily.co/api_reference.html#daily.CallClient.start_dialout
-
-- Added Daily transport support for dial-in use cases.
-
-- Added Daily transport events: `on_dialout_connected`, `on_dialout_stopped`,
-  `on_dialout_error` and `on_dialout_warning`.  See
-  https://reference-python.daily.co/api_reference.html#daily.EventHandler
-
-## [0.0.21] - 2024-05-22
-
-### Added
-
-- Added vision support to Anthropic service.
-
-- Added `WakeCheckFilter` which allows you to pass information downstream only
-  if you say a certain phrase/word.
-
-### Changed
-
-- `Filter` has been renamed to `FrameFilter` and it's now under
-  `processors/filters`.
-
-### Fixed
-
-- Fixed Anthropic service to use new frame types.
-
-- Fixed an issue in `LLMUserResponseAggregator` and `UserResponseAggregator`
-  that would cause frames after a brief pause to not be pushed to the LLM.
-
-- Clear the audio output buffer if we are interrupted.
-
-- Re-add exponential smoothing after volume calculation. This makes sure the
-  volume value being used doesn't fluctuate so much.
-
-## [0.0.20] - 2024-05-22
-
-### Added
-
-- In order to improve interruptions we now compute a loudness level using
-  [pyloudnorm](https://github.com/csteinmetz1/pyloudnorm). The audio coming
-  WebRTC transports (e.g. Daily) have an Automatic Gain Control (AGC) algorithm
-  applied to the signal, however we don't do that on our local PyAudio
-  signals. This means that currently incoming audio from PyAudio is kind of
-  broken. We will fix it in future releases.
-
-### Fixed
-
-- Fixed an issue where `StartInterruptionFrame` would cause
-  `LLMUserResponseAggregator` to push the accumulated text causing the LLM
-  respond in the wrong task. The `StartInterruptionFrame` should not trigger any
-  new LLM response because that would be spoken in a different task.
-
-- Fixed an issue where tasks and threads could be paused because the executor
-  didn't have more tasks available. This was causing issues when cancelling and
-  recreating tasks during interruptions.
-
-## [0.0.19] - 2024-05-20
-
-### Changed
-
-- `LLMUserResponseAggregator` and `LLMAssistantResponseAggregator` internal
-  messages are now exposed through the `messages` property.
-
-### Fixed
-
-- Fixed an issue where `LLMAssistantResponseAggregator` was not accumulating the
-  full response but short sentences instead. If there's an interruption we only
-  accumulate what the bot has spoken until now in a long response as well.
-
-## [0.0.18] - 2024-05-20
-
-### Fixed
-
-- Fixed an issue in `DailyOuputTransport` where transport messages were not
-  being sent.
-
-## [0.0.17] - 2024-05-19
-
-### Added
-
-- Added `google.generativeai` model support, including vision. This new `google`
-  service defaults to using `gemini-1.5-flash-latest`. Example in
-  `examples/foundational/12a-describe-video-gemini-flash.py`.
-
-- Added vision support to `openai` service. Example in
-  `examples/foundational/12a-describe-video-gemini-flash.py`.
-
-- Added initial interruptions support. The assistant contexts (or aggregators)
-  should now be placed after the output transport. This way, only the completed
-  spoken context is added to the assistant context.
-
-- Added `VADParams` so you can control voice confidence level and others.
-
-- `VADAnalyzer` now uses an exponential smoothed volume to improve speech
-  detection. This is useful when voice confidence is high (because there's
-  someone talking near you) but volume is low.
-
-### Fixed
-
-- Fixed an issue where TTSService was not pushing TextFrames downstream.
-
-- Fixed issues with Ctrl-C program termination.
-
-- Fixed an issue that was causing `StopTaskFrame` to actually not exit the
-  `PipelineTask`.
-
-## [0.0.16] - 2024-05-16
-
-### Fixed
-
-- `DailyTransport`: don't publish camera and audio tracks if not enabled.
-
-- Fixed an issue in `BaseInputTransport` that was causing frames pushed
-  downstream not pushed in the right order.
-
-## [0.0.15] - 2024-05-15
-
-### Fixed
-
-- Quick hot fix for receiving `DailyTransportMessage`.
-
-## [0.0.14] - 2024-05-15
-
-### Added
-
-- Added `DailyTransport` event `on_participant_left`.
-
-- Added support for receiving `DailyTransportMessage`.
-
-### Fixed
-
-- Images are now resized to the size of the output camera. This was causing
-  images not being displayed.
-
-- Fixed an issue in `DailyTransport` that would not allow the input processor to
-  shutdown if no participant ever joined the room.
-
-- Fixed base transports start and stop. In some situation processors would halt
-  or not shutdown properly.
-
-## [0.0.13] - 2024-05-14
-
-### Changed
-
-- `MoondreamService` argument `model_id` is now `model`.
-
-- `VADAnalyzer` arguments have been renamed for more clarity.
-
-### Fixed
-
-- Fixed an issue with `DailyInputTransport` and `DailyOutputTransport` that
-  could cause some threads to not start properly.
-
-- Fixed `STTService`. Add `max_silence_secs` and `max_buffer_secs` to handle
-  better what's being passed to the STT service. Also add exponential smoothing
-  to the RMS.
-
-- Fixed `WhisperSTTService`. Add `no_speech_prob` to avoid garbage output text.
-
-## [0.0.12] - 2024-05-14
-
-### Added
-
-- Added `DailyTranscriptionSettings` to be able to specify transcription
-  settings much easier (e.g. language).
-
-### Other
-
-- Updated `simple-chatbot` with Spanish.
-
-- Add missing dependencies in some of the examples.
-
-## [0.0.11] - 2024-05-13
-
-### Added
-
-- Allow stopping pipeline tasks with new `StopTaskFrame`.
-
-### Changed
-
-- TTS, STT and image generation service now use `AsyncGenerator`.
-
-### Fixed
-
-- `DailyTransport`: allow registering for participant transcriptions even if
-  input transport is not initialized yet.
-
-### Other
-
-- Updated `storytelling-chatbot`.
-
-## [0.0.10] - 2024-05-13
-
-### Added
-
-- Added Intel GPU support to `MoondreamService`.
-
-- Added support for sending transport messages (e.g. to communicate with an app
-  at the other end of the transport).
-
-- Added `FrameProcessor.push_error()` to easily send an `ErrorFrame` upstream.
-
-### Fixed
-
-- Fixed Azure services (TTS and image generation).
-
-### Other
-
-- Updated `simple-chatbot`, `moondream-chatbot` and `translation-chatbot`
-  examples.
-
-## [0.0.9] - 2024-05-12
-
-### Changed
-
-Many things have changed in this version. Many of the main ideas such as frames,
-processors, services and transports are still there but some things have changed
-a bit.
-
-- `Frame`s describe the basic units for processing. For example, text, image or
-  audio frames. Or control frames to indicate a user has started or stopped
-  speaking.
-
-- `FrameProcessor`s process frames (e.g. they convert a `TextFrame` to an
-  `ImageRawFrame`) and push new frames downstream or upstream to their linked
-  peers.
-
-- `FrameProcessor`s can be linked together. The easiest wait is to use the
-  `Pipeline` which is a container for processors. Linking processors allow
-  frames to travel upstream or downstream easily.
-
-- `Transport`s are a way to send or receive frames. There can be local
-  transports (e.g. local audio or native apps), network transports
-  (e.g. websocket) or service transports (e.g. https://daily.co).
-
-- `Pipeline`s are just a processor container for other processors.
-
-- A `PipelineTask` know how to run a pipeline.
-
-- A `PipelineRunner` can run one or more tasks and it is also used, for example,
-  to capture Ctrl-C from the user.
-
-## [0.0.8] - 2024-04-11
-
-### Added
-
-- Added `FireworksLLMService`.
-
-- Added `InterimTranscriptionFrame` and enable interim results in
-  `DailyTransport` transcriptions.
-
-### Changed
-
-- `FalImageGenService` now uses new `fal_client` package.
-
-### Fixed
-
-- `FalImageGenService`: use `asyncio.to_thread` to not block main loop when
-  generating images.
-
-- Allow `TranscriptionFrame` after an end frame (transcriptions can be delayed
-  and received after `UserStoppedSpeakingFrame`).
-
-## [0.0.7] - 2024-04-10
-
-### Added
-
-- Add `use_cpu` argument to `MoondreamService`.
-
-## [0.0.6] - 2024-04-10
-
-### Added
-
-- Added `FalImageGenService.InputParams`.
-
-- Added `URLImageFrame` and `UserImageFrame`.
-
-- Added `UserImageRequestFrame` and allow requesting an image from a participant.
-
-- Added base `VisionService` and `MoondreamService`
-
-### Changed
-
-- Don't pass `image_size` to `ImageGenService`, images should have their own size.
-
-- `ImageFrame` now receives a tuple`(width,height)` to specify the size.
-
-- `on_first_other_participant_joined` now gets a participant argument.
-
-### Fixed
-
-- Check if camera, speaker and microphone are enabled before writing to them.
-
-### Performance
-
-- `DailyTransport` only subscribe to desired participant video track.
-
-## [0.0.5] - 2024-04-06
-
-### Changed
-
-- Use `camera_bitrate` and `camera_framerate`.
-
-- Increase `camera_framerate` to 30 by default.
-
-### Fixed
-
-- Fixed `LocalTransport.read_audio_frames`.
-
-## [0.0.4] - 2024-04-04
-
-### Added
-
-- Added project optional dependencies `[silero,openai,...]`.
-
-### Changed
-
-- Moved thransports to its own directory.
-
-- Use `OPENAI_API_KEY` instead of `OPENAI_CHATGPT_API_KEY`.
-
-### Fixed
-
-- Don't write to microphone/speaker if not enabled.
-
-### Other
-
-- Added live translation example.
-
-- Fix foundational examples.
-
-## [0.0.3] - 2024-03-13
-
-### Other
-
-- Added `storybot` and `chatbot` examples.
-
-## [0.0.2] - 2024-03-12
-
-Initial public release.

CHANGELOG.md.template

@@ -1,62 +0,0 @@
-# Changelog
-
-All notable changes to the **<project name>** SDK will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-Please make sure to add your changes to the appropriate categories:
-
-## [Unreleased]
-
-### Added
-
-<!-- for new functionality -->
-
-- n/a
-
-### Changed
-
-<!-- for changed functionality -->
-
-- n/a
-
-### Deprecated
-
-<!-- for soon-to-be removed functionality -->
-
-- n/a
-
-### Removed
-
-<!-- for removed functionality -->
-
-- n/a
-
-### Fixed
-
-<!-- for fixed bugs -->
-
-- n/a
-
-### Performance
-
-<!-- for performance-relevant changes -->
-
-- n/a
-
-### Security
-
-<!-- for security-relevant changes -->
-
-- n/a
-
-### Other
-
-<!-- for everything else -->
-
-- n/a
-
-## [0.1.0] - YYYY-MM-DD
-
-Initial release.

README.md

@@ -1,172 +1,115 @@
-<div align="center">
- <img alt="pipecat" width="300px" height="auto" src="https://raw.githubusercontent.com/pipecat-ai/pipecat/main/pipecat.png">
-</div>
+# dailyai — an open source framework for real-time, multi-modal, conversational AI applications
 
-# Pipecat
+Build things like this:
 
-[![PyPI](https://img.shields.io/pypi/v/pipecat-ai)](https://pypi.org/project/pipecat-ai) [![Discord](https://img.shields.io/discord/1239284677165056021
-)](https://discord.gg/pipecat)
+[![AI-powered voice patient intake for healthcare](https://img.youtube.com/vi/lDevgsp9vn0/0.jpg)](https://www.youtube.com/watch?v=lDevgsp9vn0)
 
-`pipecat` is a framework for building voice (and multimodal) conversational agents. Things like personal coaches, meeting assistants, [story-telling toys for kids](https://storytelling-chatbot.fly.dev/), customer support bots, [intake flows](https://www.youtube.com/watch?v=lDevgsp9vn0), and snarky social companions.
 
-Take a look at some example apps:
 
-<p float="left">
-    <a href="https://github.com/pipecat-ai/pipecat/tree/main/examples/simple-chatbot"><img src="https://raw.githubusercontent.com/pipecat-ai/pipecat/main/examples/simple-chatbot/image.png" width="280" /></a>&nbsp;
-    <a href="https://github.com/pipecat-ai/pipecat/tree/main/examples/storytelling-chatbot"><img src="https://raw.githubusercontent.com/pipecat-ai/pipecat/main/examples/storytelling-chatbot/image.png" width="280" /></a>
-    <br/>
-    <a href="https://github.com/pipecat-ai/pipecat/tree/main/examples/translation-chatbot"><img src="https://raw.githubusercontent.com/pipecat-ai/pipecat/main/examples/translation-chatbot/image.png" width="280" /></a>&nbsp;
-    <a href="https://github.com/pipecat-ai/pipecat/tree/main/examples/moondream-chatbot"><img src="https://raw.githubusercontent.com/pipecat-ai/pipecat/main/examples/moondream-chatbot/image.png" width="280" /></a>
-</p>
 
-## Getting started with voice agents
+**`dailyai` started as a toolkit for implementing generative AI voice bots.** Things like personal coaches, meeting assistants, story-telling toys for kids, customer support bots, and snarky social companions.
 
-You can get started with Pipecat running on your local machine, then move your agent processes to the cloud when you’re ready. You can also add a 📞 telephone number, 🖼️ image output, 📺 video input, use different LLMs, and more.
 
-```shell
+In 2023 a *lot* of us got excited about the possibility of having open-ended conversations with LLMs. It became clear pretty quickly that we were all solving the same [low-level problems](https://www.daily.co/blog/how-to-talk-to-an-llm-with-your-voice/):
+- low-latency, reliable audio transport
+- echo cancellation
+- phrase endpointing (knowing when the bot should respond to human speech)
+- interruptibility
+- writing clean code to stream data through "pipelines" of speech-to-text, LLM inference, and text-to-speech models
+
+As our applications expanded to include additional things like image generation, function calling, and vision models, we started to think about what a complete framework for these kinds of apps could look like.
+
+Today, `dailyai` is:
+
+1. a set of code building blocks for interacting with generative AI services and creating low-latency, interruptible data pipelines that use multiple services
+2. transport services that moves audio, video, and events across the Internet
+3. implementations of specific generative AI services
+
+Currently implemented services:
+- Speech-to-text
+  - Deepgram
+  - Whisper
+- LLMs
+  - Azure
+  - OpenAI
+- Image generation
+  - Azure
+  - Fal
+  - OpenAI
+- Text-to-speech
+  - Azure
+  - Deepgram
+  - ElevenLabs
+- Transport
+  - Daily
+  - Local (in progress, intended as a quick start example service)
+
+If you'd like to [implement a service]((https://github.com/daily-co/daily-ai-sdk/tree/main/src/dailyai/services)), we welcome PRs! Our goal is to support lots of services in all of the above categories, plus new categories (like real-time video) as they emerge.
+
+## Getting started
+
+Today, the easiest way to get started with `dailyai` is to use [Daily](https://www.daily.co/) as your transport service. This toolkit started life as an internal SDK at Daily and millions of minutes of AI conversation have been served using it and its earlier prototype incarnations. (The [transport base class](https://github.com/daily-co/daily-ai-sdk/blob/main/src/dailyai/services/base_transport_service.py) is easy to extend, though, so feel free to submit PRs if you'd like to implement another transport service.)
+
+```
 # install the module
-pip install pipecat-ai
+pip install dailyai
 
 # set up an .env file with API keys
 cp dot-env.template .env
 ```
 
-By default, in order to minimize dependencies, only the basic framework functionality is available. Some third-party AI services require additional dependencies that you can install with:
-
-```shell
-pip install "pipecat-ai[option,...]"
-```
-
-Your project may or may not need these, so they're made available as optional requirements. Here is a list:
-
-- **AI services**: `anthropic`, `azure`, `deepgram`, `google`, `fal`, `moondream`, `openai`, `openpipe`, `playht`, `silero`, `whisper`
-- **Transports**: `local`, `websocket`, `daily`
-
 ## Code examples
 
-- [foundational](https://github.com/pipecat-ai/pipecat/tree/main/examples/foundational) — small snippets that build on each other, introducing one or two concepts at a time
-- [example apps](https://github.com/pipecat-ai/pipecat/tree/main/examples/) — complete applications that you can use as starting points for development
+There are two directories of examples:
 
-## A simple voice agent running locally
+- [foundational](https://github.com/daily-co/daily-ai-sdk/tree/main/examples/foundational) — demos that build on each other, introducing one or two concepts at a time
+- [starter apps](https://github.com/daily-co/daily-ai-sdk/tree/main/examples/starter-apps) — complete applications that you can use as starting points for development
 
-Here is a very basic Pipecat bot that greets a user when they join a real-time session. We'll use [Daily](https://daily.co) for real-time media transport, and [ElevenLabs](https://elevenlabs.io/) for text-to-speech.
+To run the example below you need to sign up for a [free Daily account](https://dashboard.daily.co/u/signup) and create a Daily room (so you can hear the LLM talking). After that, join the room's URL directly from a browser tab and run:
 
-```python
-#app.py
-
-import asyncio
-import aiohttp
-
-from pipecat.frames.frames import EndFrame, TextFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.task import PipelineTask
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-
-async def main():
-  async with aiohttp.ClientSession() as session:
-    # Use Daily as a real-time media transport (WebRTC)
-    transport = DailyTransport(
-      room_url=...,
-      token=...,
-      "Bot Name",
-      DailyParams(audio_out_enabled=True))
-
-    # Use Eleven Labs for Text-to-Speech
-    tts = ElevenLabsTTSService(
-      aiohttp_session=session,
-      api_key=...,
-      voice_id=...,
-      )
-
-    # Simple pipeline that will process text to speech and output the result
-    pipeline = Pipeline([tts, transport.output()])
-
-    # Create Pipecat processor that can run one or more pipelines tasks
-    runner = PipelineRunner()
-
-    # Assign the task callable to run the pipeline
-    task = PipelineTask(pipeline)
-
-    # Register an event handler to play audio when a
-    # participant joins the transport WebRTC session
-    @transport.event_handler("on_participant_joined")
-    async def on_new_participant_joined(transport, participant):
-      participant_name = participant["info"]["userName"] or ''
-      # Queue a TextFrame that will get spoken by the TTS service (Eleven Labs)
-      await task.queue_frames([TextFrame(f"Hello there, {participant_name}!"), EndFrame()])
-
-    # Run the pipeline task
-    await runner.run(task)
-
-if __name__ == "__main__":
-  asyncio.run(main())
 ```
-
-Run it with:
-
-```shell
-python app.py
+python examples/foundational/02-llm-say-one-thing.py
 ```
 
-Daily provides a prebuilt WebRTC user interface. Whilst the app is running, you can visit at `https://<yourdomain>.daily.co/<room_url>` and listen to the bot say hello!
-
-
-## WebRTC for production use
-
-WebSockets are fine for server-to-server communication or for initial development. But for production use, you’ll need client-server audio to use a protocol designed for real-time media transport. (For an explanation of the difference between WebSockets and WebRTC, see [this post.](https://www.daily.co/blog/how-to-talk-to-an-llm-with-your-voice/#webrtc))
-
-One way to get up and running quickly with WebRTC is to sign up for a Daily developer account. Daily gives you SDKs and global infrastructure for audio (and video) routing. Every account gets 10,000 audio/video/transcription minutes free each month.
-
-Sign up [here](https://dashboard.daily.co/u/signup) and [create a room](https://docs.daily.co/reference/rest-api/rooms) in the developer Dashboard.
-
-## What is VAD?
-
-Voice Activity Detection &mdash; very important for knowing when a user has finished speaking to your bot. If you are not using press-to-talk, and want Pipecat to detect when the user has finished talking, VAD is an essential component for a natural feeling conversation.
-
-Pipecast makes use of WebRTC VAD by default when using a WebRTC transport layer. Optionally, you can use Silero VAD for improved accuracy at the cost of higher CPU usage.
-
-```shell
-pip install pipecat-ai[silero]
-```
-
-The first time your run your bot with Silero, startup may take a while whilst it downloads and caches the model in the background. You can check the progress of this in the console.
-
-
 ## Hacking on the framework itself
 
 _Note that you may need to set up a virtual environment before following the instructions below. For instance, you might need to run the following from the root of the repo:_
 
-```shell
-python3 -m venv venv
-source venv/bin/activate
+```
+python3 -m venv env
+source env/bin/activate
 ```
 
 From the root of this repo, run the following:
 
-```shell
-pip install -r dev-requirements.txt -r {env}-requirements.txt
+```
+pip install -r requirements.txt
 python -m build
 ```
 
 This builds the package. To use the package locally (eg to run sample files), run
 
-```shell
+```
 pip install --editable .
 ```
 
 If you want to use this package from another directory, you can run:
 
-```shell
+```
 pip install path_to_this_repo
 ```
 
 ### Running tests
 
-From the root directory, run:
+To run tests you need to install `pytest`:
 
-```shell
+```
+pip install pytest
+```
+
+Then, from the root directory, run:
+
+```
 pytest --doctest-modules --ignore-glob="*to_be_updated*" src tests
 ```
 
@@ -213,9 +156,3 @@ Install the
     "--max-line-length=100"
 ],
 ```
-
-## Getting help
-
-➡️ [Join our Discord](https://discord.gg/pipecat)
-
-➡️ [Reach us on X](https://x.com/pipecat_ai)

dev-requirements.txt

@@ -1,8 +0,0 @@
-autopep8~=2.1.0
-build~=1.2.1
-grpcio-tools~=1.62.2
-pip-tools~=7.4.1
-pyright~=1.1.367
-pytest~=8.2.0
-setuptools~=69.5.1
-setuptools_scm~=8.1.0

docs/README.md

@@ -1,10 +1,17 @@
-# Pipecat Docs
+# Daily AI SDK Docs
 
 ## [Architecture Overview](architecture.md)
 
-Learn about the thinking behind the framework's design.
+Learn about the thinking behind the SDK's design.
 
 ## [A Frame's Progress](frame-progress.md)
 
 See how a Frame is processed through a Transport, a Pipeline, and a series of Frame Processors.
 
+## [Example Code](examples/)
+
+The repo includes several example apps in the `examples` directory. The docs explain how they work.
+
+## [API Reference](api/)
+
+Complete documentation of the available classes and methods in the SDK.

docs/architecture.md

@@ -1,4 +1,4 @@
-# Pipecat architecture guide
+# Daily AI SDK Architecture Guide
 
 ## Frames
 
@@ -10,8 +10,8 @@ Frame processors operate on frames. Every frame processor implements a `process_
 
 ## Pipelines
 
-Pipelines are lists of frame processors linked together. Frame processors can push frames upstream or downstream to their peers. A very simple pipeline might chain an LLM frame processor to a text-to-speech frame processor, with a transport as an output.
+Pipelines are lists of frame processors that read from a source queue and send the processed frames to a sink queue. A very simple pipeline might chain an LLM frame processor to a text-to-speech frame processor, with a transport's send queue as its sync. Placing LLM message frames on the pipeline's source queue will cause the LLM's response to be spoken. See example #2 for an implementation of this.
 
 ## Transports
 
-Transports provide input and output frame processors to receive or send frames respectively. For example, the `DailyTransport` does this with a WebRTC session joined to a Daily.co room.
+Transports provide a receive queue, which is input from "the outside world", and a sink queue, which is data that will be sent "to the outside world". The `LocalTransportService` does this with the local camera, mic, display and speaker. The `DailyTransportService` does this with a WebRTC session joined to a Daily.co room.

docs/examples/01-say-one-thing.md

@@ -0,0 +1,119 @@
+# 01: Say One Thing
+
+_video here - youtube?_
+
+This example uses a text-to-speech (TTS) service to say one predefined sentence. But first, a quick overview of the general structure of these examples.
+
+## Running the demos
+
+All of the demos have something like this at the bottom of the file:
+
+```python
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))
+```
+
+### `configure()`
+
+The `configure()` function comes from `examples/foundational/support/runner.py`, and it allows you to configure the examples from the command line directly, or using environment variables:
+
+```bash
+python 01-say-one-thing.py -u https://YOUR_DOMAIN.daily.co/YOUR_ROOM -k YOUR_API_KEY
+# or
+DAILY_ROOM_URL=https://YOUR_DOMAIN.daily.co/YOUR_ROOM DAILY_API_KEY=YOUR_API_KEY python 01-say-one-thing.py
+# or set DAILY_ROOM_URL and DAILY_API_KEY in a .env file
+python 01-say-one-thing.py
+```
+
+You'll need a Daily account to run these demos. You can sign up for free at [daily.co](https://daily.co). Once you've signed up you can create a room from the [Dashboard](https://dashboard.daily.co/rooms), and grab [your API key](https://dashboard.daily.co/developers) while you're there.
+
+Some functionality (such as transcription) requires the bot to have owner privileges in the room. `runner.py` uses the Daily REST API to create a meeting token with owner privileges. You can learn more about meeting tokens in the [Daily docs](https://docs.daily.co/reference/rest-api/meeting-tokens).
+
+### `asyncio.run()`
+
+The AI SDK makes heavy use of Python's `asyncio` module. [This is a reasonable intro to the topic](https://builtin.com/data-science/asyncio) if you haven't worked with `asyncio` and coroutines before.
+
+You can learn a bit more about the specifics of how the Daily AI SDK uses coroutines in the [Architecture Guide](../architecture.md).
+
+## The `main()` function
+
+All of the examples have a `main()` function with a similar structure:
+
+- Configure the transport
+- Configure the AI service(s) used in the demo
+- Configure any event listeners
+- Define a processing pipeline
+- Run the example's coroutine(s)
+
+### Configuring the transport
+
+The first section of the `main()` function configures the transport object:
+
+```python
+meeting_duration_minutes = 5
+transport = DailyTransportService(
+    room_url,
+    None,
+    "Say One Thing",
+    meeting_duration_minutes,
+)
+transport.mic_enabled = True
+```
+
+The [Architecture Guide](../architecture.md) explains the transport object in more detail. In this case, we're configuring a Daily transport object and enabling the virtual microphone, so our bot can play audio.
+
+### Configuring the services
+
+As described in the [Architecture Guide](../architecture.md), 'a 'Service' is a class that processes 'Frames' as part of a 'Pipeline'. In this demo app, we'll only need one service: a text-to-speech generator. We can create an instance of the `ElevenLabsTTSService` class with this line of code:
+
+```python
+tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID"))
+```
+
+You'll need to make sure and set those environment variables somewhere. The easiest way to do that is to copy the `example.env` file in the repo and rename it to `.env`, and then add your credentials to that file. `runner.py` loads the `python-dotenv` module and initializes it, making the values in that file available in the environment.
+
+### Configuring event listeners
+
+This part isn't strictly necessary for an app like this. You could include the contents of the `on_participant_joined` function directly in the body of the `main()` function, and it would run as soon as you started the script from the command line.
+
+Instead, we can use an event handler to wait to run that code until someone else joins the meeting. We'll define a function called `greet_user()`, and use the `@transport.event_handler("on_participant_joined")` decorator to tell the SDK that we want to run that function whenever a user joins the room.
+
+```python
+@transport.event_handler("on_participant_joined")
+async def greet_user(transport, participant):
+    if participant["info"]["isLocal"]:
+        return
+
+    await tts.say(
+        "Hello there, " + participant["info"]["userName"] + "!",
+        transport.send_queue,
+    )
+
+    # wait for the output queue to be empty, then leave the meeting
+    await transport.stop_when_done()
+```
+
+### Defining a processing pipeline
+
+In this example, we don't actually have much of a processing pipeline! In fact, we're doing the whole thing inside the `greet_user()` function already.
+
+Pipelines usually look like a bunch of nested calls to the `run()` or `run_to_queue()` function from different Services. In this example, we're using the `say()` function from the TTS service. This is effectively a convenience wrapper around the `run_to_queue()` function, which we'll discuss more later. It's important to `await` this function to ensure that the speech frames are queued for playback before the next line of code, because of the `stop_when_done()` function being called immediately afterward.
+
+The output of the `say()` function goes to the transport's `send_queue`. This queue is the all-important connection between the world of the Services pipeline that's generating frames asynchronously and the ordered playback of audio and visual media in the WebRTC call.
+
+### Running the coroutines
+
+In this example, we don't actually have any separate processing pipelines—everything happens as a result of an event from the transport. So we only need to run the transport's coroutine, and await its completion:
+
+```python
+await transport.run()
+```
+
+In future examples, we'll run more processes in parallel. For now, this script can run until the transport exits—which will happen based on calling `stop_when_done()` in the `greet_user()` function.
+
+## Next Steps
+
+Next, we'll start connecting multiple AI services together by building a service pipeline.
+
+## [02 - LLM Say One Thing »](02-llm-say-one-thing.md)

docs/examples/README.md

@@ -0,0 +1,5 @@
+# Daily AI SDK Examples
+
+The docs in this folder pair with the example apps located in `examples/foundational`. They are designed to serve as a quick references for building different kinds of AI apps. But the examples also build on one another, so it can be really helpful to walk through them in order.
+
+To start, you can learn about the overall structure of the examples in [01 - Say One Thing](01-say-one-thing.md).

dot-env.template

@@ -1,38 +1,5 @@
-# Anthropic
-ANTHROPIC_API_KEY=...
-
-# Azure
-AZURE_SPEECH_REGION=...
-AZURE_SPEECH_API_KEY=...
-
-AZURE_CHATGPT_API_KEY=...
-AZURE_CHATGPT_ENDPOINT=https://...
-AZURE_CHATGPT_MODEL=...
-
-AZURE_DALLE_API_KEY=...
-AZURE_DALLE_ENDPOINT=https://...
-AZURE_DALLE_MODEL=...
-
-# Daily
-DAILY_API_KEY=...
-DAILY_SAMPLE_ROOM_URL=https://...
-
-# ElevenLabs
+OPENAI_API_KEY=...
 ELEVENLABS_API_KEY=...
 ELEVENLABS_VOICE_ID=...
-
-# Fal
-FAL_KEY=...
-
-# Fireworks
-FIREWORKS_API_KEY=...
-
-# PlayHT
-PLAY_HT_USER_ID=...
-PLAY_HT_API_KEY=...
-
-# OpenAI
-OPENAI_API_KEY=...
-
-#OpenPipe
-OPENPIPE_API_KEY=...
+DAILY_API_KEY=...
+DAILY_SAMPLE_ROOM_URL=https://...

examples/README.md

@@ -1,86 +0,0 @@
-
-
-# Pipecat — Examples
-
-## Foundational snippets
-Small snippets that build on each other, introducing one or two concepts at a time.
-
-➡️ [Take a look](https://github.com/pipecat-ai/pipecat/tree/main/examples/foundational)
-
-## Chatbot examples
-Collection of self-contained real-time voice and video AI demo applications built with Pipecat.
-
-### Quickstart
-
-Each project has its own set of dependencies and configuration variables. They intentionally avoids shared code across projects — you can grab whichever demo folder you want to work with as a starting point.
-
-We recommend you start with a virtual environment:
-
-```shell
-cd pipecat-ai/examples/simple-chatbot
-
-python -m venv venv
-
-source venv/bin/activate
-
-pip install -r requirements.txt
-```
-
-Next, follow the steps in the README for each demo.
-
-ℹ️ Make sure you `pip install -r requirements.txt` for each demo project, so you can be sure to have the necessary service dependencies that extend the functionality of Pipecat. You can read more about the framework architecture [here](https://github.com/pipecat-ai/pipecat/tree/main/docs).
-
-## Projects:
-
-| Project                                      | Description                                                                                                                                | Services                                                          |
-|----------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------|
-| [Simple Chatbot](simple-chatbot)             | Basic voice-driven conversational bot. A good starting point for learning the flow of the framework.                                       | Deepgram, ElevenLabs, OpenAI, Daily, Daily Prebuilt UI            |
-| [Storytelling Chatbot](storytelling-chatbot) | Stitches together multiple third-party services to create a collaborative storytime experience.                                            | Deepgram, ElevenLabs, OpenAI, Fal, Daily, Custom UI               |
-| [Translation Chatbot](translation-chatbot)   | Listens for user speech, then translates that speech to Spanish and speaks the translation back. Demonstrates multi-participant use-cases. | Deepgram, Azure, OpenAI, Daily, Daily Prebuilt UI                 |
-| [Moondream Chatbot](moondream-chatbot)       | Demonstrates how to add vision capabilities to GPT4. **Note: works best with a GPU**                                                       | Deepgram, ElevenLabs, OpenAI, Moondream, Daily, Daily Prebuilt UI |
-| [Patient intake](patient-intake)             | A chatbot that can call functions in response to user input.                                                                               | Deepgram, ElevenLabs, OpenAI, Daily, Daily Prebuilt UI            |
-| [Dialin Chatbot](dialin-chatbot)             | A chatbot that connects to an incoming phone call from Daily or Twilio.                                                                    | Deepgram, ElevenLabs, OpenAI, Daily, Twilio                       |
-| [Twilio Chatbot](twilio-chatbot)             | A chatbot that connects to an incoming phone call from Twilio.                                                                             | Deepgram, ElevenLabs, OpenAI, Daily, Twilio                       |
-
-> [!IMPORTANT]
-> These example projects use Daily as a WebRTC transport and can be joined using their hosted Prebuilt UI.
-> It provides a quick way to join a real-time session with your bot and test your ideas without building any frontend code. If you'd like to see an example of a custom UI, try Storybot.
-
-
-## FAQ
-
-### Deployment
-
-For each of these demos we've included a `Dockerfile`. Out of the box, this should provide everything needed to get the respective demo running on a VM:
-
-```shell
-docker build username/app:tag .
-
-docker run -p 7860:7860 --env-file ./.env username/app:tag
-
-docker push ...
-```
-
-### SSL
-
-If you're working with a custom UI (such as with the Storytelling Chatbot), it's important to ensure your deployment platform supports HTTPS, as accessing user devices such as mics and webcams requires SSL.
-
-If you try to run a custom UI without SSL, you may see an error in the console telling you that `navigator` is undefined, or no devices are available.
-
-### Are these examples production ready?
-
-Yes, kind of.
-
-These demos attempt to keep things simple and are unopinionated regarding environment or scalability.
-
-We're using FastAPI to spawn a subprocess for the bots / agents — useful for small tests, but not so great for production grade apps with many concurrent users. You can see how this works in each project's `start` endpoint in `server.py`.
-
-Creating virtualized worker pools and on-demand instances is out of scope for these examples, but we hope to add some examples to this repo soon!
-
-For projects that have CUDA as a requirement, such as Moondream Chatbot, be sure to deploy to a GPU-powered platform (such as [fly.io](https://fly.io) or [Runpod](https://runpod.io).)
-
-## Getting help
-
-➡️ [Join our Discord](https://discord.gg/pipecat)
-
-➡️ [Reach us on Twitter](https://x.com/pipecat_ai)

examples/dialin-chatbot/.dockerignore

@@ -1,3 +0,0 @@
-**/.DS_Store
-.env
-.env.*

examples/dialin-chatbot/.gitignore

@@ -1,165 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-runpod.toml
-
-# custom script to recursively upgrade items in requirements.py
-upgrade_requirements.py
-.DS_Store

examples/dialin-chatbot/Dockerfile

@@ -1,40 +0,0 @@
-FROM python:3.11-bullseye
-
-ARG DEBIAN_FRONTEND=noninteractive
-ARG USE_PERSISTENT_DATA
-ENV PYTHONUNBUFFERED=1
-# Expose FastAPI port
-ENV FAST_API_PORT=7860
-EXPOSE 7860
-
-# Install system dependencies
-RUN apt-get update && apt-get install --no-install-recommends -y \
-    build-essential \
-    git \
-    ffmpeg \
-    google-perftools \
-    ca-certificates curl gnupg \
-    && apt-get clean && rm -rf /var/lib/apt/lists/*
-
-# Set up a new user named "user" with user ID 1000
-RUN useradd -m -u 1000 user
-
-# Set home to the user's home directory
-ENV HOME=/home/user \
-    PATH=/home/user/.local/bin:$PATH \
-    PYTHONPATH=$HOME/app \
-    PYTHONUNBUFFERED=1
-
-# Switch to the "user" user
-USER user
-
-# Set the working directory to the user's home directory
-WORKDIR $HOME/app
-
-# Install Python dependencies
-COPY *.py .
-COPY ./requirements.txt requirements.txt
-RUN pip3 install --no-cache-dir --upgrade -r requirements.txt
-
-# Start the FastAPI server
-CMD python3 bot_runner.py --host "0.0.0.0" --port ${FAST_API_PORT}

examples/dialin-chatbot/README.md

@@ -1,85 +0,0 @@
-<div align="center">
- <img alt="pipecat" width="300px" height="auto" src="image.png">
-</div>
-
-# Dialin example
-
-Example project that demonstrates how to add phone number dialin to your Pipecat bots. We include examples for both Daily (`bot_daily.py`) and Twilio (`bot_twilio.py`), depending on who you want to use as a phone vendor.
-
-- 🔁 Transport: Daily WebRTC
-- 💬 Speech-to-Text: Deepgram via Daily transport
-- 🤖 LLM: GPT4-o / OpenAI
-- 🔉 Text-to-Speech: ElevenLabs
-
-#### Should I use Daily or Twilio as a vendor?
-
-If you're starting from scratch, using Daily to provision phone numbers alongside Daily as a transport offers some convenience (such as automatic call forwarding.)
-
-If you already have Twilio numbers and workflows that you want to connect to your Pipecat bots, there is some additional configuration required (you'll need to create a `on_dialin_ready` and use the Twilio client to trigger the forward.)
-
-You can read more about this, as well as see respective walkthroughs in our docs.
-
-## Setup
-
-```shell
-# Install the requirements
-pip install -r requirements.txt
-
-# Setup your env
-mv env.example .env
-```
-
-## Using Daily numbers
-
-Run `bot_runner.py` to handle incoming HTTP requests:
-
-`python bot_runner.py --host localhost`
-
-Then target the following URL:
-
-`POST /daily_start_bot`
-
-For more configuration options, please consult Daily's API documentation.
-
-
-## Using Twilio numbers
-
-As above, but target the following URL:
-
-`POST /twilio_start_bot`
-
-For more configuration options, please consult Twilio's API documentation.
-
-## Deployment example
-
-A Dockerfile is included in this demo for convenience. Here is an example of how to build and deploy your bot to [fly.io](https://fly.io).
-
-*Please note: This demo spawns agents as subprocesses for convenience / demonstration purposes. You would likely not want to do this in production as it would limit concurrency to available system resources. For more information on how to deploy your bots using VMs, refer to the Pipecat documentation.*
-
-### Build the docker image
-
-`docker build -t tag:project .`
-
-### Launch the fly project
-
-`mv fly.example.toml fly.toml`
-
-`fly launch` (using the included fly.toml)
-
-### Setup your secrets on Fly
-
-Set the necessary secrets (found in `env.example`)
-
-`fly secrets set DAILY_API_KEY=... OPENAI_API_KEY=... ELEVENLABS_API_KEY=... ELEVENLABS_VOICE_ID=...`
-
-If you're using Twilio as a number vendor:
-
-`fly secrets set TWILIO_ACCOUNT_SID=... TWILIO_AUTH_TOKEN=...`
-
-### Deploy!
-
-`fly deploy`
-
-## Need to do something more advanced?
-
-This demo covers the basics of bot telephony. If you want to know more about working with PSTN / SIP, please ping us on [Discord](https://discord.gg/pipecat).

examples/dialin-chatbot/bot_daily.py

@@ -1,111 +0,0 @@
-import asyncio
-import aiohttp
-import os
-import sys
-import argparse
-
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import LLMAssistantResponseAggregator, LLMUserResponseAggregator
-from pipecat.frames.frames import (
-    LLMMessagesFrame,
-    EndFrame
-)
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.openai import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport, DailyDialinSettings
-from pipecat.vad.silero import SileroVADAnalyzer
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-daily_api_key = os.getenv("DAILY_API_KEY", "")
-daily_api_url = os.getenv("DAILY_API_URL", "https://api.daily.co/v1")
-
-
-async def main(room_url: str, token: str, callId: str, callDomain: str):
-    async with aiohttp.ClientSession() as session:
-        # diallin_settings are only needed if Daily's SIP URI is used
-        # If you are handling this via Twilio, Telnyx, set this to None
-        # and handle call-forwarding when on_dialin_ready fires.
-        diallin_settings = DailyDialinSettings(
-            call_id=callId,
-            call_domain=callDomain
-        )
-
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Chatbot",
-            DailyParams(
-                api_url=daily_api_url,
-                api_key=daily_api_key,
-                dialin_settings=diallin_settings,
-                audio_in_enabled=True,
-                audio_out_enabled=True,
-                camera_out_enabled=False,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer(),
-                transcription_enabled=True,
-            )
-        )
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY", ""),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
-        )
-
-        llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
-
-        messages = [
-            {
-                "role": "system",
-                "content": "You are Chatbot, a friendly, helpful robot. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way, but keep your responses brief. Start by saying 'Oh, hello! Who dares dial me at this hour?!'.",
-            },
-        ]
-
-        tma_in = LLMUserResponseAggregator(messages)
-        tma_out = LLMAssistantResponseAggregator(messages)
-
-        pipeline = Pipeline([
-            transport.input(),
-            tma_in,
-            llm,
-            tts,
-            transport.output(),
-            tma_out,
-        ])
-
-        task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            transport.capture_participant_transcription(participant["id"])
-            await task.queue_frames([LLMMessagesFrame(messages)])
-
-        @transport.event_handler("on_participant_left")
-        async def on_participant_left(transport, participant, reason):
-            await task.queue_frame(EndFrame())
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Pipecat Simple ChatBot")
-    parser.add_argument("-u", type=str, help="Room URL")
-    parser.add_argument("-t", type=str, help="Token")
-    parser.add_argument("-i", type=str, help="Call ID")
-    parser.add_argument("-d", type=str, help="Call Domain")
-    config = parser.parse_args()
-
-    asyncio.run(main(config.u, config.t, config.i, config.d))

examples/dialin-chatbot/bot_runner.py

@@ -1,220 +0,0 @@
-"""
-bot_runner.py
-
-HTTP service that listens for incoming calls from either Daily or Twilio,
-provisioning a room and starting a Pipecat bot in response.
-
-Refer to README for more information.
-"""
-import os
-import argparse
-import subprocess
-from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper, DailyRoomObject, DailyRoomProperties, DailyRoomSipParams, DailyRoomParams
-from fastapi import FastAPI, Request, HTTPException
-from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import JSONResponse, PlainTextResponse
-from twilio.twiml.voice_response import VoiceResponse
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-
-# ------------ Configuration ------------ #
-
-MAX_SESSION_TIME = 5 * 60  # 5 minutes
-REQUIRED_ENV_VARS = ['OPENAI_API_KEY', 'DAILY_API_KEY',
-                     'ELEVENLABS_API_KEY', 'ELEVENLABS_VOICE_ID']
-
-daily_rest_helper = DailyRESTHelper(
-    os.getenv("DAILY_API_KEY", ""),
-    os.getenv("DAILY_API_URL", 'https://api.daily.co/v1'))
-
-
-# ----------------- API ----------------- #
-
-app = FastAPI()
-
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"]
-)
-
-"""
-Create Daily room, tell the bot if the room is created for Twilio's SIP or Daily's SIP (vendor).
-When the vendor is Daily, the bot handles the call forwarding automatically,
-i.e, forwards the call from the "hold music state" to the Daily Room's SIP URI.
-
-Alternatively, when the vendor is Twilio (not Daily), the bot is responsible for
-updating the state on Twilio. So when `dialin-ready` fires, it takes appropriate
-action using the Twilio Client library.
-"""
-
-
-def _create_daily_room(room_url, callId, callDomain=None, vendor="daily"):
-    if not room_url:
-        params = DailyRoomParams(
-            properties=DailyRoomProperties(
-                # Note: these are the default values, except for the display name
-                sip=DailyRoomSipParams(
-                    display_name="dialin-user",
-                    video=False,
-                    sip_mode="dial-in",
-                    num_endpoints=1
-                )
-            )
-        )
-
-        print(f"Creating new room...")
-        room: DailyRoomObject = daily_rest_helper.create_room(params=params)
-
-    else:
-        # Check passed room URL exist (we assume that it already has a sip set up!)
-        try:
-            print(f"Joining existing room: {room_url}")
-            room: DailyRoomObject = daily_rest_helper.get_room_from_url(
-                room_url)
-        except Exception:
-            raise HTTPException(
-                status_code=500, detail=f"Room not found: {room_url}")
-
-    print(f"Daily room: {room.url} {room.config.sip_endpoint}")
-
-    # Give the agent a token to join the session
-    token = daily_rest_helper.get_token(room.url, MAX_SESSION_TIME)
-
-    if not room or not token:
-        raise HTTPException(
-            status_code=500, detail=f"Failed to get room or token token")
-
-    # Spawn a new agent, and join the user session
-    # Note: this is mostly for demonstration purposes (refer to 'deployment' in docs)
-    if vendor == "daily":
-        bot_proc = f"python3 -m bot_daily -u {room.url} -t {token} -i {
-            callId} -d {callDomain}"
-    else:
-        bot_proc = f"python3 -m bot_twilio -u {room.url} -t {
-            token} -i {callId} -s {room.config.sip_endpoint}"
-
-    try:
-        subprocess.Popen(
-            [bot_proc],
-            shell=True,
-            bufsize=1,
-            cwd=os.path.dirname(os.path.abspath(__file__))
-        )
-    except Exception as e:
-        raise HTTPException(
-            status_code=500, detail=f"Failed to start subprocess: {e}")
-
-    return room
-
-
-@app.post("/twilio_start_bot", response_class=PlainTextResponse)
-async def twilio_start_bot(request: Request):
-    print(f"POST /twilio_voice_bot")
-
-    # twilio_start_bot is invoked directly by Twilio (as a web hook).
-    # On Twilio, under Active Numbers, pick the phone number
-    # Click Configure and under Voice Configuration,
-    # "a call comes in" choose webhook and point the URL to
-    # where this code is hosted.
-    data = {}
-    try:
-        # shouldnt have received json, twilio sends form data
-        form_data = await request.form()
-        data = dict(form_data)
-    except Exception:
-        pass
-
-    room_url = os.getenv("DAILY_SAMPLE_ROOM_URL", None)
-    callId = data.get('CallSid')
-
-    if not callId:
-        raise HTTPException(
-            status_code=500, detail="Missing 'CallSid' in request")
-
-    print("CallId: %s" % callId)
-
-    # create room and tell the bot to join the created room
-    # note: Twilio does not require a callDomain
-    room: DailyRoomObject = _create_daily_room(
-        room_url, callId, None, "twilio")
-
-    print(f"Put Twilio on hold...")
-    # We have the room and the SIP URI,
-    # but we do not know if the Daily SIP Worker and the Bot have joined the call
-    # put the call on hold until the 'on_dialin_ready' fires.
-    # Then, the bot will update the called sid with the sip uri.
-    # http://com.twilio.music.classical.s3.amazonaws.com/BusyStrings.mp3
-    resp = VoiceResponse()
-    resp.play(
-        url="http://com.twilio.sounds.music.s3.amazonaws.com/MARKOVICHAMP-Borghestral.mp3", loop=10)
-    return str(resp)
-
-
-@app.post("/daily_start_bot")
-async def daily_start_bot(request: Request) -> JSONResponse:
-    # The /daily_start_bot is invoked when a call is received on Daily's SIP URI
-    # daily_start_bot will create the room, put the call on hold until
-    # the bot and sip worker are ready. Daily will automatically
-    # forward the call to the SIP URi when dialin_ready fires.
-
-    # Use specified room URL, or create a new one if not specified
-    room_url = os.getenv("DAILY_SAMPLE_ROOM_URL", None)
-    # Get the dial-in properties from the request
-    try:
-        data = await request.json()
-        if "test" in data:
-            # Pass through any webhook checks
-            return JSONResponse({"test": True})
-        callId = data.get("callId", None)
-        callDomain = data.get("callDomain", None)
-    except Exception:
-        raise HTTPException(
-            status_code=500,
-            detail="Missing properties 'callId' or 'callDomain'")
-
-    print(f"CallId: {callId}, CallDomain: {callDomain}")
-    room: DailyRoomObject = _create_daily_room(
-        room_url, callId, callDomain, "daily")
-
-    # Grab a token for the user to join with
-    return JSONResponse({
-        "room_url": room.url,
-        "sipUri": room.config.sip_endpoint
-    })
-
-# ----------------- Main ----------------- #
-
-
-if __name__ == "__main__":
-    # Check environment variables
-    for env_var in REQUIRED_ENV_VARS:
-        if env_var not in os.environ:
-            raise Exception(f"Missing environment variable: {env_var}.")
-
-    parser = argparse.ArgumentParser(description="Pipecat Bot Runner")
-    parser.add_argument("--host", type=str,
-                        default=os.getenv("HOST", "0.0.0.0"), help="Host address")
-    parser.add_argument("--port", type=int,
-                        default=os.getenv("PORT", 7860), help="Port number")
-    parser.add_argument("--reload", action="store_true",
-                        default=True, help="Reload code on change")
-
-    config = parser.parse_args()
-
-    try:
-        import uvicorn
-
-        uvicorn.run(
-            "bot_runner:app",
-            host=config.host,
-            port=config.port,
-            reload=config.reload
-        )
-
-    except KeyboardInterrupt:
-        print("Pipecat runner shutting down...")

examples/dialin-chatbot/bot_twilio.py

@@ -1,125 +0,0 @@
-import asyncio
-import aiohttp
-import os
-import sys
-import argparse
-
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import LLMAssistantResponseAggregator, LLMUserResponseAggregator
-from pipecat.frames.frames import (
-    LLMMessagesFrame,
-    EndFrame
-)
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.openai import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from twilio.rest import Client
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-twilio_account_sid = os.getenv('TWILIO_ACCOUNT_SID')
-twilio_auth_token = os.getenv('TWILIO_AUTH_TOKEN')
-twilioclient = Client(twilio_account_sid, twilio_auth_token)
-
-daily_api_key = os.getenv("DAILY_API_KEY", "")
-
-
-async def main(room_url: str, token: str, callId: str, sipUri: str):
-    async with aiohttp.ClientSession() as session:
-        # diallin_settings are only needed if Daily's SIP URI is used
-        # If you are handling this via Twilio, Telnyx, set this to None
-        # and handle call-forwarding when on_dialin_ready fires.
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Chatbot",
-            DailyParams(
-                api_key=daily_api_key,
-                dialin_settings=None,  # Not required for Twilio
-                audio_in_enabled=True,
-                audio_out_enabled=True,
-                camera_out_enabled=False,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer(),
-                transcription_enabled=True,
-            )
-        )
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY", ""),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
-        )
-
-        llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
-
-        messages = [
-            {
-                "role": "system",
-                "content": "You are Chatbot, a friendly, helpful robot. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way, but keep your responses brief. Start by saying 'Hello! Who dares dial me at this hour?!'.",
-            },
-        ]
-
-        tma_in = LLMUserResponseAggregator(messages)
-        tma_out = LLMAssistantResponseAggregator(messages)
-
-        pipeline = Pipeline([
-            transport.input(),
-            tma_in,
-            llm,
-            tts,
-            transport.output(),
-            tma_out,
-        ])
-
-        task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            transport.capture_participant_transcription(participant["id"])
-            await task.queue_frames([LLMMessagesFrame(messages)])
-
-        @transport.event_handler("on_participant_left")
-        async def on_participant_left(transport, participant, reason):
-            await task.queue_frame(EndFrame())
-
-        @transport.event_handler("on_dialin_ready")
-        async def on_dialin_ready(transport, cdata):
-            # For Twilio, Telnyx, etc. You need to update the state of the call
-            # and forward it to the sip_uri..
-            print(f"Forwarding call: {callId} {sipUri}")
-
-            try:
-                # The TwiML is updated using Twilio's client library
-                call = twilioclient.calls(callId).update(
-                    twiml=f'<Response><Dial><Sip>{sipUri}</Sip></Dial></Response>'
-                )
-            except Exception as e:
-                raise Exception(f"Failed to forward call: {str(e)}")
-
-        runner = PipelineRunner()
-        await runner.run(task)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Pipecat Simple ChatBot")
-    parser.add_argument("-u", type=str, help="Room URL")
-    parser.add_argument("-t", type=str, help="Token")
-    parser.add_argument("-i", type=str, help="Call ID")
-    parser.add_argument("-s", type=str, help="SIP URI")
-    config = parser.parse_args()
-
-    asyncio.run(main(config.u, config.t, config.i, config.s))

examples/dialin-chatbot/env.example

@@ -1,8 +0,0 @@
-DAILY_SAMPLE_ROOM_URL=https://yourdomain.daily.co/yourroom # (optional: for joining the bot to the same room repeatedly for local dev)
-DAILY_API_KEY=.
-DAILY_API_URL=api.daily.co/v1
-OPENAI_API_KEY=
-ELEVENLABS_API_KEY=
-ELEVENLABS_VOICE_ID=
-TWILIO_ACCOUNT_SID=
-TWILIO_AUTH_TOKEN=

examples/dialin-chatbot/fly.example.toml

@@ -1,19 +0,0 @@
-# fly.toml app configuration file generated for pipecat-dialin-demo on 2024-06-03T15:57:57+02:00
-#
-# See https://fly.io/docs/reference/configuration/ for information about how to use this file.
-#
-
-app = 'pipecat-dialin-demo'
-primary_region = 'sjc'
-
-[build]
-
-[http_service]
-  internal_port = 7860
-  force_https = true
-  auto_stop_machines = true
-  auto_start_machines = true
-  min_machines_running = 1
-
-[[vm]]
-  size = 'performance-1x'

examples/dialin-chatbot/requirements.txt

@@ -1,7 +0,0 @@
-pipecat-ai[daily,openai,silero]
-fastapi
-uvicorn
-requests
-python-dotenv
-loguru
-twilio

examples/fast-chatbot/.gitignore

@@ -1,165 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-runpod.toml
-
-# custom script to recursively upgrade items in requirements.py
-upgrade_requirements.py
-.DS_Store

examples/fast-chatbot/bot.py

@@ -1,164 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-from loguru import logger
-import argparse
-import asyncio
-import aiohttp
-import os
-import sys
-import time
-from typing import Optional
-
-from pydantic import BaseModel, ValidationError
-
-from pipecat.vad.vad_analyzer import VADParams
-from pipecat.vad.silero import SileroVADAnalyzer
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.services.openai import OpenAILLMService
-from pipecat.services.deepgram import DeepgramSTTService
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.frames.frames import LLMMessagesFrame, EndFrame
-
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantResponseAggregator, LLMUserResponseAggregator
-)
-
-from helpers import (
-    ClearableDeepgramTTSService,
-    AudioVolumeTimer,
-    TranscriptionTimingLogger
-)
-
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level=os.getenv("LOG_LEVEL", "DEBUG"))
-
-
-class BotSettings(BaseModel):
-    room_url: str
-    room_token: str
-    bot_name: str = "Pipecat"
-    prompt: Optional[str] = "You are a helpful assistant."
-    deepgram_api_key: Optional[str] = os.getenv("DEEPGRAM_API_KEY", None)
-    deepgram_voice: Optional[str] = os.getenv("DEEPGRAM_VOICE", "aura-asteria-en")
-    deepgram_tts_base_url: Optional[str] = os.getenv(
-        "DEEPGRAM_TTS_BASE_URL", "https://api.deepgram.com/v1/speak")
-    deepgram_stt_base_url: Optional[str] = os.getenv(
-        "DEEPGRAM_STT_BASE_URL", "https://api.deepgram.com/v1/speak")
-    openai_api_key: Optional[str] = os.getenv("OPENAI_API_KEY", None),
-    openai_model: Optional[str] = os.getenv("OPENAI_MODEL", None),
-    openai_base_url: Optional[str] = os.getenv("OPENAI_BASE_URL", None)
-    vad_stop_secs: Optional[float] = os.getenv("VAD_STOP_SECS", 0.200)
-
-
-async def main(settings: BotSettings):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            settings.room_url,
-            settings.room_token,
-            settings.bot_name,
-            DailyParams(
-                audio_out_enabled=True,
-                transcription_enabled=False,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer(params=VADParams(
-                    stop_secs=settings.vad_stop_secs
-                )),
-                vad_audio_passthrough=True
-            )
-        )
-
-        stt = DeepgramSTTService(
-            name="STT",
-            api_key=settings.deepgram_api_key,
-            url=settings.deepgram_stt_base_url
-        )
-
-        tts = ClearableDeepgramTTSService(
-            name="Voice",
-            aiohttp_session=session,
-            api_key=settings.deepgram_api_key,
-            voice=settings.deepgram_voice,
-            **({'base_url': url} if (url := settings.deepgram_tts_base_url) else {})
-        )
-
-        llm = OpenAILLMService(
-            name="LLM",
-            api_key=settings.openai_api_key,
-            model=settings.openai_model,
-            base_url=settings.openai_base_url,
-        )
-
-        messages = [
-            {
-                "role": "system",
-                "content": settings.prompt,
-            },
-        ]
-
-        avt = AudioVolumeTimer()
-        tl = TranscriptionTimingLogger(avt)
-
-        tma_in = LLMUserResponseAggregator(messages)
-        tma_out = LLMAssistantResponseAggregator(messages)
-
-        pipeline = Pipeline([
-            transport.input(),   # Transport user input
-            avt,                 # Audio volume timer
-            stt,                 # Speech-to-text
-            tl,                  # Transcription timing logger
-            tma_in,              # User responses
-            llm,                 # LLM
-            tts,                 # TTS
-            transport.output(),  # Transport bot output
-            tma_out,             # Assistant spoken responses
-        ])
-
-        task = PipelineTask(
-            pipeline,
-            PipelineParams(
-                allow_interruptions=True,
-                enable_metrics=True,
-                report_only_initial_ttfb=True
-            ))
-
-        # When the participant leaves, we exit the bot.
-        @transport.event_handler("on_participant_left")
-        async def on_participant_left(transport, participant, reason):
-            await task.queue_frame(EndFrame())
-
-        # When the first participant joins, the bot should introduce itself.
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            # Provide some air whilst tracks subscribe
-            time.sleep(2)
-            messages.append(
-                {
-                    "role": "system",
-                    "content": "Briefly introduce yourself by saying 'hello, I'm FastBot, how can I help you today?'"})
-            await task.queue_frames([LLMMessagesFrame(messages)])
-
-        runner = PipelineRunner()
-        await runner.run(task)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Pipecat Bot")
-    parser.add_argument("-s", "--settings", type=str, required=True, help="Pipecat bot settings")
-
-    args, unknown = parser.parse_known_args()
-
-    try:
-        settings = BotSettings.model_validate_json(args.settings)
-        asyncio.run(main(settings))
-    except ValidationError as e:
-        print(e)

examples/fast-chatbot/bot_runner.py

@@ -1,164 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import os
-import argparse
-import subprocess
-
-from pydantic import BaseModel, ValidationError
-from typing import Optional
-
-from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper, DailyRoomObject, DailyRoomProperties, DailyRoomParams
-
-from fastapi import FastAPI, Request, HTTPException
-from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import JSONResponse
-
-from bot import BotSettings
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-
-# ------------ Configuration ------------ #
-
-MAX_SESSION_TIME = 5 * 60  # 5 minutes
-REQUIRED_ENV_VARS = ['DAILY_API_URL', 'DAILY_API_KEY', 'DEEPGRAM_API_KEY']
-
-daily_rest_helper = DailyRESTHelper(
-    os.getenv("DAILY_API_KEY", ""),
-    os.getenv("DAILY_API_URL", 'https://api.daily.co/v1'))
-
-
-class RunnerSettings(BaseModel):
-    prompt: Optional[
-        str] = "You are a fast, low-latency chatbot. Your goal is to demonstrate voice-driven AI capabilities at human-like speeds. When introducing yourself briefly mention your goal is to showcase speed and conversational flow. The technology powering you is Daily for transport, Cerebrium for GPU hosting, Llama 3 (8-B version) LLM, and Deepgram for speech-to-text and text-to-speech. You are hosted on the east coast of the United States. Respond to what the user said in a creative and helpful way, but keep responses short and legible. Ensure responses contain only words. Check again that you have not included special characters other than '?' or '!'."
-    deepgram_voice: Optional[str] = os.getenv("DEEPGRAM_VOICE")
-    openai_model: Optional[str] = os.getenv("OPENAI_MODEL", "gpt-4o")
-    openai_api_key: Optional[str] = os.getenv("OPENAI_API_KEY")
-    test: Optional[bool] = None
-
-# ----------------- API ----------------- #
-
-
-app = FastAPI()
-
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"]
-)
-
-# ----------------- Main ----------------- #
-
-
-@app.post("/start_bot")
-async def start_bot(request: Request) -> JSONResponse:
-    runner_settings = RunnerSettings()
-    try:
-        request_body = await request.body()
-        if len(request_body) > 0:
-            runner_settings = RunnerSettings.model_validate_json(request_body)
-    except ValidationError as e:
-        raise HTTPException(
-            status_code=400,
-            detail=f"Invalid request: {e}")
-    except Exception as e:
-        # If no data in request, pass
-        pass
-
-    # Is this a webhook creation request?
-    if runner_settings.test is not None:
-        return JSONResponse({"test": True})
-
-    # Use specified room URL, or create a new one if not specified
-    room_url = os.getenv("DAILY_SAMPLE_ROOM_URL", "")
-
-    if not room_url:
-        params = DailyRoomParams(
-            properties=DailyRoomProperties()
-        )
-        try:
-            room: DailyRoomObject = daily_rest_helper.create_room(params=params)
-        except Exception as e:
-            raise HTTPException(
-                status_code=500,
-                detail=f"Unable to provision room {e}")
-    else:
-        # Check passed room URL exists, we should assume that it already has a sip set up
-        try:
-            room: DailyRoomObject = daily_rest_helper.get_room_from_url(room_url)
-        except Exception:
-            raise HTTPException(
-                status_code=500, detail=f"Room not found: {room_url}")
-
-    # Give the agent a token to join the session
-    token = daily_rest_helper.get_token(room.url, MAX_SESSION_TIME)
-
-    if not room or not token:
-        raise HTTPException(
-            status_code=500, detail=f"Failed to get token for room: {room_url}")
-
-    # Spawn a new agent, and join the user session
-    try:
-        bot_settings = BotSettings(
-            room_url=room.url,
-            room_token=token,
-            prompt=runner_settings.prompt,
-            deepgram_voice=runner_settings.deepgram_voice,
-            openai_model=runner_settings.openai_model,
-            openai_api_key=runner_settings.openai_api_key,
-        )
-        bot_settings_str = bot_settings.model_dump_json(exclude_none=True)
-
-        subprocess.Popen(
-            [f"python3 -m bot -s '{bot_settings_str}'"],
-            shell=True,
-            bufsize=1,
-            cwd=os.path.dirname(os.path.abspath(__file__)))
-    except Exception as e:
-        raise HTTPException(
-            status_code=500, detail=f"Failed to start subprocess: {e}")
-
-    # Grab a token for the user to join with
-    user_token = daily_rest_helper.get_token(room.url, MAX_SESSION_TIME)
-
-    return JSONResponse({
-        "room_url": room.url,
-        "token": user_token,
-    })
-
-
-if __name__ == "__main__":
-    # Check environment variables
-    for env_var in REQUIRED_ENV_VARS:
-        if env_var not in os.environ:
-            raise Exception(f"Missing environment variable: {env_var}.")
-
-    parser = argparse.ArgumentParser(description="Pipecat Bot Runner")
-    parser.add_argument("--host", type=str,
-                        default=os.getenv("HOST", "0.0.0.0"), help="Host address")
-    parser.add_argument("--port", type=int,
-                        default=os.getenv("PORT", 7860), help="Port number")
-    parser.add_argument("--reload", action="store_true",
-                        default=True, help="Reload code on change")
-
-    config = parser.parse_args()
-
-    try:
-        import uvicorn
-
-        uvicorn.run(
-            "bot_runner:app",
-            host=config.host,
-            port=config.port,
-            reload=config.reload
-        )
-
-    except KeyboardInterrupt:
-        print("Pipecat runner shutting down...")

examples/fast-chatbot/env.example

@@ -1,12 +0,0 @@
-DAILY_SAMPLE_ROOM_URL= #optional: use the same room each time, or create a new one if unset
-DAILY_API_KEY=
-DAILY_API_URL=
-
-DEEPGRAM_API_KEY=
-DEEPGRAM_VOICE=
-DEEPGRAM_STT_URL=
-DEEPGRAM_TTS_BASE_URL=
-
-OPENAI_API_KEY=
-OPENAI_MODEL=
-OPENAI_BASE_URL=

examples/fast-chatbot/helpers.py

@@ -1,267 +0,0 @@
-from loguru import logger
-import asyncio
-import math
-import struct
-import time
-from dataclasses import dataclass, field
-from typing import List
-
-
-from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.frames.frames import (
-    Frame,
-    AudioRawFrame,
-    InterimTranscriptionFrame,
-    TranscriptionFrame,
-    TextFrame,
-    StartInterruptionFrame,
-    LLMFullResponseStartFrame,
-    TTSStoppedFrame,
-    MetricsFrame
-)
-
-from pipecat.vad.vad_analyzer import VADAnalyzer, VADState
-from pipecat.services.deepgram import DeepgramTTSService
-from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame
-
-
-class GreedyLLMAggregator(FrameProcessor):
-    def __init__(self, context: OpenAILLMContext = None, **kwargs):
-        super().__init__(**kwargs)
-        self.context: OpenAILLMContext = context if context else OpenAILLMContext()
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        logger.debug(f"{frame}")
-
-        try:
-            if isinstance(frame, InterimTranscriptionFrame):
-                return
-
-            if isinstance(frame, TranscriptionFrame):
-                # append transcribed text to last "user" frame
-                if self.context.messages and self.context.messages[-1]["role"] == "user":
-                    last_frame = self.context.messages.pop()
-                else:
-                    last_frame = {"role": "user", "content": ""}
-
-                last_frame["content"] += " " + frame.text
-                self.context.messages.append(last_frame)
-
-                oai_context_frame = OpenAILLMContextFrame(context=self.context)
-                logger.debug(f"pushing frame {oai_context_frame}")
-                await self.push_frame(oai_context_frame)
-                return
-
-            await self.push_frame(frame, direction)
-        except Exception as e:
-            logger.debug(f"error: {e}")
-
-
-class ClearableDeepgramTTSService(DeepgramTTSService):
-    def __init___(self, **kwargs):
-        super().__init(**kwargs)
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        if isinstance(frame, StartInterruptionFrame):
-            self._current_sentence = ""
-
-
-@dataclass
-class BufferedSentence:
-    audio_frames: List[AudioRawFrame] = field(default_factory=list)
-    text_frame: TextFrame = None
-
-
-class VADGate(FrameProcessor):
-
-    def __init__(
-            self,
-            vad_analyzer: VADAnalyzer = None,
-            context: OpenAILLMContext = None,
-            **kwargs):
-        super().__init__(**kwargs)
-        self.vad_analyzer = vad_analyzer
-        self.context = context
-
-        self._audio_pusher_task = None
-        self._expect_text_frame_next = False
-        self._sentences: List[BufferedSentence] = []
-
-    # queue output from tts one sentence at a time. associate a buffer of audio frames with the content of
-    # each text frame.
-    #
-    # start a coroutine to service the queue and send sentences down the pipeline when possible.
-    # 1. do not send anything when we are not in VADState.QUIET
-    # 2. if we are in VADState.QUIET, send a sentence, estimate how long it will take for that sentence
-    #    to output, sleep until it's time to send another sentence
-    # 3. each time we send a sentence, append it to the conversation context
-    # 3. when the sentence buffer becomes empty, cancel the coroutine
-    # 4. if we get a new LLMFullResponse, treat that as a cancellation, too
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        try:
-
-            # A TTSService will emit a series of AudioRawFrame objects, then a TTSStoppedFrame,
-            # then a TextFrame.
-
-            if self._expect_text_frame_next:
-                self._expect_text_frame_next = False
-                if isinstance(frame, TextFrame):
-                    self._sentences[-1].text_frame = frame
-                else:
-                    logger.debug(f"expected a text frame, but received {frame}")
-                    await self.push_frame(frame, direction)
-                return
-            else:
-                if isinstance(frame, TextFrame):
-                    logger.error(f"XXXXXXXXXXXXXXXXXXX received a text frame, wasn't expecting it.")
-
-            if isinstance(frame, AudioRawFrame):
-                # if our buffer is empty or has a "finished" sentence at the end,
-                # then we need to start buffering a new sentence
-                if not self._sentences or self._sentences[-1].text_frame:
-                    self._sentences.append(BufferedSentence())
-                self._sentences[-1].audio_frames.append(frame)
-                await self.maybe_start_audio_pusher_task()
-                return
-
-            if isinstance(frame, TTSStoppedFrame):
-                self._expect_text_frame_next = True
-                await self.push_frame(frame, direction)
-                return
-
-            # There are two ways we can be interrupted. During greedy inference, a new
-            # LLM response can start. Or, during playout, we can get a traditional
-            # user interruption frame.
-            if (isinstance(frame, LLMFullResponseStartFrame) or
-                    isinstance(frame, StartInterruptionFrame)):
-                logger.debug(f"{frame} - Handle interruption in VADGate")
-                self._sentences = []
-                if self._audio_pusher_task:
-                    self._audio_pusher_task.cancel()
-                    self._audio_pusher_task = None
-                await self.push_frame(frame, direction)
-                return
-
-            await self.push_frame(frame, direction)
-        except Exception as e:
-            logger.debug(f"error: {e}")
-
-    async def maybe_start_audio_pusher_task(self):
-        try:
-            if self._audio_pusher_task:
-                return
-            self._audio_pusher_task = self.get_event_loop().create_task(self.push_audio())
-
-        except Exception as e:
-            logger.debug(f"Exception {e}")
-
-    async def push_audio(self):
-        try:
-            while True:
-                if not self._sentences:
-                    await asyncio.sleep(0.01)
-                    continue
-
-                if self.vad_analyzer._vad_state != VADState.QUIET:
-                    await asyncio.sleep(0.01)
-                    continue
-
-                # we only want to push completed sentence buffers
-                if not self._sentences[0].text_frame:
-                    await asyncio.sleep(0.01)
-                    continue
-
-                s = self._sentences.pop(0)
-                if not s.audio_frames:
-                    continue
-                sample_rate = s.audio_frames[0].sample_rate
-                duration = 0
-                logger.debug(f"Pushing {len(s.audio_frames)} audio frames")
-                for frame in s.audio_frames:
-                    await self.push_frame(frame)
-                    # assume linear16 encoding (2 bytes per sample). todo: add some more
-                    # metadata to AudioRawFrame, maybe
-                    duration += (len(frame.audio) / 2 / frame.num_channels) / sample_rate
-                await asyncio.sleep(duration - 20 / 1000)
-                if self.context:
-                    logger.debug(f"Appending assistant message to context: [{s.text_frame.text}]")
-                    self.context.messages.append(
-                        {"role": "assistant", "content": s.text_frame.text}
-                    )
-                await self.push_frame(s.text_frame)
-
-        except Exception as e:
-            logger.debug(f"Exception {e}")
-
-
-class TranscriptionTimingLogger(FrameProcessor):
-    def __init__(self, avt):
-        super().__init__()
-        self.name = "Transcription"
-        self._avt = avt
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        try:
-            await super().process_frame(frame, direction)
-            if isinstance(frame, TranscriptionFrame):
-                elapsed = time.time() - self._avt.last_transition_ts
-                logger.debug(f"Transcription TTF: {elapsed}")
-                await self.push_frame(MetricsFrame(ttfb={self.name: elapsed}))
-
-            await self.push_frame(frame, direction)
-        except Exception as e:
-            logger.debug(f"Exception {e}")
-
-
-class AudioVolumeTimer(FrameProcessor):
-    def __init__(self):
-        super().__init__()
-        self.last_transition_ts = 0
-        self._prev_volume = -80
-        self._speech_volume_threshold = -50
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        if isinstance(frame, AudioRawFrame):
-            volume = self.calculate_volume(frame)
-            # print(f"Audio volume: {volume:.2f} dB")
-            if (volume >= self._speech_volume_threshold and
-                    self._prev_volume < self._speech_volume_threshold):
-                # logger.debug("transition above speech volume threshold")
-                self.last_transition_ts = time.time()
-            elif (volume < self._speech_volume_threshold and
-                    self._prev_volume >= self._speech_volume_threshold):
-                # logger.debug("transition below non-speech volume threshold")
-                self.last_transition_ts = time.time()
-            self._prev_volume = volume
-
-        await self.push_frame(frame, direction)
-
-    def calculate_volume(self, frame: AudioRawFrame) -> float:
-        if frame.num_channels != 1:
-            raise ValueError(f"Expected 1 channel, got {frame.num_channels}")
-
-        # Unpack audio data into 16-bit integers
-        fmt = f"{len(frame.audio) // 2}h"
-        audio_samples = struct.unpack(fmt, frame.audio)
-
-        # Calculate RMS
-        sum_squares = sum(sample**2 for sample in audio_samples)
-        rms = math.sqrt(sum_squares / len(audio_samples))
-
-        # Convert RMS to decibels (dB)
-        # Reference: maximum value for 16-bit audio is 32767
-        if rms > 0:
-            db = 20 * math.log10(rms / 32767)
-        else:
-            db = -96  # Minimum value (almost silent)
-
-        return db

examples/fast-chatbot/requirements.txt

@@ -1,6 +0,0 @@
-pipecat-ai[daily,openai,silero,deepgram]
-fastapi
-uvicorn
-requests
-python-dotenv
-loguru

examples/foundational/01-say-one-thing.py

@@ -1,36 +1,31 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
 import asyncio
 import aiohttp
+import logging
 import os
-import sys
+from dailyai.pipeline.frames import EndFrame, TextFrame
+from dailyai.pipeline.pipeline import Pipeline
 
-from pipecat.frames.frames import EndFrame, TextFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.task import PipelineTask
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
+from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
 
 from runner import configure
 
-from loguru import logger
-
 from dotenv import load_dotenv
 load_dotenv(override=True)
 
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
 
 
 async def main(room_url):
     async with aiohttp.ClientSession() as session:
         transport = DailyTransport(
-            room_url, None, "Say One Thing", DailyParams(audio_out_enabled=True))
+            room_url,
+            None,
+            "Say One Thing",
+            mic_enabled=True,
+        )
 
         tts = ElevenLabsTTSService(
             aiohttp_session=session,
@@ -38,18 +33,21 @@ async def main(room_url):
             voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
         )
 
-        runner = PipelineRunner()
-
-        task = PipelineTask(Pipeline([tts, transport.output()]))
+        pipeline = Pipeline([tts])
 
         # Register an event handler so we can play the audio when the
         # participant joins.
         @transport.event_handler("on_participant_joined")
-        async def on_new_participant_joined(transport, participant):
-            participant_name = participant["info"]["userName"] or ''
-            await task.queue_frames([TextFrame(f"Hello there, {participant_name}!"), EndFrame()])
+        async def on_participant_joined(transport, participant):
+            if participant["info"]["isLocal"]:
+                return
+
+            participant_name = participant["info"]["userName"] or ''
+            await pipeline.queue_frames([TextFrame("Hello there, " + participant_name + "!"), EndFrame()])
+
+        await transport.run(pipeline)
+        del tts
 
-        await runner.run(task)
 
 if __name__ == "__main__":
     (url, token) = configure()

examples/foundational/01a-local-audio.py

@@ -1,53 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import aiohttp
-import os
-import sys
-
-from pipecat.frames.frames import EndFrame, TextFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.transports.base_transport import TransportParams
-from pipecat.transports.local.audio import LocalAudioTransport
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def main():
-    async with aiohttp.ClientSession() as session:
-        transport = LocalAudioTransport(TransportParams(audio_out_enabled=True))
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
-        )
-
-        pipeline = Pipeline([tts, transport.output()])
-
-        task = PipelineTask(pipeline)
-
-        async def say_something():
-            await asyncio.sleep(1)
-            await task.queue_frames([TextFrame("Hello there!"), EndFrame()])
-
-        runner = PipelineRunner()
-
-        await asyncio.gather(runner.run(task), say_something())
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

examples/foundational/01a-local-transport.py

@@ -0,0 +1,38 @@
+import asyncio
+import aiohttp
+import logging
+import os
+
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.transports.local_transport import LocalTransport
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        meeting_duration_minutes = 1
+        transport = LocalTransport(
+            duration_minutes=meeting_duration_minutes, mic_enabled=True
+        )
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        async def say_something():
+            await asyncio.sleep(1)
+            await tts.say(
+                "Hello there.",
+                transport.send_queue,
+            )
+            await transport.stop_when_done()
+
+        await asyncio.gather(transport.run(), say_something())
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/foundational/02-llm-say-one-thing.py

@@ -1,31 +1,23 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
 import asyncio
-import aiohttp
 import os
-import sys
+import logging
 
-from pipecat.frames.frames import EndFrame, LLMMessagesFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.openai import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
+import aiohttp
+
+from dailyai.pipeline.frames import EndFrame, LLMMessagesFrame
+from dailyai.pipeline.pipeline import Pipeline
+from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.open_ai_services import OpenAILLMService
 
 from runner import configure
 
-from loguru import logger
-
 from dotenv import load_dotenv
 load_dotenv(override=True)
 
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
 
 
 async def main(room_url):
@@ -34,7 +26,8 @@ async def main(room_url):
             room_url,
             None,
             "Say One Thing From an LLM",
-            DailyParams(audio_out_enabled=True))
+            mic_enabled=True,
+        )
 
         tts = ElevenLabsTTSService(
             aiohttp_session=session,
@@ -44,7 +37,7 @@ async def main(room_url):
 
         llm = OpenAILLMService(
             api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
+            model="gpt-4-turbo-preview")
 
         messages = [
             {
@@ -52,15 +45,13 @@ async def main(room_url):
                 "content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world.",
             }]
 
-        runner = PipelineRunner()
+        pipeline = Pipeline([llm, tts])
 
-        task = PipelineTask(Pipeline([llm, tts, transport.output()]))
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            await pipeline.queue_frames([LLMMessagesFrame(messages), EndFrame()])
 
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            await task.queue_frames([LLMMessagesFrame(messages), EndFrame()])
-
-        await runner.run(task)
+        await transport.run(pipeline)
 
 
 if __name__ == "__main__":

examples/foundational/03-still-frame.py

@@ -1,30 +1,21 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
 import asyncio
 import aiohttp
+import logging
 import os
-import sys
 
-from pipecat.frames.frames import TextFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.services.fal import FalImageGenService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
+from dailyai.pipeline.frames import TextFrame
+from dailyai.pipeline.pipeline import Pipeline
+from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.fal_ai_services import FalImageGenService
 
 from runner import configure
 
-from loguru import logger
-
 from dotenv import load_dotenv
 load_dotenv(override=True)
 
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
 
 
 async def main(room_url):
@@ -33,34 +24,32 @@ async def main(room_url):
             room_url,
             None,
             "Show a still frame image",
-            DailyParams(
-                camera_out_enabled=True,
-                camera_out_width=1024,
-                camera_out_height=1024
-            )
+            camera_enabled=True,
+            camera_width=1024,
+            camera_height=1024,
+            duration_minutes=1
         )
 
         imagegen = FalImageGenService(
-            params=FalImageGenService.InputParams(
-                image_size="square_hd"
-            ),
+            image_size="square_hd",
             aiohttp_session=session,
-            key=os.getenv("FAL_KEY"),
+            key_id=os.getenv("FAL_KEY_ID"),
+            key_secret=os.getenv("FAL_KEY_SECRET"),
         )
 
-        runner = PipelineRunner()
+        pipeline = Pipeline([imagegen])
 
-        task = PipelineTask(Pipeline([imagegen, transport.output()]))
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
             # Note that we do not put an EndFrame() item in the pipeline for this demo.
             # This means that the bot will stay in the channel until it times out.
             # An EndFrame() in the pipeline would cause the transport to shut
             # down.
-            await task.queue_frames([TextFrame("a cat in the style of picasso")])
+            await pipeline.queue_frames(
+                [TextFrame("a cat in the style of picasso")]
+            )
 
-        await runner.run(task)
+        await transport.run(pipeline)
 
 
 if __name__ == "__main__":

examples/foundational/03a-image-local.py

@@ -0,0 +1,55 @@
+import asyncio
+import aiohttp
+import logging
+import os
+
+import tkinter as tk
+
+from dailyai.pipeline.frames import TextFrame
+from dailyai.services.fal_ai_services import FalImageGenService
+from dailyai.transports.local_transport import LocalTransport
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+local_joined = False
+participant_joined = False
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        meeting_duration_minutes = 2
+        tk_root = tk.Tk()
+        tk_root.title("Calendar")
+        transport = LocalTransport(
+            tk_root=tk_root,
+            mic_enabled=True,
+            camera_enabled=True,
+            camera_width=1024,
+            camera_height=1024,
+            duration_minutes=meeting_duration_minutes,
+        )
+
+        imagegen = FalImageGenService(
+            image_size="1024x1024",
+            aiohttp_session=session,
+            key_id=os.getenv("FAL_KEY_ID"),
+            key_secret=os.getenv("FAL_KEY_SECRET"),
+        )
+        image_task = asyncio.create_task(
+            imagegen.run_to_queue(
+                transport.send_queue, [
+                    TextFrame("a cat in the style of picasso")]))
+
+        async def run_tk():
+            while not transport._stop_threads.is_set():
+                tk_root.update()
+                tk_root.update_idletasks()
+                await asyncio.sleep(0.1)
+
+        await asyncio.gather(transport.run(), image_task, run_tk())
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/foundational/03a-local-still-frame.py

@@ -1,68 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import aiohttp
-import os
-import sys
-
-import tkinter as tk
-
-from pipecat.frames.frames import TextFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.services.fal import FalImageGenService
-from pipecat.transports.base_transport import TransportParams
-from pipecat.transports.local.tk import TkLocalTransport
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def main():
-    async with aiohttp.ClientSession() as session:
-        tk_root = tk.Tk()
-        tk_root.title("Picasso Cat")
-
-        transport = TkLocalTransport(
-            tk_root,
-            TransportParams(
-                camera_out_enabled=True,
-                camera_out_width=1024,
-                camera_out_height=1024))
-
-        imagegen = FalImageGenService(
-            params=FalImageGenService.InputParams(
-                image_size="square_hd"
-            ),
-            aiohttp_session=session,
-            key=os.getenv("FAL_KEY"),
-        )
-
-        pipeline = Pipeline([imagegen, transport.output()])
-
-        task = PipelineTask(pipeline)
-        await task.queue_frames([TextFrame("a cat in the style of picasso")])
-
-        runner = PipelineRunner()
-
-        async def run_tk():
-            while runner.is_active():
-                tk_root.update()
-                tk_root.update_idletasks()
-                await asyncio.sleep(0.1)
-
-        await asyncio.gather(runner.run(task), run_tk())
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

examples/foundational/04-utterance-and-speech.py

@@ -1,40 +1,37 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
+import asyncio
+import logging
+import os
 
 import aiohttp
-import asyncio
-import os
-import sys
+from dailyai.pipeline.merge_pipeline import SequentialMergePipeline
+from dailyai.pipeline.pipeline import Pipeline
 
-from pipecat.pipeline.merge_pipeline import SequentialMergePipeline
-from pipecat.pipeline.pipeline import Pipeline
-
-from pipecat.frames.frames import EndPipeFrame, LLMMessagesFrame, TextFrame
-from pipecat.pipeline.task import PipelineTask
-from pipecat.services.azure import AzureLLMService, AzureTTSService
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.transport_services import TransportServiceOutput
-from pipecat.services.transports.daily_transport import DailyTransport
+from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
+from dailyai.services.deepgram_ai_services import DeepgramTTSService
+from dailyai.pipeline.frames import EndPipeFrame, LLMMessagesFrame, TextFrame
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
 
 from runner import configure
 
-from loguru import logger
-
 from dotenv import load_dotenv
 load_dotenv(override=True)
 
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
 
 
 async def main(room_url: str):
     async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(room_url, None, "Static And Dynamic Speech")
-
-        meeting = TransportServiceOutput(transport, mic_enabled=True)
+        transport = DailyTransport(
+            room_url,
+            None,
+            "Static And Dynamic Speech",
+            duration_minutes=1,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+        )
 
         llm = AzureLLMService(
             api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
@@ -46,6 +43,10 @@ async def main(room_url: str):
             region=os.getenv("AZURE_SPEECH_REGION"),
         )
 
+        deepgram_tts = DeepgramTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("DEEPGRAM_API_KEY"),
+        )
         elevenlabs_tts = ElevenLabsTTSService(
             aiohttp_session=session,
             api_key=os.getenv("ELEVENLABS_API_KEY"),
@@ -55,13 +56,11 @@ async def main(room_url: str):
         messages = [{"role": "system",
                      "content": "tell the user a joke about llamas"}]
 
-        # Start a task to run the LLM to create a joke, and convert the LLM
-        # output to audio frames. This task will run in parallel with generating
-        # and speaking the audio for static text, so there's no delay to speak
-        # the LLM response.
+        # Start a task to run the LLM to create a joke, and convert the LLM output to audio frames. This task
+        # will run in parallel with generating and speaking the audio for static text, so there's no delay to
+        # speak the LLM response.
         llm_pipeline = Pipeline([llm, elevenlabs_tts])
-        llm_task = PipelineTask(llm_pipeline)
-        await llm_task.queue_frames([LLMMessagesFrame(messages), EndPipeFrame()])
+        await llm_pipeline.queue_frames([LLMMessagesFrame(messages), EndPipeFrame()])
 
         simple_tts_pipeline = Pipeline([azure_tts])
         await simple_tts_pipeline.queue_frames(

examples/foundational/05-sync-speech-and-image.py

@@ -1,76 +1,64 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
 import asyncio
 import aiohttp
 import os
-import sys
+import logging
 
 from dataclasses import dataclass
+from typing import AsyncGenerator
 
-from pipecat.frames.frames import (
-    AppFrame,
-    EndFrame,
-    Frame,
-    ImageRawFrame,
-    LLMFullResponseStartFrame,
-    LLMMessagesFrame,
-    TextFrame
+from dailyai.pipeline.aggregators import (
+    GatedAggregator,
+    LLMFullResponseAggregator,
+    ParallelPipeline,
+    SentenceAggregator,
 )
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.pipeline.parallel_task import ParallelTask
-from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.processors.aggregators.gated import GatedAggregator
-from pipecat.processors.aggregators.llm_response import LLMFullResponseAggregator
-from pipecat.processors.aggregators.sentence import SentenceAggregator
-from pipecat.services.openai import OpenAILLMService
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.fal import FalImageGenService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
+from dailyai.pipeline.frames import (
+    Frame,
+    TextFrame,
+    EndFrame,
+    ImageFrame,
+    LLMMessagesFrame,
+    LLMResponseStartFrame,
+)
+from dailyai.pipeline.frame_processor import FrameProcessor
+
+from dailyai.pipeline.pipeline import Pipeline
+from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.fal_ai_services import FalImageGenService
 
 from runner import configure
 
-from loguru import logger
-
 from dotenv import load_dotenv
 load_dotenv(override=True)
 
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
 
 
 @dataclass
-class MonthFrame(AppFrame):
+class MonthFrame(Frame):
     month: str
 
-    def __str__(self):
-        return f"{self.name}(month: {self.month})"
-
 
 class MonthPrepender(FrameProcessor):
     def __init__(self):
-        super().__init__()
         self.most_recent_month = "Placeholder, month frame not yet received"
         self.prepend_to_next_text_frame = False
 
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
         if isinstance(frame, MonthFrame):
             self.most_recent_month = frame.month
         elif self.prepend_to_next_text_frame and isinstance(frame, TextFrame):
-            await self.push_frame(TextFrame(f"{self.most_recent_month}: {frame.text}"))
+            yield TextFrame(f"{self.most_recent_month}: {frame.text}")
             self.prepend_to_next_text_frame = False
-        elif isinstance(frame, LLMFullResponseStartFrame):
+        elif isinstance(frame, LLMResponseStartFrame):
             self.prepend_to_next_text_frame = True
-            await self.push_frame(frame)
+            yield frame
         else:
-            await self.push_frame(frame, direction)
+            yield frame
 
 
 async def main(room_url):
@@ -79,12 +67,11 @@ async def main(room_url):
             room_url,
             None,
             "Month Narration Bot",
-            DailyParams(
-                audio_out_enabled=True,
-                camera_out_enabled=True,
-                camera_out_width=1024,
-                camera_out_height=1024
-            )
+            mic_enabled=True,
+            camera_enabled=True,
+            mic_sample_rate=16000,
+            camera_width=1024,
+            camera_height=1024,
         )
 
         tts = ElevenLabsTTSService(
@@ -95,36 +82,34 @@ async def main(room_url):
 
         llm = OpenAILLMService(
             api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
+            model="gpt-4-turbo-preview")
 
         imagegen = FalImageGenService(
-            params=FalImageGenService.InputParams(
-                image_size="square_hd"
-            ),
+            image_size="square_hd",
             aiohttp_session=session,
-            key=os.getenv("FAL_KEY"),
+            key_id=os.getenv("FAL_KEY_ID"),
+            key_secret=os.getenv("FAL_KEY_SECRET"),
         )
 
         gated_aggregator = GatedAggregator(
-            gate_open_fn=lambda frame: isinstance(frame, ImageRawFrame),
-            gate_close_fn=lambda frame: isinstance(frame, LLMFullResponseStartFrame),
-            start_open=False
-        )
+            gate_open_fn=lambda frame: isinstance(
+                frame, ImageFrame), gate_close_fn=lambda frame: isinstance(
+                frame, LLMResponseStartFrame), start_open=False, )
 
         sentence_aggregator = SentenceAggregator()
         month_prepender = MonthPrepender()
         llm_full_response_aggregator = LLMFullResponseAggregator()
 
-        pipeline = Pipeline([
-            llm,                     # LLM
-            sentence_aggregator,     # Aggregates LLM output into full sentences
-            ParallelTask(            # Run pipelines in parallel aggregating the result
-                [month_prepender, tts],                   # Create "Month: sentence" and output audio
-                [llm_full_response_aggregator, imagegen]  # Aggregate full LLM response
-            ),
-            gated_aggregator,        # Queues everything until an image is available
-            transport.output()       # Transport output
-        ])
+        pipeline = Pipeline(
+            processors=[
+                llm,
+                sentence_aggregator,
+                ParallelPipeline(
+                    [[month_prepender, tts], [llm_full_response_aggregator, imagegen]]
+                ),
+                gated_aggregator,
+            ],
+        )
 
         frames = []
         for month in [
@@ -147,18 +132,13 @@ async def main(room_url):
                     "content": f"Describe a nature photograph suitable for use in a calendar, for the month of {month}. Include only the image description with no preamble. Limit the description to one sentence, please.",
                 }
             ]
-            frames.append(MonthFrame(month=month))
+            frames.append(MonthFrame(month))
             frames.append(LLMMessagesFrame(messages))
 
         frames.append(EndFrame())
+        await pipeline.queue_frames(frames)
 
-        runner = PipelineRunner()
-
-        task = PipelineTask(pipeline)
-
-        await task.queue_frames(frames)
-
-        await runner.run(task)
+        await transport.run(pipeline, override_pipeline_source_queue=False)
 
 
 if __name__ == "__main__":

examples/foundational/05a-local-sync-speech-and-image.py

@@ -1,174 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import aiohttp
-import asyncio
-import os
-import sys
-
-import tkinter as tk
-
-from pipecat.frames.frames import AudioRawFrame, Frame, URLImageRawFrame, LLMMessagesFrame, TextFrame
-from pipecat.pipeline.parallel_pipeline import ParallelPipeline
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.processors.aggregators.llm_response import LLMFullResponseAggregator
-from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.services.openai import OpenAILLMService
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.fal import FalImageGenService
-from pipecat.transports.base_transport import TransportParams
-from pipecat.transports.local.tk import TkLocalTransport
-
-from loguru import logger
-
-from dotenv import load_dotenv
-
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def main():
-    async with aiohttp.ClientSession() as session:
-        tk_root = tk.Tk()
-        tk_root.title("Calendar")
-
-        runner = PipelineRunner()
-
-        async def get_month_data(month):
-            messages = [{"role": "system", "content": f"Describe a nature photograph suitable for use in a calendar, for the month of {month}. Include only the image description with no preamble. Limit the description to one sentence, please.", }]
-
-            class ImageDescription(FrameProcessor):
-                def __init__(self):
-                    super().__init__()
-                    self.text = ""
-
-                async def process_frame(self, frame: Frame, direction: FrameDirection):
-                    await super().process_frame(frame, direction)
-
-                    if isinstance(frame, TextFrame):
-                        self.text = frame.text
-                    await self.push_frame(frame, direction)
-
-            class AudioGrabber(FrameProcessor):
-                def __init__(self):
-                    super().__init__()
-                    self.audio = bytearray()
-
-                async def process_frame(self, frame: Frame, direction: FrameDirection):
-                    await super().process_frame(frame, direction)
-
-                    if isinstance(frame, AudioRawFrame):
-                        self.audio.extend(frame.audio)
-                        self.frame = AudioRawFrame(
-                            bytes(self.audio), frame.sample_rate, frame.num_channels)
-
-            class ImageGrabber(FrameProcessor):
-                def __init__(self):
-                    super().__init__()
-                    self.frame = None
-
-                async def process_frame(self, frame: Frame, direction: FrameDirection):
-                    await super().process_frame(frame, direction)
-
-                    if isinstance(frame, URLImageRawFrame):
-                        self.frame = frame
-
-            llm = OpenAILLMService(
-                api_key=os.getenv("OPENAI_API_KEY"),
-                model="gpt-4o")
-
-            tts = ElevenLabsTTSService(
-                aiohttp_session=session,
-                api_key=os.getenv("ELEVENLABS_API_KEY"),
-                voice_id=os.getenv("ELEVENLABS_VOICE_ID"))
-
-            imagegen = FalImageGenService(
-                params=FalImageGenService.InputParams(
-                    image_size="square_hd"
-                ),
-                aiohttp_session=session,
-                key=os.getenv("FAL_KEY"))
-
-            aggregator = LLMFullResponseAggregator()
-
-            description = ImageDescription()
-
-            audio_grabber = AudioGrabber()
-
-            image_grabber = ImageGrabber()
-
-            pipeline = Pipeline([
-                llm,
-                aggregator,
-                description,
-                ParallelPipeline([tts, audio_grabber],
-                                 [imagegen, image_grabber])
-            ])
-
-            task = PipelineTask(pipeline)
-            await task.queue_frame(LLMMessagesFrame(messages))
-            await task.stop_when_done()
-
-            await runner.run(task)
-
-            return {
-                "month": month,
-                "text": description.text,
-                "image": image_grabber.frame,
-                "audio": audio_grabber.frame,
-            }
-
-        transport = TkLocalTransport(
-            tk_root,
-            TransportParams(
-                audio_out_enabled=True,
-                camera_out_enabled=True,
-                camera_out_width=1024,
-                camera_out_height=1024))
-
-        pipeline = Pipeline([transport.output()])
-
-        task = PipelineTask(pipeline)
-
-        # We only specify 5 months as we create tasks all at once and we might
-        # get rate limited otherwise.
-        months: list[str] = [
-            "January",
-            "February",
-            # "March",
-            # "April",
-            # "May",
-        ]
-
-        # We create one task per month. This will be executed concurrently.
-        month_tasks = [asyncio.create_task(get_month_data(month)) for month in months]
-
-        # Now we wait for each month task in the order they're completed. The
-        # benefit is we'll have as little delay as possible before the first
-        # month, and likely no delay between months, but the months won't
-        # display in order.
-        async def show_images(month_tasks):
-            for month_data_task in asyncio.as_completed(month_tasks):
-                data = await month_data_task
-                await task.queue_frames([data["image"], data["audio"]])
-
-            await runner.stop_when_done()
-
-        async def run_tk():
-            while not task.has_finished():
-                tk_root.update()
-                tk_root.update_idletasks()
-                await asyncio.sleep(0.1)
-
-        await asyncio.gather(runner.run(task), show_images(month_tasks), run_tk())
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

examples/foundational/05a-local-sync-speech-and-text.py

@@ -0,0 +1,146 @@
+import aiohttp
+import argparse
+import asyncio
+import logging
+import tkinter as tk
+import os
+
+from dailyai.pipeline.frames import AudioFrame, ImageFrame
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.fal_ai_services import FalImageGenService
+from dailyai.transports.local_transport import LocalTransport
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+async def main(room_url):
+    async with aiohttp.ClientSession() as session:
+        meeting_duration_minutes = 5
+        tk_root = tk.Tk()
+        tk_root.title("Calendar")
+
+        transport = LocalTransport(
+            mic_enabled=True,
+            camera_enabled=True,
+            camera_width=1024,
+            camera_height=1024,
+            duration_minutes=meeting_duration_minutes,
+            tk_root=tk_root,
+        )
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        dalle = FalImageGenService(
+            image_size="1024x1024",
+            aiohttp_session=session,
+            key_id=os.getenv("FAL_KEY_ID"),
+            key_secret=os.getenv("FAL_KEY_SECRET"),
+        )
+
+        # Get a complete audio chunk from the given text. Splitting this into its own
+        # coroutine lets us ensure proper ordering of the audio chunks on the
+        # send queue.
+        async def get_all_audio(text):
+            all_audio = bytearray()
+            async for audio in tts.run_tts(text):
+                all_audio.extend(audio)
+
+            return all_audio
+
+        async def get_month_data(month):
+            messages = [
+                {
+                    "role": "system",
+                    "content": f"Describe a nature photograph suitable for use in a calendar, for the month of {month}. Include only the image description with no preamble. Limit the description to one sentence, please.",
+                }
+            ]
+
+            image_description = await llm.run_llm(messages)
+            if not image_description:
+                return
+
+            to_speak = f"{month}: {image_description}"
+            audio_task = asyncio.create_task(get_all_audio(to_speak))
+            image_task = asyncio.create_task(
+                dalle.run_image_gen(image_description))
+            (audio, image_data) = await asyncio.gather(audio_task, image_task)
+
+            return {
+                "month": month,
+                "text": image_description,
+                "image_url": image_data[0],
+                "image": image_data[1],
+                "audio": audio,
+            }
+
+        months: list[str] = [
+            "January",
+            "February",
+            "March",
+            "April",
+            "May",
+            "June",
+            "July",
+            "August",
+            "September",
+            "October",
+            "November",
+            "December",
+        ]
+
+        async def show_images():
+            # This will play the months in the order they're completed. The benefit
+            # is we'll have as little delay as possible before the first month, and
+            # likely no delay between months, but the months won't display in
+            # order.
+            for month_data_task in asyncio.as_completed(month_tasks):
+                data = await month_data_task
+                if data:
+                    await transport.send_queue.put(
+                        [
+                            ImageFrame(data["image_url"], data["image"]),
+                            AudioFrame(data["audio"]),
+                        ]
+                    )
+
+            await asyncio.sleep(25)
+
+            # wait for the output queue to be empty, then leave the meeting
+            await transport.stop_when_done()
+
+        async def run_tk():
+            while not transport._stop_threads.is_set():
+                tk_root.update()
+                tk_root.update_idletasks()
+                await asyncio.sleep(0.1)
+
+        month_tasks = [
+            asyncio.create_task(
+                get_month_data(month)) for month in months]
+
+        await asyncio.gather(transport.run(), show_images(), run_tk())
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
+    parser.add_argument(
+        "-u",
+        "--url",
+        type=str,
+        required=True,
+        help="URL of the Daily room to join")
+
+    args, unknown = parser.parse_known_args()
+
+    asyncio.run(main(args.url))

examples/foundational/06-listen-and-respond.py

@@ -1,37 +1,26 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
 import asyncio
 import aiohttp
+import logging
 import os
-import sys
+from dailyai.pipeline.frames import LLMMessagesFrame
+from dailyai.pipeline.pipeline import Pipeline
 
-from pipecat.frames.frames import LLMMessagesFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantResponseAggregator,
-    LLMUserResponseAggregator,
+from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.ai_services import FrameLogger
+from dailyai.pipeline.aggregators import (
+    LLMAssistantContextAggregator,
+    LLMUserContextAggregator,
 )
-from pipecat.processors.logger import FrameLogger
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.openai import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
 from runner import configure
 
-from loguru import logger
-
 from dotenv import load_dotenv
 load_dotenv(override=True)
 
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
 
 
 async def main(room_url: str, token):
@@ -40,12 +29,12 @@ async def main(room_url: str, token):
             room_url,
             token,
             "Respond bot",
-            DailyParams(
-                audio_out_enabled=True,
-                transcription_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer()
-            )
+            duration_minutes=5,
+            start_transcription=True,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            camera_enabled=False,
+            vad_enabled=True,
         )
 
         tts = ElevenLabsTTSService(
@@ -56,46 +45,42 @@ async def main(room_url: str, token):
 
         llm = OpenAILLMService(
             api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
-
-        fl = FrameLogger("!!! after LLM", "red")
-        fltts = FrameLogger("@@@ out of tts", "green")
-        flend = FrameLogger("### out of the end", "magenta")
-
+            model="gpt-4-turbo-preview")
+        fl = FrameLogger("Inner")
+        fl2 = FrameLogger("Outer")
         messages = [
             {
                 "role": "system",
-                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
+                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
             },
         ]
-        tma_in = LLMUserResponseAggregator(messages)
-        tma_out = LLMAssistantResponseAggregator(messages)
 
-        pipeline = Pipeline([
-            transport.input(),
-            tma_in,
-            llm,
-            fl,
-            tts,
-            fltts,
-            transport.output(),
-            tma_out,
-            flend
-        ])
+        tma_in = LLMUserContextAggregator(
+            messages, transport._my_participant_id)
+        tma_out = LLMAssistantContextAggregator(
+            messages, transport._my_participant_id
+        )
+        pipeline = Pipeline(
+            processors=[
+                fl,
+                tma_in,
+                llm,
+                fl2,
+                tts,
+                tma_out,
+            ],
+        )
 
-        task = PipelineTask(pipeline)
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            transport.capture_participant_transcription(participant["id"])
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
             # Kick off the conversation.
             messages.append(
                 {"role": "system", "content": "Please introduce yourself to the user."})
-            await task.queue_frames([LLMMessagesFrame(messages)])
+            await pipeline.queue_frames([LLMMessagesFrame(messages)])
 
-        runner = PipelineRunner()
-
-        await runner.run(task)
+        transport.transcription_settings["extra"]["endpointing"] = True
+        transport.transcription_settings["extra"]["punctuate"] = True
+        await transport.run(pipeline)
 
 
 if __name__ == "__main__":

examples/foundational/06a-image-sync.py

@@ -1,62 +1,43 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
 import asyncio
-import aiohttp
 import os
-import sys
-
+import logging
+from typing import AsyncGenerator
+import aiohttp
 from PIL import Image
 
-from pipecat.frames.frames import ImageRawFrame, Frame, SystemFrame, TextFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantResponseAggregator,
-    LLMUserResponseAggregator,
+from dailyai.pipeline.frames import ImageFrame, Frame
+from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.ai_services import AIService
+from dailyai.pipeline.aggregators import (
+    LLMAssistantContextAggregator,
+    LLMUserContextAggregator,
 )
-from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.services.openai import OpenAILLMService
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.transports.services.daily import DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.fal_ai_services import FalImageGenService
 
-from pipecat.transports.services.daily import DailyParams
 from runner import configure
 
-from loguru import logger
-
 from dotenv import load_dotenv
 load_dotenv(override=True)
 
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
 
 
-class ImageSyncAggregator(FrameProcessor):
+class ImageSyncAggregator(AIService):
     def __init__(self, speaking_path: str, waiting_path: str):
-        super().__init__()
         self._speaking_image = Image.open(speaking_path)
-        self._speaking_image_format = self._speaking_image.format
         self._speaking_image_bytes = self._speaking_image.tobytes()
 
         self._waiting_image = Image.open(waiting_path)
-        self._waiting_image_format = self._waiting_image.format
         self._waiting_image_bytes = self._waiting_image.tobytes()
 
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        if not isinstance(frame, SystemFrame):
-            await self.push_frame(ImageRawFrame(image=self._speaking_image_bytes, size=(1024, 1024), format=self._speaking_image_format))
-            await self.push_frame(frame)
-            await self.push_frame(ImageRawFrame(image=self._waiting_image_bytes, size=(1024, 1024), format=self._waiting_image_format))
-        else:
-            await self.push_frame(frame)
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        yield ImageFrame(None, self._speaking_image_bytes)
+        yield frame
+        yield ImageFrame(None, self._waiting_image_bytes)
 
 
 async def main(room_url: str, token):
@@ -65,15 +46,13 @@ async def main(room_url: str, token):
             room_url,
             token,
             "Respond bot",
-            DailyParams(
-                audio_out_enabled=True,
-                camera_out_width=1024,
-                camera_out_height=1024,
-                transcription_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer()
-            )
+            5,
         )
+        transport._camera_enabled = True
+        transport._camera_width = 1024
+        transport._camera_height = 1024
+        transport._mic_enabled = True
+        transport._mic_sample_rate = 16000
 
         tts = ElevenLabsTTSService(
             aiohttp_session=session,
@@ -83,44 +62,59 @@ async def main(room_url: str, token):
 
         llm = OpenAILLMService(
             api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
+            model="gpt-4-turbo-preview")
 
-        messages = [
-            {
-                "role": "system",
-                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
-            },
-        ]
-
-        tma_in = LLMUserResponseAggregator(messages)
-        tma_out = LLMAssistantResponseAggregator(messages)
-
-        image_sync_aggregator = ImageSyncAggregator(
-            os.path.join(os.path.dirname(__file__), "assets", "speaking.png"),
-            os.path.join(os.path.dirname(__file__), "assets", "waiting.png"),
+        img = FalImageGenService(
+            image_size="1024x1024",
+            aiohttp_session=session,
+            key_id=os.getenv("FAL_KEY_ID"),
+            key_secret=os.getenv("FAL_KEY_SECRET"),
         )
 
-        pipeline = Pipeline([
-            transport.input(),
-            image_sync_aggregator,
-            tma_in,
-            llm,
-            tts,
-            transport.output(),
-            tma_out
-        ])
+        async def get_images():
+            get_speaking_task = asyncio.create_task(
+                img.run_image_gen("An image of a cat speaking")
+            )
+            get_waiting_task = asyncio.create_task(
+                img.run_image_gen("An image of a cat waiting")
+            )
 
-        task = PipelineTask(pipeline)
+            (speaking_data, waiting_data) = await asyncio.gather(
+                get_speaking_task, get_waiting_task
+            )
 
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            participant_name = participant["info"]["userName"] or ''
-            transport.capture_participant_transcription(participant["id"])
-            await task.queue_frames([TextFrame(f"Hi, this is {participant_name}.")])
+            return speaking_data, waiting_data
 
-        runner = PipelineRunner()
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            await tts.say("Hi, I'm listening!", transport.send_queue)
 
-        await runner.run(task)
+        async def handle_transcriptions():
+            messages = [
+                {
+                    "role": "system",
+                    "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
+                },
+            ]
+
+            tma_in = LLMUserContextAggregator(
+                messages, transport._my_participant_id)
+            tma_out = LLMAssistantContextAggregator(
+                messages, transport._my_participant_id
+            )
+            image_sync_aggregator = ImageSyncAggregator(
+                os.path.join(
+                    os.path.dirname(__file__), "assets", "speaking.png"), os.path.join(
+                    os.path.dirname(__file__), "assets", "waiting.png"), )
+            await tts.run_to_queue(
+                transport.send_queue,
+                image_sync_aggregator.run(
+                    tma_out.run(llm.run(tma_in.run(transport.get_receive_frames())))
+                ),
+            )
+
+        transport.transcription_settings["extra"]["punctuate"] = True
+        await asyncio.gather(transport.run(), handle_transcriptions())
 
 
 if __name__ == "__main__":

examples/foundational/07-interruptible.py

@@ -1,34 +1,26 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
 import asyncio
 import aiohttp
+import logging
 import os
-import sys
+from dailyai.pipeline.aggregators import (
+    LLMResponseAggregator,
+    UserResponseAggregator,
+)
 
-from pipecat.frames.frames import LLMMessagesFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantResponseAggregator, LLMUserResponseAggregator)
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.openai import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
+from dailyai.pipeline.pipeline import Pipeline
+from dailyai.services.ai_services import FrameLogger
+from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
 
 from runner import configure
 
-from loguru import logger
-
 from dotenv import load_dotenv
 load_dotenv(override=True)
 
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
 
 
 async def main(room_url: str, token):
@@ -37,12 +29,12 @@ async def main(room_url: str, token):
             room_url,
             token,
             "Respond bot",
-            DailyParams(
-                audio_out_enabled=True,
-                transcription_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer()
-            )
+            duration_minutes=5,
+            start_transcription=True,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            camera_enabled=False,
+            vad_enabled=True,
         )
 
         tts = ElevenLabsTTSService(
@@ -53,44 +45,30 @@ async def main(room_url: str, token):
 
         llm = OpenAILLMService(
             api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
+            model="gpt-4-turbo-preview")
 
-        messages = [
-            {
-                "role": "system",
-                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
-            },
-        ]
+        pipeline = Pipeline([FrameLogger(), llm, FrameLogger(), tts])
 
-        tma_in = LLMUserResponseAggregator(messages)
-        tma_out = LLMAssistantResponseAggregator(messages)
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            await transport.say("Hi, I'm listening!", tts)
 
-        pipeline = Pipeline([
-            transport.input(),   # Transport user input
-            tma_in,              # User responses
-            llm,                 # LLM
-            tts,                 # TTS
-            transport.output(),  # Transport bot output
-            tma_out              # Assistant spoken responses
-        ])
+        async def run_conversation():
+            messages = [
+                {
+                    "role": "system",
+                    "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
+                },
+            ]
 
-        task = PipelineTask(pipeline, PipelineParams(
-            allow_interruptions=True,
-            enable_metrics=True,
-            report_only_initial_ttfb=True,
-        ))
+            await transport.run_interruptible_pipeline(
+                pipeline,
+                post_processor=LLMResponseAggregator(messages),
+                pre_processor=UserResponseAggregator(messages),
+            )
 
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            transport.capture_participant_transcription(participant["id"])
-            # Kick off the conversation.
-            messages.append(
-                {"role": "system", "content": "Please introduce yourself to the user."})
-            await task.queue_frames([LLMMessagesFrame(messages)])
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
+        transport.transcription_settings["extra"]["punctuate"] = False
+        await asyncio.gather(transport.run(), run_conversation())
 
 
 if __name__ == "__main__":

examples/foundational/07a-interruptible-anthropic.py

@@ -1,95 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import aiohttp
-import os
-import sys
-
-from pipecat.frames.frames import LLMMessagesFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantResponseAggregator, LLMUserResponseAggregator)
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.anthropic import AnthropicLLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def main(room_url: str, token):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Respond bot",
-            DailyParams(
-                audio_out_enabled=True,
-                transcription_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer()
-            )
-        )
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
-        )
-
-        llm = AnthropicLLMService(
-            api_key=os.getenv("ANTHROPIC_API_KEY"),
-            model="claude-3-opus-20240229")
-
-        # todo: think more about how to handle system prompts in a more general way. OpenAI,
-        # Google, and Anthropic all have slightly different approaches to providing a system
-        # prompt.
-        messages = [
-            {
-                "role": "system",
-                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way. Say hello.",
-            },
-        ]
-
-        tma_in = LLMUserResponseAggregator(messages)
-        tma_out = LLMAssistantResponseAggregator(messages)
-
-        pipeline = Pipeline([
-            transport.input(),   # Transport user input
-            tma_in,              # User responses
-            llm,                 # LLM
-            tts,                 # TTS
-            transport.output(),  # Transport bot output
-            tma_out              # Assistant spoken responses
-        ])
-
-        task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            transport.capture_participant_transcription(participant["id"])
-            # Kick off the conversation.
-            await task.queue_frames([LLMMessagesFrame(messages)])
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/07b-interruptible-langchain.py

@@ -1,125 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import os
-import sys
-
-import aiohttp
-
-from pipecat.frames.frames import LLMMessagesFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantResponseAggregator, LLMUserResponseAggregator)
-from pipecat.processors.frameworks.langchain import LangchainProcessor
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain_community.chat_message_histories import ChatMessageHistory
-from langchain_core.chat_history import BaseChatMessageHistory
-from langchain_core.runnables.history import RunnableWithMessageHistory
-from langchain_openai import ChatOpenAI
-
-from loguru import logger
-
-from runner import configure
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-message_store = {}
-
-
-def get_session_history(session_id: str) -> BaseChatMessageHistory:
-    if session_id not in message_store:
-        message_store[session_id] = ChatMessageHistory()
-    return message_store[session_id]
-
-
-async def main(room_url: str, token):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Respond bot",
-            DailyParams(
-                audio_out_enabled=True,
-                transcription_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer(),
-            ),
-        )
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
-        )
-
-        prompt = ChatPromptTemplate.from_messages(
-            [
-                ("system",
-                 "Be nice and helpful. Answer very briefly and without special characters like `#` or `*`. "
-                 "Your response will be synthesized to voice and those characters will create unnatural sounds.",
-                 ),
-                MessagesPlaceholder("chat_history"),
-                ("human", "{input}"),
-            ])
-        chain = prompt | ChatOpenAI(model="gpt-4o", temperature=0.7)
-        history_chain = RunnableWithMessageHistory(
-            chain,
-            get_session_history,
-            history_messages_key="chat_history",
-            input_messages_key="input")
-        lc = LangchainProcessor(history_chain)
-
-        tma_in = LLMUserResponseAggregator()
-        tma_out = LLMAssistantResponseAggregator()
-
-        pipeline = Pipeline(
-            [
-                transport.input(),      # Transport user input
-                tma_in,                 # User responses
-                lc,                     # Langchain
-                tts,                    # TTS
-                transport.output(),     # Transport bot output
-                tma_out,                # Assistant spoken responses
-            ]
-        )
-
-        task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            transport.capture_participant_transcription(participant["id"])
-            lc.set_participant_id(participant["id"])
-            # Kick off the conversation.
-            # the `LLMMessagesFrame` will be picked up by the LangchainProcessor using
-            # only the content of the last message to inject it in the prompt defined
-            # above. So no role is required here.
-            messages = [(
-                {
-                    "content": "Please briefly introduce yourself to the user."
-                }
-            )]
-            await task.queue_frames([LLMMessagesFrame(messages)])
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/07c-interruptible-deepgram.py

@@ -1,97 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import aiohttp
-import os
-import sys
-
-from pipecat.frames.frames import LLMMessagesFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantResponseAggregator, LLMUserResponseAggregator)
-from pipecat.services.deepgram import DeepgramSTTService, DeepgramTTSService
-from pipecat.services.openai import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def main(room_url: str, token):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Respond bot",
-            DailyParams(
-                audio_out_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer(),
-                vad_audio_passthrough=True
-            )
-        )
-
-        stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
-
-        tts = DeepgramTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("DEEPGRAM_API_KEY"),
-            voice="aura-helios-en"
-        )
-
-        llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
-
-        messages = [
-            {
-                "role": "system",
-                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
-            },
-        ]
-
-        tma_in = LLMUserResponseAggregator(messages)
-        tma_out = LLMAssistantResponseAggregator(messages)
-
-        pipeline = Pipeline([
-            transport.input(),   # Transport user input
-            stt,                 # STT
-            tma_in,              # User responses
-            llm,                 # LLM
-            tts,                 # TTS
-            transport.output(),  # Transport bot output
-            tma_out              # Assistant spoken responses
-        ])
-
-        task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            transport.capture_participant_transcription(participant["id"])
-            # Kick off the conversation.
-            messages.append(
-                {"role": "system", "content": "Please introduce yourself to the user."})
-            await task.queue_frames([LLMMessagesFrame(messages)])
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/07d-interruptible-cartesia.py

@@ -1,94 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import os
-import sys
-
-from pipecat.frames.frames import LLMMessagesFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantResponseAggregator, LLMUserResponseAggregator)
-from pipecat.services.cartesia import CartesiaTTSService
-from pipecat.services.openai import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def main(room_url: str, token):
-    transport = DailyTransport(
-        room_url,
-        token,
-        "Respond bot",
-        DailyParams(
-            audio_out_enabled=True,
-            audio_out_sample_rate=44100,
-            transcription_enabled=True,
-            vad_enabled=True,
-            vad_analyzer=SileroVADAnalyzer()
-        )
-    )
-
-    tts = CartesiaTTSService(
-        api_key=os.getenv("CARTESIA_API_KEY"),
-        voice_name="British Lady",
-        output_format="pcm_44100"
-    )
-
-    llm = OpenAILLMService(
-        api_key=os.getenv("OPENAI_API_KEY"),
-        model="gpt-4o")
-
-    messages = [
-        {
-            "role": "system",
-            "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
-        },
-    ]
-
-    tma_in = LLMUserResponseAggregator(messages)
-    tma_out = LLMAssistantResponseAggregator(messages)
-
-    pipeline = Pipeline([
-        transport.input(),   # Transport user input
-        tma_in,              # User responses
-        llm,                 # LLM
-        tts,                 # TTS
-        transport.output(),  # Transport bot output
-        tma_out              # Assistant spoken responses
-    ])
-
-    task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))
-
-    @transport.event_handler("on_first_participant_joined")
-    async def on_first_participant_joined(transport, participant):
-        transport.capture_participant_transcription(participant["id"])
-        # Kick off the conversation.
-        messages.append(
-            {"role": "system", "content": "Please introduce yourself to the user."})
-        await task.queue_frames([LLMMessagesFrame(messages)])
-
-    runner = PipelineRunner()
-
-    await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/07e-interruptible-playht.py

@@ -1,93 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import os
-import sys
-
-from pipecat.frames.frames import LLMMessagesFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantResponseAggregator, LLMUserResponseAggregator)
-from pipecat.services.playht import PlayHTTTSService
-from pipecat.services.openai import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def main(room_url: str, token):
-    transport = DailyTransport(
-        room_url,
-        token,
-        "Respond bot",
-        DailyParams(
-            audio_out_enabled=True,
-            audio_out_sample_rate=16000,
-            transcription_enabled=True,
-            vad_enabled=True,
-            vad_analyzer=SileroVADAnalyzer()
-        )
-    )
-
-    tts = PlayHTTTSService(
-        user_id=os.getenv("PLAYHT_USER_ID"),
-        api_key=os.getenv("PLAYHT_API_KEY"),
-        voice_url="s3://voice-cloning-zero-shot/801a663f-efd0-4254-98d0-5c175514c3e8/jennifer/manifest.json",
-    )
-
-    llm = OpenAILLMService(
-        api_key=os.getenv("OPENAI_API_KEY"),
-        model="gpt-4o")
-
-    messages = [
-        {
-            "role": "system",
-            "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
-        },
-    ]
-
-    tma_in = LLMUserResponseAggregator(messages)
-    tma_out = LLMAssistantResponseAggregator(messages)
-
-    pipeline = Pipeline([
-        transport.input(),   # Transport user input
-        tma_in,              # User responses
-        llm,                 # LLM
-        tts,                 # TTS
-        transport.output(),  # Transport bot output
-        tma_out              # Assistant spoken responses
-    ])
-
-    task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))
-
-    @transport.event_handler("on_first_participant_joined")
-    async def on_first_participant_joined(transport, participant):
-        transport.capture_participant_transcription(participant["id"])
-        # Kick off the conversation.
-        messages.append(
-            {"role": "system", "content": "Please introduce yourself to the user."})
-        await task.queue_frames([LLMMessagesFrame(messages)])
-
-    runner = PipelineRunner()
-
-    await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/07f-interruptible-azure.py

@@ -1,100 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import os
-import sys
-
-from pipecat.frames.frames import LLMMessagesFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantResponseAggregator, LLMUserResponseAggregator)
-from pipecat.services.azure import AzureLLMService, AzureSTTService, AzureTTSService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def main(room_url: str, token):
-    transport = DailyTransport(
-        room_url,
-        token,
-        "Respond bot",
-        DailyParams(
-            audio_out_enabled=True,
-            audio_out_sample_rate=16000,
-            vad_enabled=True,
-            vad_analyzer=SileroVADAnalyzer(),
-            vad_audio_passthrough=True,
-        )
-    )
-
-    stt = AzureSTTService(
-        api_key=os.getenv("AZURE_SPEECH_API_KEY"),
-        region=os.getenv("AZURE_SPEECH_REGION"),
-    )
-
-    tts = AzureTTSService(
-        api_key=os.getenv("AZURE_SPEECH_API_KEY"),
-        region=os.getenv("AZURE_SPEECH_REGION"),
-    )
-
-    llm = AzureLLMService(
-        api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
-        endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
-        model=os.getenv("AZURE_CHATGPT_MODEL"),
-    )
-
-    messages = [
-        {
-            "role": "system",
-            "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
-        },
-    ]
-
-    tma_in = LLMUserResponseAggregator(messages)
-    tma_out = LLMAssistantResponseAggregator(messages)
-
-    pipeline = Pipeline([
-        transport.input(),   # Transport user input
-        stt,                 # STT
-        tma_in,              # User responses
-        llm,                 # LLM
-        tts,                 # TTS
-        transport.output(),  # Transport bot output
-        tma_out              # Assistant spoken responses
-    ])
-
-    task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))
-
-    @transport.event_handler("on_first_participant_joined")
-    async def on_first_participant_joined(transport, participant):
-        transport.capture_participant_transcription(participant["id"])
-        # Kick off the conversation.
-        messages.append(
-            {"role": "system", "content": "Please introduce yourself to the user."})
-        await task.queue_frames([LLMMessagesFrame(messages)])
-
-    runner = PipelineRunner()
-
-    await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/07g-interruptible-openai-tts.py

@@ -1,92 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import os
-import sys
-
-from pipecat.frames.frames import LLMMessagesFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantResponseAggregator, LLMUserResponseAggregator)
-from pipecat.services.openai import OpenAITTSService
-from pipecat.services.openai import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def main(room_url: str, token):
-    transport = DailyTransport(
-        room_url,
-        token,
-        "Respond bot",
-        DailyParams(
-            audio_out_enabled=True,
-            audio_out_sample_rate=24000,
-            transcription_enabled=True,
-            vad_enabled=True,
-            vad_analyzer=SileroVADAnalyzer()
-        )
-    )
-
-    tts = OpenAITTSService(
-        api_key=os.getenv("OPENAI_API_KEY"),
-        voice="alloy"
-    )
-
-    llm = OpenAILLMService(
-        api_key=os.getenv("OPENAI_API_KEY"),
-        model="gpt-4o")
-
-    messages = [
-        {
-            "role": "system",
-            "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
-        },
-    ]
-
-    tma_in = LLMUserResponseAggregator(messages)
-    tma_out = LLMAssistantResponseAggregator(messages)
-
-    pipeline = Pipeline([
-        transport.input(),   # Transport user input
-        tma_in,              # User responses
-        llm,                 # LLM
-        tts,                 # TTS
-        transport.output(),  # Transport bot output
-        tma_out              # Assistant spoken responses
-    ])
-
-    task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))
-
-    @transport.event_handler("on_first_participant_joined")
-    async def on_first_participant_joined(transport, participant):
-        transport.capture_participant_transcription(participant["id"])
-        # Kick off the conversation.
-        messages.append(
-            {"role": "system", "content": "Please introduce yourself to the user."})
-        await task.queue_frames([LLMMessagesFrame(messages)])
-
-    runner = PipelineRunner()
-
-    await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/07h-interruptible-openpipe.py

@@ -1,102 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import aiohttp
-import os
-import sys
-
-from pipecat.frames.frames import LLMMessagesFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantResponseAggregator,
-    LLMUserResponseAggregator,
-)
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.openpipe import OpenPipeLLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from runner import configure
-
-from loguru import logger
-import time
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def main(room_url: str, token):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Respond bot",
-            DailyParams(
-                audio_out_enabled=True,
-                transcription_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer()
-            )
-        )
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
-        )
-
-        timestamp = int(time.time())
-        llm = OpenPipeLLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
-            openpipe_api_key=os.getenv("OPENPIPE_API_KEY"),
-            model="gpt-4o",
-            tags={
-                "conversation_id": f"pipecat-{timestamp}"
-            }
-        )
-
-        messages = [
-            {
-                "role": "system",
-                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
-            },
-        ]
-        tma_in = LLMUserResponseAggregator(messages)
-        tma_out = LLMAssistantResponseAggregator(messages)
-
-        pipeline = Pipeline([
-            transport.input(),   # Transport user input
-            tma_in,              # User responses
-            llm,                 # LLM
-            tts,                 # TTS
-            transport.output(),  # Transport bot output
-            tma_out              # Assistant spoken responses
-        ])
-
-        task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True))
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            transport.capture_participant_transcription(participant["id"])
-            # Kick off the conversation.
-            messages.append(
-                {"role": "system", "content": "Please introduce yourself to the user."})
-            await task.queue_frames([LLMMessagesFrame(messages)])
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/08-bots-arguing.py

@@ -3,14 +3,14 @@ import aiohttp
 import asyncio
 import logging
 import os
-from pipecat.pipeline.aggregators import SentenceAggregator
-from pipecat.pipeline.pipeline import Pipeline
+from dailyai.pipeline.aggregators import SentenceAggregator
+from dailyai.pipeline.pipeline import Pipeline
 
-from pipecat.transports.daily_transport import DailyTransport
-from pipecat.services.azure_ai_services import AzureLLMService, AzureTTSService
-from pipecat.services.elevenlabs_ai_services import ElevenLabsTTSService
-from pipecat.services.fal_ai_services import FalImageGenService
-from pipecat.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame
+from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.fal_ai_services import FalImageGenService
+from dailyai.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame
 
 from runner import configure
 
@@ -18,7 +18,7 @@ from dotenv import load_dotenv
 load_dotenv(override=True)
 
 logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
-logger = logging.getLogger("pipecat")
+logger = logging.getLogger("dailyai")
 logger.setLevel(logging.DEBUG)
 
 
@@ -51,11 +51,10 @@ async def main(room_url: str):
             voice_id="jBpfuIE2acCO8z3wKNLl",
         )
         dalle = FalImageGenService(
-            params=FalImageGenService.InputParams(
-                image_size="1024x1024"
-            ),
+            image_size="1024x1024",
             aiohttp_session=session,
-            key=os.getenv("FAL_KEY"),
+            key_id=os.getenv("FAL_KEY_ID"),
+            key_secret=os.getenv("FAL_KEY_SECRET"),
         )
 
         bot1_messages = [
@@ -92,7 +91,7 @@ async def main(room_url: str):
                 if isinstance(frame, TextFrame):
                     message += frame.text
                 elif isinstance(frame, AudioFrame):
-                    all_audio.extend(frame.audio)
+                    all_audio.extend(frame.data)
 
             return (message, all_audio)
 
@@ -123,7 +122,7 @@ async def main(room_url: str):
                 )
                 await transport.send_queue.put(
                     [
-                        ImageFrame(image_data1[1], image_data1[2]),
+                        ImageFrame(None, image_data1[1]),
                         AudioFrame(audio1),
                     ]
                 )
@@ -135,7 +134,7 @@ async def main(room_url: str):
                 )
                 await transport.send_queue.put(
                     [
-                        ImageFrame(image_data2[1], image_data2[2]),
+                        ImageFrame(None, image_data2[1]),
                         AudioFrame(audio2),
                     ]
                 )

examples/foundational/09-mirror.py

@@ -1,54 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import sys
-
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.transports.services.daily import DailyTransport, DailyParams
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def main(room_url, token):
-    transport = DailyTransport(
-        room_url, token, "Test",
-        DailyParams(
-            audio_in_enabled=True,
-            audio_out_enabled=True,
-            camera_out_enabled=True,
-            camera_out_is_live=True,
-            camera_out_width=1280,
-            camera_out_height=720
-        )
-    )
-
-    @transport.event_handler("on_first_participant_joined")
-    async def on_first_participant_joined(transport, participant):
-        transport.capture_participant_video(participant["id"])
-
-    pipeline = Pipeline([transport.input(), transport.output()])
-
-    runner = PipelineRunner()
-
-    task = PipelineTask(pipeline)
-
-    await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/09a-local-mirror.py

@@ -1,66 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import sys
-
-import tkinter as tk
-
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.transports.base_transport import TransportParams
-from pipecat.transports.local.tk import TkLocalTransport
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def main(room_url, token):
-    tk_root = tk.Tk()
-    tk_root.title("Local Mirror")
-
-    daily_transport = DailyTransport(room_url, token, "Test", DailyParams(audio_in_enabled=True))
-
-    tk_transport = TkLocalTransport(
-        tk_root,
-        TransportParams(
-            audio_out_enabled=True,
-            camera_out_enabled=True,
-            camera_out_is_live=True,
-            camera_out_width=1280,
-            camera_out_height=720))
-
-    @daily_transport.event_handler("on_first_participant_joined")
-    async def on_first_participant_joined(transport, participant):
-        transport.capture_participant_video(participant["id"])
-
-    pipeline = Pipeline([daily_transport.input(), tk_transport.output()])
-
-    task = PipelineTask(pipeline)
-
-    async def run_tk():
-        while not task.has_finished():
-            tk_root.update()
-            tk_root.update_idletasks()
-            await asyncio.sleep(0.1)
-
-    runner = PipelineRunner()
-
-    await asyncio.gather(runner.run(task), run_tk())
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/10-wake-phrase.py

@@ -1,94 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import aiohttp
-import os
-import sys
-
-from pipecat.processors.filters.wake_check_filter import WakeCheckFilter
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantResponseAggregator, LLMUserResponseAggregator)
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.openai import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def main(room_url: str, token):
-
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Robot",
-            DailyParams(
-                audio_out_enabled=True,
-                transcription_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer()
-            )
-        )
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
-        )
-
-        llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
-
-        messages = [
-            {
-                "role": "system",
-                "content": "You are a helpful assistant. Respond to what the user said in a creative and helpful way. Keep your responses brief.",
-            },
-        ]
-
-        hey_robot_filter = WakeCheckFilter(["hey robot", "hey, robot"])
-        tma_in = LLMUserResponseAggregator(messages)
-        tma_out = LLMAssistantResponseAggregator(messages)
-
-        pipeline = Pipeline([
-            transport.input(),   # Transport user input
-            hey_robot_filter,    # Filter out speech not directed at the robot
-            tma_in,              # User responses
-            llm,                 # LLM
-            tts,                 # TTS
-            transport.output(),  # Transport bot output
-            tma_out              # Assistant spoken responses
-        ])
-
-        task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            transport.capture_participant_transcription(participant["id"])
-            await tts.say("Hi! If you want to talk to me, just say 'Hey Robot'.")
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/10-wake-word.py

@@ -0,0 +1,191 @@
+import aiohttp
+import asyncio
+import logging
+import os
+import random
+from typing import AsyncGenerator
+from PIL import Image
+
+from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.pipeline.aggregators import (
+    LLMUserContextAggregator,
+    LLMAssistantContextAggregator,
+)
+from dailyai.pipeline.frames import (
+    Frame,
+    TextFrame,
+    ImageFrame,
+    SpriteFrame,
+    TranscriptionFrame,
+)
+from dailyai.services.ai_services import AIService
+
+from runner import configure
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+sprites = {}
+image_files = [
+    "sc-default.png",
+    "sc-talk.png",
+    "sc-listen-1.png",
+    "sc-think-1.png",
+    "sc-think-2.png",
+    "sc-think-3.png",
+    "sc-think-4.png",
+]
+
+script_dir = os.path.dirname(__file__)
+
+for file in image_files:
+    # Build the full path to the image file
+    full_path = os.path.join(script_dir, "assets", file)
+    # Get the filename without the extension to use as the dictionary key
+    filename = os.path.splitext(os.path.basename(full_path))[0]
+    # Open the image and convert it to bytes
+    with Image.open(full_path) as img:
+        sprites[file] = img.tobytes()
+
+# When the bot isn't talking, show a static image of the cat listening
+quiet_frame = ImageFrame("", sprites["sc-listen-1.png"])
+# When the bot is talking, build an animation from two sprites
+talking_list = [sprites["sc-default.png"], sprites["sc-talk.png"]]
+talking = [random.choice(talking_list) for x in range(30)]
+talking_frame = SpriteFrame(images=talking)
+
+# TODO: Support "thinking" as soon as we get a valid transcript, while LLM
+# is processing
+thinking_list = [
+    sprites["sc-think-1.png"],
+    sprites["sc-think-2.png"],
+    sprites["sc-think-3.png"],
+    sprites["sc-think-4.png"],
+]
+thinking_frame = SpriteFrame(images=thinking_list)
+
+
+class TranscriptFilter(AIService):
+    def __init__(self, bot_participant_id=None):
+        self.bot_participant_id = bot_participant_id
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, TranscriptionFrame):
+            if frame.participantId != self.bot_participant_id:
+                yield frame
+
+
+class NameCheckFilter(AIService):
+    def __init__(self, names: list[str]):
+        self.names = names
+        self.sentence = ""
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        content: str = ""
+
+        # TODO: split up transcription by participant
+        if isinstance(frame, TextFrame):
+            content = frame.text
+
+        self.sentence += content
+        if self.sentence.endswith((".", "?", "!")):
+            if any(name in self.sentence for name in self.names):
+                out = self.sentence
+                self.sentence = ""
+                yield TextFrame(out)
+            else:
+                out = self.sentence
+                self.sentence = ""
+
+
+class ImageSyncAggregator(AIService):
+    def __init__(self):
+        pass
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        yield talking_frame
+        yield frame
+        yield quiet_frame
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransport(
+            room_url,
+            token,
+            "Santa Cat",
+            duration_minutes=3,
+            start_transcription=True,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            camera_enabled=True,
+            camera_width=720,
+            camera_height=1280,
+        )
+        transport._mic_enabled = True
+        transport._mic_sample_rate = 16000
+        transport._camera_enabled = True
+        transport._camera_width = 720
+        transport._camera_height = 1280
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id="jBpfuIE2acCO8z3wKNLl",
+        )
+        isa = ImageSyncAggregator()
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            await tts.say(
+                "Hi! If you want to talk to me, just say 'hey Santa Cat'.",
+                transport.send_queue,
+            )
+
+        async def handle_transcriptions():
+            messages = [
+                {
+                    "role": "system",
+                    "content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while.  Your responses should only be a few sentences long.",
+                },
+            ]
+
+            tma_in = LLMUserContextAggregator(
+                messages, transport._my_participant_id)
+            tma_out = LLMAssistantContextAggregator(
+                messages, transport._my_participant_id
+            )
+            tf = TranscriptFilter(transport._my_participant_id)
+            ncf = NameCheckFilter(["Santa Cat", "Santa"])
+            await tts.run_to_queue(
+                transport.send_queue,
+                isa.run(
+                    tma_out.run(
+                        llm.run(
+                            tma_in.run(
+                                ncf.run(tf.run(transport.get_receive_frames())))
+                        )
+                    )
+                ),
+            )
+
+        async def starting_image():
+            await transport.send_queue.put(quiet_frame)
+
+        transport.transcription_settings["extra"]["punctuate"] = True
+        await asyncio.gather(transport.run(), handle_transcriptions(), starting_image())
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/foundational/11-sound-effects.py

@@ -1,45 +1,33 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
 import aiohttp
 import asyncio
+import logging
 import os
-import sys
 import wave
 
-from pipecat.frames.frames import (
+from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.pipeline.aggregators import (
+    LLMUserContextAggregator,
+    LLMAssistantContextAggregator,
+)
+from dailyai.services.ai_services import AIService, FrameLogger
+from dailyai.pipeline.frames import (
     Frame,
-    AudioRawFrame,
-    LLMFullResponseEndFrame,
+    AudioFrame,
+    LLMResponseEndFrame,
     LLMMessagesFrame,
 )
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMUserResponseAggregator,
-    LLMAssistantResponseAggregator,
-)
-from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.processors.logger import FrameLogger
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.openai import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
+from typing import AsyncGenerator
 
 from runner import configure
 
-from loguru import logger
-
 from dotenv import load_dotenv
 load_dotenv(override=True)
 
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
 
 sounds = {}
 sound_files = ["ding1.wav", "ding2.wav"]
@@ -53,34 +41,33 @@ for file in sound_files:
     filename = os.path.splitext(os.path.basename(full_path))[0]
     # Open the image and convert it to bytes
     with wave.open(full_path) as audio_file:
-        sounds[file] = AudioRawFrame(audio_file.readframes(-1),
-                                     audio_file.getframerate(), audio_file.getnchannels())
+        sounds[file] = audio_file.readframes(-1)
 
 
-class OutboundSoundEffectWrapper(FrameProcessor):
+class OutboundSoundEffectWrapper(AIService):
+    def __init__(self):
+        pass
 
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        if isinstance(frame, LLMFullResponseEndFrame):
-            await self.push_frame(sounds["ding1.wav"])
-            # In case anything else downstream needs it
-            await self.push_frame(frame, direction)
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, LLMResponseEndFrame):
+            yield AudioFrame(sounds["ding1.wav"])
+            # In case anything else up the stack needs it
+            yield frame
         else:
-            await self.push_frame(frame, direction)
+            yield frame
 
 
-class InboundSoundEffectWrapper(FrameProcessor):
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
+class InboundSoundEffectWrapper(AIService):
+    def __init__(self):
+        pass
 
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
         if isinstance(frame, LLMMessagesFrame):
-            await self.push_frame(sounds["ding2.wav"])
-            # In case anything else downstream needs it
-            await self.push_frame(frame, direction)
+            yield AudioFrame(sounds["ding2.wav"])
+            # In case anything else up the stack needs it
+            yield frame
         else:
-            await self.push_frame(frame, direction)
+            yield frame
 
 
 async def main(room_url: str, token):
@@ -89,17 +76,15 @@ async def main(room_url: str, token):
             room_url,
             token,
             "Respond bot",
-            DailyParams(
-                audio_out_enabled=True,
-                transcription_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer()
-            )
+            duration_minutes=5,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            camera_enabled=False,
         )
 
         llm = OpenAILLMService(
             api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
+            model="gpt-4-turbo-preview")
 
         tts = ElevenLabsTTSService(
             aiohttp_session=session,
@@ -107,44 +92,47 @@ async def main(room_url: str, token):
             voice_id="ErXwobaYiN019PkySvjV",
         )
 
-        messages = [
-            {
-                "role": "system",
-                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
-            },
-        ]
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            await tts.say("Hi, I'm listening!", transport.send_queue)
+            await transport.send_queue.put(AudioFrame(sounds["ding1.wav"]))
 
-        tma_in = LLMUserResponseAggregator(messages)
-        tma_out = LLMAssistantResponseAggregator(messages)
-        out_sound = OutboundSoundEffectWrapper()
-        in_sound = InboundSoundEffectWrapper()
-        fl = FrameLogger("LLM Out")
-        fl2 = FrameLogger("Transcription In")
+        async def handle_transcriptions():
+            messages = [
+                {
+                    "role": "system",
+                    "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
+                },
+            ]
 
-        pipeline = Pipeline([
-            transport.input(),
-            tma_in,
-            in_sound,
-            fl2,
-            llm,
-            fl,
-            tts,
-            out_sound,
-            transport.output(),
-            tma_out
-        ])
+            tma_in = LLMUserContextAggregator(
+                messages, transport._my_participant_id)
+            tma_out = LLMAssistantContextAggregator(
+                messages, transport._my_participant_id
+            )
+            out_sound = OutboundSoundEffectWrapper()
+            in_sound = InboundSoundEffectWrapper()
+            fl = FrameLogger("LLM Out")
+            fl2 = FrameLogger("Transcription In")
+            await out_sound.run_to_queue(
+                transport.send_queue,
+                tts.run(
+                    fl.run(
+                        tma_out.run(
+                            llm.run(
+                                fl2.run(
+                                    in_sound.run(
+                                        tma_in.run(transport.get_receive_frames())
+                                    )
+                                )
+                            )
+                        )
+                    )
+                ),
+            )
 
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            transport.capture_participant_transcription(participant["id"])
-            await tts.say("Hi, I'm listening!")
-            await transport.send_audio(sounds["ding1.wav"])
-
-        runner = PipelineRunner()
-
-        task = PipelineTask(pipeline)
-
-        await runner.run(task)
+        transport.transcription_settings["extra"]["punctuate"] = True
+        await asyncio.gather(transport.run(), handle_transcriptions())
 
 
 if __name__ == "__main__":

examples/foundational/12-describe-video.py

@@ -1,112 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import aiohttp
-import os
-import sys
-
-from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.processors.aggregators.user_response import UserResponseAggregator
-from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
-from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.moondream import MoondreamService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-class UserImageRequester(FrameProcessor):
-
-    def __init__(self, participant_id: str | None = None):
-        super().__init__()
-        self._participant_id = participant_id
-
-    def set_participant_id(self, participant_id: str):
-        self._participant_id = participant_id
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        if self._participant_id and isinstance(frame, TextFrame):
-            await self.push_frame(UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM)
-        await self.push_frame(frame, direction)
-
-
-async def main(room_url: str, token):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Describe participant video",
-            DailyParams(
-                audio_out_enabled=True,
-                transcription_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer()
-            )
-        )
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
-        )
-
-        user_response = UserResponseAggregator()
-
-        image_requester = UserImageRequester()
-
-        vision_aggregator = VisionImageFrameAggregator()
-
-        # If you run into weird description, try with use_cpu=True
-        moondream = MoondreamService()
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
-        )
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            await tts.say("Hi there! Feel free to ask me what I see.")
-            transport.capture_participant_video(participant["id"], framerate=0)
-            transport.capture_participant_transcription(participant["id"])
-            image_requester.set_participant_id(participant["id"])
-
-        pipeline = Pipeline([
-            transport.input(),
-            user_response,
-            image_requester,
-            vision_aggregator,
-            moondream,
-            tts,
-            transport.output()
-        ])
-
-        task = PipelineTask(pipeline)
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/12a-describe-video-gemini-flash.py

@@ -1,108 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import aiohttp
-import os
-import sys
-
-from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.processors.aggregators.user_response import UserResponseAggregator
-from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
-from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.google import GoogleLLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-class UserImageRequester(FrameProcessor):
-
-    def __init__(self, participant_id: str | None = None):
-        super().__init__()
-        self._participant_id = participant_id
-
-    def set_participant_id(self, participant_id: str):
-        self._participant_id = participant_id
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        if self._participant_id and isinstance(frame, TextFrame):
-            await self.push_frame(UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM)
-        await self.push_frame(frame, direction)
-
-
-async def main(room_url: str, token):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Describe participant video",
-            DailyParams(
-                audio_in_enabled=True,  # This is so Silero VAD can get audio data
-                audio_out_enabled=True,
-                transcription_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer()
-            )
-        )
-
-        user_response = UserResponseAggregator()
-
-        image_requester = UserImageRequester()
-
-        vision_aggregator = VisionImageFrameAggregator()
-
-        google = GoogleLLMService(
-            model="gemini-1.5-flash-latest",
-            api_key=os.getenv("GOOGLE_API_KEY"))
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
-        )
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            await tts.say("Hi there! Feel free to ask me what I see.")
-            transport.capture_participant_video(participant["id"], framerate=0)
-            transport.capture_participant_transcription(participant["id"])
-            image_requester.set_participant_id(participant["id"])
-
-        pipeline = Pipeline([
-            transport.input(),
-            user_response,
-            image_requester,
-            vision_aggregator,
-            google,
-            tts,
-            transport.output()
-        ])
-
-        task = PipelineTask(pipeline)
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/12b-describe-video-gpt-4o.py

@@ -1,108 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import aiohttp
-import os
-import sys
-
-from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.processors.aggregators.user_response import UserResponseAggregator
-from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
-from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.openai import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-class UserImageRequester(FrameProcessor):
-
-    def __init__(self, participant_id: str | None = None):
-        super().__init__()
-        self._participant_id = participant_id
-
-    def set_participant_id(self, participant_id: str):
-        self._participant_id = participant_id
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        if self._participant_id and isinstance(frame, TextFrame):
-            await self.push_frame(UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM)
-        await self.push_frame(frame, direction)
-
-
-async def main(room_url: str, token):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Describe participant video",
-            DailyParams(
-                audio_out_enabled=True,
-                transcription_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer()
-            )
-        )
-
-        user_response = UserResponseAggregator()
-
-        image_requester = UserImageRequester()
-
-        vision_aggregator = VisionImageFrameAggregator()
-
-        openai = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o"
-        )
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
-        )
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            await tts.say("Hi there! Feel free to ask me what I see.")
-            transport.capture_participant_video(participant["id"], framerate=0)
-            transport.capture_participant_transcription(participant["id"])
-            image_requester.set_participant_id(participant["id"])
-
-        pipeline = Pipeline([
-            transport.input(),
-            user_response,
-            image_requester,
-            vision_aggregator,
-            openai,
-            tts,
-            transport.output()
-        ])
-
-        task = PipelineTask(pipeline)
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/12c-describe-video-anthropic.py

@@ -1,108 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import aiohttp
-import os
-import sys
-
-from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.processors.aggregators.user_response import UserResponseAggregator
-from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
-from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.anthropic import AnthropicLLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-class UserImageRequester(FrameProcessor):
-
-    def __init__(self, participant_id: str | None = None):
-        super().__init__()
-        self._participant_id = participant_id
-
-    def set_participant_id(self, participant_id: str):
-        self._participant_id = participant_id
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        if self._participant_id and isinstance(frame, TextFrame):
-            await self.push_frame(UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM)
-        await self.push_frame(frame, direction)
-
-
-async def main(room_url: str, token):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Describe participant video",
-            DailyParams(
-                audio_out_enabled=True,
-                transcription_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer()
-            )
-        )
-
-        user_response = UserResponseAggregator()
-
-        image_requester = UserImageRequester()
-
-        vision_aggregator = VisionImageFrameAggregator()
-
-        anthropic = AnthropicLLMService(
-            api_key=os.getenv("ANTHROPIC_API_KEY"),
-            model="claude-3-sonnet-20240229"
-        )
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
-        )
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            await tts.say("Hi there! Feel free to ask me what I see.")
-            transport.capture_participant_video(participant["id"], framerate=0)
-            transport.capture_participant_transcription(participant["id"])
-            image_requester.set_participant_id(participant["id"])
-
-        pipeline = Pipeline([
-            transport.input(),
-            user_response,
-            image_requester,
-            vision_aggregator,
-            anthropic,
-            tts,
-            transport.output()
-        ])
-
-        task = PipelineTask(pipeline)
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/13-whisper-transcription.py

@@ -1,55 +1,45 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
 import asyncio
-import sys
+import logging
 
-from pipecat.frames.frames import Frame, TranscriptionFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.services.whisper import WhisperSTTService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
+from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.whisper_ai_services import WhisperSTTService
 
 from runner import configure
 
-from loguru import logger
-
 from dotenv import load_dotenv
 load_dotenv(override=True)
 
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-class TranscriptionLogger(FrameProcessor):
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        if isinstance(frame, TranscriptionFrame):
-            print(f"Transcription: {frame.text}")
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
 
 
 async def main(room_url: str):
-    transport = DailyTransport(room_url, None, "Transcription bot",
-                               DailyParams(audio_in_enabled=True))
+    transport = DailyTransport(
+        room_url,
+        None,
+        "Transcription bot",
+        start_transcription=True,
+        mic_enabled=False,
+        camera_enabled=False,
+        speaker_enabled=True,
+    )
 
     stt = WhisperSTTService()
+    transcription_output_queue = asyncio.Queue()
 
-    tl = TranscriptionLogger()
+    async def handle_transcription():
+        print("`````````TRANSCRIPTION`````````")
+        while True:
+            item = await transcription_output_queue.get()
+            print(item.text)
 
-    pipeline = Pipeline([transport.input(), stt, tl])
+    async def handle_speaker():
+        await stt.run_to_queue(
+            transcription_output_queue, transport.get_receive_frames()
+        )
 
-    task = PipelineTask(pipeline)
-
-    runner = PipelineRunner()
-
-    await runner.run(task)
+    await asyncio.gather(transport.run(), handle_speaker(), handle_transcription())
 
 
 if __name__ == "__main__":

examples/foundational/13a-whisper-local.py

@@ -1,54 +1,66 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
+import argparse
 import asyncio
-import sys
+import logging
+from dailyai.pipeline.frames import EndFrame, TranscriptionFrame
 
-from pipecat.frames.frames import Frame, TranscriptionFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.services.whisper import WhisperSTTService
-from pipecat.transports.base_transport import TransportParams
-from pipecat.transports.local.audio import LocalAudioTransport
+from dailyai.transports.local_transport import LocalTransport
+from dailyai.services.whisper_ai_services import WhisperSTTService
 
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
 
 
-class TranscriptionLogger(FrameProcessor):
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        if isinstance(frame, TranscriptionFrame):
-            print(f"Transcription: {frame.text}")
-
-
-async def main():
-    transport = LocalAudioTransport(TransportParams(audio_in_enabled=True))
+async def main(room_url: str):
+    global transport
+    global stt
 
+    meeting_duration_minutes = 1
+    transport = LocalTransport(
+        mic_enabled=True,
+        camera_enabled=False,
+        speaker_enabled=True,
+        duration_minutes=meeting_duration_minutes,
+        start_transcription=True,
+    )
     stt = WhisperSTTService()
+    transcription_output_queue = asyncio.Queue()
+    transport_done = asyncio.Event()
 
-    tl = TranscriptionLogger()
+    async def handle_transcription():
+        print("`````````TRANSCRIPTION`````````")
+        while not transport_done.is_set():
+            item = await transcription_output_queue.get()
+            print("got item from queue", item)
+            if isinstance(item, TranscriptionFrame):
+                print(item.text)
+            elif isinstance(item, EndFrame):
+                break
+        print("handle_transcription done")
 
-    pipeline = Pipeline([transport.input(), stt, tl])
+    async def handle_speaker():
+        await stt.run_to_queue(
+            transcription_output_queue, transport.get_receive_frames()
+        )
+        await transcription_output_queue.put(EndFrame())
+        print("handle speaker done.")
 
-    task = PipelineTask(pipeline)
+    async def run_until_done():
+        await transport.run()
+        transport_done.set()
+        print("run_until_done done")
 
-    runner = PipelineRunner()
-
-    await runner.run(task)
+    await asyncio.gather(run_until_done(), handle_speaker(), handle_transcription())
 
 
 if __name__ == "__main__":
-    asyncio.run(main())
+    parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
+    parser.add_argument(
+        "-u",
+        "--url",
+        type=str,
+        required=True,
+        help="URL of the Daily room to join")
+
+    args, unknown = parser.parse_known_args()
+    asyncio.run(main(args.url))

examples/foundational/13b-deepgram-transcription.py

@@ -1,58 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import os
-import sys
-
-from pipecat.frames.frames import Frame, TranscriptionFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.services.deepgram import DeepgramSTTService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-class TranscriptionLogger(FrameProcessor):
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        if isinstance(frame, TranscriptionFrame):
-            print(f"Transcription: {frame.text}")
-
-
-async def main(room_url: str):
-    transport = DailyTransport(room_url, None, "Transcription bot",
-                               DailyParams(audio_in_enabled=True))
-
-    stt = DeepgramSTTService(os.getenv("DEEPGRAM_API_KEY"))
-
-    tl = TranscriptionLogger()
-
-    pipeline = Pipeline([transport.input(), stt, tl])
-
-    task = PipelineTask(pipeline)
-
-    runner = PipelineRunner()
-
-    await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url))

examples/foundational/14-function-calling.py

@@ -1,140 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import aiohttp
-import os
-import sys
-
-from pipecat.frames.frames import TextFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantContextAggregator,
-    LLMUserContextAggregator,
-)
-from pipecat.processors.logger import FrameLogger
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.openai import OpenAILLMContext, OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from openai.types.chat import ChatCompletionToolParam
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def start_fetch_weather(llm):
-    await llm.push_frame(TextFrame("Let me think."))
-
-
-async def fetch_weather_from_api(llm, args):
-    return {"conditions": "nice", "temperature": "75"}
-
-
-async def main(room_url: str, token):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Respond bot",
-            DailyParams(
-                audio_out_enabled=True,
-                transcription_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer()
-            )
-        )
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
-        )
-
-        llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
-        llm.register_function(
-            "get_current_weather",
-            fetch_weather_from_api,
-            start_callback=start_fetch_weather)
-
-        fl_in = FrameLogger("Inner")
-        fl_out = FrameLogger("Outer")
-
-        tools = [
-            ChatCompletionToolParam(
-                type="function",
-                function={
-                    "name": "get_current_weather",
-                    "description": "Get the current weather",
-                    "parameters": {
-                        "type": "object",
-                        "properties": {
-                            "location": {
-                                "type": "string",
-                                "description": "The city and state, e.g. San Francisco, CA",
-                            },
-                            "format": {
-                                "type": "string",
-                                "enum": [
-                                    "celsius",
-                                    "fahrenheit"],
-                                "description": "The temperature unit to use. Infer this from the users location.",
-                            },
-                        },
-                        "required": [
-                            "location",
-                            "format"],
-                    },
-                })]
-        messages = [
-            {
-                "role": "system",
-                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
-            },
-        ]
-
-        context = OpenAILLMContext(messages, tools)
-        tma_in = LLMUserContextAggregator(context)
-        tma_out = LLMAssistantContextAggregator(context)
-        pipeline = Pipeline([
-            fl_in,
-            transport.input(),
-            tma_in,
-            llm,
-            fl_out,
-            tts,
-            transport.output(),
-            tma_out
-        ])
-
-        task = PipelineTask(pipeline)
-
-        @ transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            transport.capture_participant_transcription(participant["id"])
-            # Kick off the conversation.
-            await tts.say("Hi! Ask me about the weather in San Francisco.")
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/15-switch-voices.py

@@ -1,159 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import aiohttp
-import os
-import sys
-
-from pipecat.frames.frames import LLMMessagesFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.parallel_pipeline import ParallelPipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantContextAggregator,
-    LLMUserContextAggregator
-)
-from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
-from pipecat.processors.filters.function_filter import FunctionFilter
-from pipecat.services.cartesia import CartesiaTTSService
-from pipecat.services.openai import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from openai.types.chat import ChatCompletionToolParam
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-current_voice = "News Lady"
-
-
-async def switch_voice(llm, args):
-    global current_voice
-    current_voice = args["voice"]
-    return {"voice": f"You are now using your {current_voice} voice. Your responses should now be as if you were a {current_voice}."}
-
-
-async def news_lady_filter(frame) -> bool:
-    return current_voice == "News Lady"
-
-
-async def british_lady_filter(frame) -> bool:
-    return current_voice == "British Lady"
-
-
-async def barbershop_man_filter(frame) -> bool:
-    return current_voice == "Barbershop Man"
-
-
-async def main(room_url: str, token):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Pipecat",
-            DailyParams(
-                audio_out_enabled=True,
-                audio_out_sample_rate=44100,
-                transcription_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer()
-            )
-        )
-
-        news_lady = CartesiaTTSService(
-            api_key=os.getenv("CARTESIA_API_KEY"),
-            voice_name="Newslady",
-            output_format="pcm_44100"
-        )
-
-        british_lady = CartesiaTTSService(
-            api_key=os.getenv("CARTESIA_API_KEY"),
-            voice_name="British Lady",
-            output_format="pcm_44100"
-        )
-
-        barbershop_man = CartesiaTTSService(
-            api_key=os.getenv("CARTESIA_API_KEY"),
-            voice_name="Barbershop Man",
-            output_format="pcm_44100"
-        )
-
-        llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
-        llm.register_function("switch_voice", switch_voice)
-
-        tools = [
-            ChatCompletionToolParam(
-                type="function",
-                function={
-                    "name": "switch_voice",
-                    "description": "Switch your voice only when the user asks you to",
-                    "parameters": {
-                        "type": "object",
-                        "properties": {
-                            "voice": {
-                                "type": "string",
-                                "description": "The voice the user wants you to use",
-                            },
-                        },
-                        "required": ["voice"],
-                    },
-                })]
-        messages = [
-            {
-                "role": "system",
-                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities. Respond to what the user said in a creative and helpful way. Your output should not include non-alphanumeric characters. You can do the following voices: 'News Lady', 'British Lady' and 'Barbershop Man'.",
-            },
-        ]
-
-        context = OpenAILLMContext(messages, tools)
-        tma_in = LLMUserContextAggregator(context)
-        tma_out = LLMAssistantContextAggregator(context)
-
-        pipeline = Pipeline([
-            transport.input(),   # Transport user input
-            tma_in,              # User responses
-            llm,                 # LLM
-            ParallelPipeline(    # TTS (one of the following vocies)
-                [FunctionFilter(news_lady_filter), news_lady],            # News Lady voice
-                [FunctionFilter(british_lady_filter), british_lady],      # British Lady voice
-                [FunctionFilter(barbershop_man_filter), barbershop_man],  # Barbershop Man voice
-            ),
-            transport.output(),  # Transport bot output
-            tma_out              # Assistant spoken responses
-        ])
-
-        task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            transport.capture_participant_transcription(participant["id"])
-            # Kick off the conversation.
-            messages.append(
-                {
-                    "role": "system",
-                    "content": f"Please introduce yourself to the user and let them know the voices you can do. Your initial responses should be as if you were a {current_voice}."})
-            await task.queue_frames([LLMMessagesFrame(messages)])
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/15a-switch-languages.py

@@ -1,153 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import aiohttp
-import os
-import sys
-
-from pipecat.frames.frames import LLMMessagesFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.parallel_pipeline import ParallelPipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantContextAggregator,
-    LLMUserContextAggregator
-)
-from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
-from pipecat.processors.filters.function_filter import FunctionFilter
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.openai import OpenAILLMService
-from pipecat.services.whisper import Model, WhisperSTTService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from openai.types.chat import ChatCompletionToolParam
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-current_language = "English"
-
-
-async def switch_language(llm, args):
-    global current_language
-    current_language = args["language"]
-    return {"voice": f"Your answers from now on should be in {current_language}."}
-
-
-async def english_filter(frame) -> bool:
-    return current_language == "English"
-
-
-async def spanish_filter(frame) -> bool:
-    return current_language == "Spanish"
-
-
-async def main(room_url: str, token):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Pipecat",
-            DailyParams(
-                audio_in_enabled=True,
-                audio_out_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer(),
-                vad_audio_passthrough=True
-            )
-        )
-
-        stt = WhisperSTTService(model=Model.LARGE)
-
-        english_tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id="pNInz6obpgDQGcFmaJgB",
-        )
-
-        spanish_tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            model="eleven_multilingual_v2",
-            voice_id="9F4C8ztpNUmXkdDDbz3J",
-        )
-
-        llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
-        llm.register_function("switch_language", switch_language)
-
-        tools = [
-            ChatCompletionToolParam(
-                type="function",
-                function={
-                    "name": "switch_language",
-                    "description": "Switch to another language when the user asks you to",
-                    "parameters": {
-                        "type": "object",
-                        "properties": {
-                            "language": {
-                                "type": "string",
-                                "description": "The language the user wants you to speak",
-                            },
-                        },
-                        "required": ["language"],
-                    },
-                })]
-        messages = [
-            {
-                "role": "system",
-                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities. Respond to what the user said in a creative and helpful way. Your output should not include non-alphanumeric characters. You can speak the following languages: 'English' and 'Spanish'.",
-            },
-        ]
-
-        context = OpenAILLMContext(messages, tools)
-        tma_in = LLMUserContextAggregator(context)
-        tma_out = LLMAssistantContextAggregator(context)
-
-        pipeline = Pipeline([
-            transport.input(),   # Transport user input
-            stt,                 # STT
-            tma_in,              # User responses
-            llm,                 # LLM
-            ParallelPipeline(    # TTS (bot will speak the chosen language)
-                [FunctionFilter(english_filter), english_tts],  # English
-                [FunctionFilter(spanish_filter), spanish_tts],  # Spanish
-            ),
-            transport.output(),  # Transport bot output
-            tma_out              # Assistant spoken responses
-        ])
-
-        task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            transport.capture_participant_transcription(participant["id"])
-            # Kick off the conversation.
-            messages.append(
-                {
-                    "role": "system",
-                    "content": f"Please introduce yourself to the user and let them know the languages you speak. Your initial responses should be in {current_language}."})
-            await task.queue_frames([LLMMessagesFrame(messages)])
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/16-gpu-container-local-bot.py

@@ -1,130 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import aiohttp
-import os
-import sys
-import json
-
-from pipecat.frames.frames import LLMMessagesFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import (
-    LLMAssistantResponseAggregator, LLMUserResponseAggregator)
-from pipecat.services.deepgram import DeepgramTTSService
-from pipecat.services.openai import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport, DailyTransportMessageFrame
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-
-async def main(room_url: str, token):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Respond bot",
-            DailyParams(
-                audio_out_enabled=True,
-                transcription_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer()
-            )
-        )
-
-        tts = DeepgramTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("DEEPGRAM_API_KEY"),
-            voice="aura-asteria-en",
-            base_url="http://0.0.0.0:8080/v1/speak"
-        )
-
-        llm = OpenAILLMService(
-            # To use OpenAI
-            # api_key=os.getenv("OPENAI_API_KEY"),
-            # model="gpt-4o"
-            # Or, to use a local vLLM (or similar) api server
-            model="meta-llama/Meta-Llama-3-8B-Instruct",
-            base_url="http://0.0.0.0:8000/v1"
-        )
-
-        messages = [
-            {
-                "role": "system",
-                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
-            },
-        ]
-
-        tma_in = LLMUserResponseAggregator(messages)
-        tma_out = LLMAssistantResponseAggregator(messages)
-
-        pipeline = Pipeline([
-            transport.input(),   # Transport user input
-            tma_in,              # User responses
-            llm,                 # LLM
-            tts,                 # TTS
-            transport.output(),  # Transport bot output
-            tma_out              # Assistant spoken responses
-        ])
-
-        task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True, enable_metrics=True))
-
-        # When a participant joins, start transcription for that participant so the
-        # bot can "hear" and respond to them.
-        @transport.event_handler("on_participant_joined")
-        async def on_participant_joined(transport, participant):
-            transport.capture_participant_transcription(participant["id"])
-
-        # When the first participant joins, the bot should introduce itself.
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            messages.append(
-                {"role": "system", "content": "Please introduce yourself to the user."})
-            await task.queue_frames([LLMMessagesFrame(messages)])
-
-        # Handle "latency-ping" messages. The client will send app messages that look like
-        # this:
-        #   { "latency-ping": { ts: <client-side timestamp> }}
-        #
-        # We want to send an immediate pong back to the client from this handler function.
-        # Also, we will push a frame into the top of the pipeline and send it after the
-        #
-        @transport.event_handler("on_app_message")
-        async def on_app_message(transport, message, sender):
-            try:
-                if "latency-ping" in message:
-                    logger.debug(f"Received latency ping app message: {message}")
-                    ts = message["latency-ping"]["ts"]
-                    # Send immediately
-                    transport.output().send_message(DailyTransportMessageFrame(
-                        message={"latency-pong-msg-handler": {"ts": ts}},
-                        participant_id=sender))
-                    # And push to the pipeline for the Daily transport.output to send
-                    await tma_in.push_frame(
-                        DailyTransportMessageFrame(
-                            message={"latency-pong-pipeline-delivery": {"ts": ts}},
-                            participant_id=sender))
-            except Exception as e:
-                logger.debug(f"message handling error: {e} - {message}")
-
-        runner = PipelineRunner()
-        await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/websocket-server/frames.proto

@@ -0,0 +1,25 @@
+syntax = "proto3";
+
+package dailyai_proto;
+
+message TextFrame {
+    string text = 1;
+}
+
+message AudioFrame {
+    bytes audio = 1;
+}
+
+message TranscriptionFrame {
+    string text = 1;
+    string participant_id = 2;
+    string timestamp = 3;
+}
+
+message Frame {
+    oneof frame {
+        TextFrame text = 1;
+        AudioFrame audio = 2;
+        TranscriptionFrame transcription = 3;
+    }
+}

examples/foundational/websocket-server/index.html

@@ -0,0 +1,134 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <script src="//cdn.jsdelivr.net/npm/protobufjs@7.X.X/dist/protobuf.min.js"></script>
+    <title>WebSocket Audio Stream</title>
+</head>
+
+<body>
+    <h1>WebSocket Audio Stream</h1>
+    <button id="startAudioBtn">Start Audio</button>
+    <button id="stopAudioBtn">Stop Audio</button>
+    <script>
+        const SAMPLE_RATE = 16000;
+        const BUFFER_SIZE = 8192;
+        const MIN_AUDIO_SIZE = 6400;
+
+        let audioContext;
+        let microphoneStream;
+        let scriptProcessor;
+        let source;
+        let frame;
+        let audioChunks = [];
+        let isPlaying = false;
+        let ws;
+
+        const proto = protobuf.load("frames.proto", (err, root) => {
+            if (err) throw err;
+            frame = root.lookupType("dailyai_proto.Frame");
+        });
+
+        function initWebSocket() {
+            ws = new WebSocket('ws://localhost:8765');
+
+            ws.addEventListener('open', () => console.log('WebSocket connection established.'));
+            ws.addEventListener('message', handleWebSocketMessage);
+            ws.addEventListener('close', (event) => console.log("WebSocket connection closed.", event.code, event.reason));
+            ws.addEventListener('error', (event) => console.error('WebSocket error:', event));
+        }
+
+        async function handleWebSocketMessage(event) {
+            const arrayBuffer = await event.data.arrayBuffer();
+            enqueueAudioFromProto(arrayBuffer);
+        }
+
+        function enqueueAudioFromProto(arrayBuffer) {
+            const parsedFrame = frame.decode(new Uint8Array(arrayBuffer));
+            if (!parsedFrame?.audio) return false;
+
+            const frameCount = parsedFrame.audio.data.length / 2;
+            const audioOutBuffer = audioContext.createBuffer(1, frameCount, SAMPLE_RATE);
+            const nowBuffering = audioOutBuffer.getChannelData(0);
+            const view = new Int16Array(parsedFrame.audio.data.buffer);
+
+            for (let i = 0; i < frameCount; i++) {
+                const word = view[i];
+                nowBuffering[i] = ((word + 32768) % 65536 - 32768) / 32768.0;
+            }
+
+            audioChunks.push(audioOutBuffer);
+            if (!isPlaying) playNextChunk();
+        }
+
+        function playNextChunk() {
+            if (audioChunks.length === 0) {
+                isPlaying = false;
+                return;
+            }
+
+            isPlaying = true;
+            const audioOutBuffer = audioChunks.shift();
+            const source = audioContext.createBufferSource();
+            source.buffer = audioOutBuffer;
+            source.connect(audioContext.destination);
+            source.onended = playNextChunk;
+            source.start();
+        }
+
+        function startAudio() {
+            if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+                alert('getUserMedia is not supported in your browser.');
+                return;
+            }
+
+            navigator.mediaDevices.getUserMedia({ audio: true })
+                .then((stream) => {
+                    microphoneStream = stream;
+                    audioContext = new (window.AudioContext || window.webkitAudioContext)();
+                    scriptProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
+                    source = audioContext.createMediaStreamSource(stream);
+                    source.connect(scriptProcessor);
+                    scriptProcessor.connect(audioContext.destination);
+
+                    const audioBuffer = [];
+                    const skipRatio = Math.floor(audioContext.sampleRate / (SAMPLE_RATE * 2));
+
+                    scriptProcessor.onaudioprocess = (event) => {
+                        const rawLeftChannelData = event.inputBuffer.getChannelData(0);
+                        for (let i = 0; i < rawLeftChannelData.length; i += skipRatio) {
+                            const normalized = ((rawLeftChannelData[i] * 32768.0) + 32768) % 65536 - 32768;
+                            const swappedBytes = ((normalized & 0xff) << 8) | ((normalized >> 8) & 0xff);
+                            audioBuffer.push(swappedBytes);
+                        }
+
+                        if (audioBuffer.length >= MIN_AUDIO_SIZE) {
+                            const audioFrame = frame.create({ audio: { audio: audioBuffer.slice(0, MIN_AUDIO_SIZE) } });
+                            const encodedFrame = new Uint8Array(frame.encode(audioFrame).finish());
+                            ws.send(encodedFrame);
+                            audioBuffer.splice(0, MIN_AUDIO_SIZE);
+                        }
+                    };
+
+                    initWebSocket();
+                })
+                .catch((error) => console.error('Error accessing microphone:', error));
+        }
+
+        function stopAudio() {
+            if (ws) {
+                ws.close();
+                scriptProcessor.disconnect();
+                source.disconnect();
+                ws = undefined;
+            }
+        }
+
+        document.getElementById('startAudioBtn').addEventListener('click', startAudio);
+        document.getElementById('stopAudioBtn').addEventListener('click', stopAudio);
+    </script>
+</body>
+
+</html>

examples/foundational/websocket-server/sample.py

@@ -0,0 +1,50 @@
+import asyncio
+import aiohttp
+import logging
+import os
+from dailyai.pipeline.frame_processor import FrameProcessor
+from dailyai.pipeline.frames import TextFrame, TranscriptionFrame
+from dailyai.pipeline.pipeline import Pipeline
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.transports.websocket_transport import WebsocketTransport
+from dailyai.services.whisper_ai_services import WhisperSTTService
+
+logging.basicConfig(format="%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+class WhisperTranscriber(FrameProcessor):
+    async def process_frame(self, frame):
+        if isinstance(frame, TranscriptionFrame):
+            print(f"Transcribed: {frame.text}")
+        else:
+            yield frame
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        transport = WebsocketTransport(
+            mic_enabled=True,
+            speaker_enabled=True,
+        )
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        pipeline = Pipeline([
+            WhisperSTTService(),
+            WhisperTranscriber(),
+            tts,
+        ])
+
+        @transport.on_connection
+        async def queue_frame():
+            await pipeline.queue_frames([TextFrame("Hello there!")])
+
+        await transport.run(pipeline)
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/image-gen.py

@@ -0,0 +1,125 @@
+import argparse
+import asyncio
+import requests
+import time
+import urllib.parse
+import random
+
+from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
+from dailyai.pipeline.frames import Frame, FrameType
+from dailyai.services.fal_ai_services import FalImageGenService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+
+
+async def main(room_url: str, token):
+    global transport
+    global llm
+    global tts
+
+    transport = DailyTransport(
+        room_url,
+        token,
+        "Imagebot",
+        1,
+    )
+    transport._mic_enabled = True
+    transport._camera_enabled = True
+    transport._mic_sample_rate = 16000
+    transport._camera_width = 1024
+    transport._camera_height = 1024
+
+    llm = AzureLLMService()
+    tts = AzureTTSService()
+    img = FalImageGenService()
+
+    async def handle_transcriptions():
+        print("handle_transcriptions got called")
+
+        sentence = ""
+        async for message in transport.get_transcriptions():
+            print(f"transcription message: {message}")
+            if message["session_id"] == transport._my_participant_id:
+                continue
+            finder = message["text"].find("start over")
+            print(f"finder: {finder}")
+            if finder >= 0:
+                async for audio in tts.run_tts(f"Resetting."):
+                    transport.output_queue.put(
+                        Frame(FrameType.AUDIO_FRAME, audio))
+                sentence = ""
+                continue
+            # todo: we could differentiate between transcriptions from
+            # different participants
+            sentence += f" {message['text']}"
+            print(f"sentence is now: {sentence}")
+            # TODO: Cache this audio
+            phrase = random.choice(
+                ["OK.", "Got it.", "Sure.", "You bet.", "Sure thing."])
+            async for audio in tts.run_tts(phrase):
+                transport.output_queue.put(Frame(FrameType.AUDIO_FRAME, audio))
+            img_result = img.run_image_gen(sentence, "1024x1024")
+            awaited_img = await asyncio.gather(img_result)
+            transport.output_queue.put(
+                [
+                    Frame(FrameType.IMAGE_FRAME, awaited_img[0][1]),
+                ]
+            )
+
+    @transport.event_handler("on_participant_joined")
+    async def on_participant_joined(transport, participant):
+        print(f"participant joined: {participant['info']['userName']}")
+        if participant["info"]["isLocal"]:
+            return
+        async for audio in tts.run_tts("Describe an image, and I'll create it."):
+            audio_generator = tts.run_tts(
+                f"Hello, {participant['info']['userName']}! Describe an image and I'll create it. To start over, just say 'start over'.")
+            async for audio in audio_generator:
+                transport.output_queue.put(Frame(FrameType.AUDIO_FRAME, audio))
+
+    transport.transcription_settings["extra"]["punctuate"] = False
+    transport.transcription_settings["extra"]["endpointing"] = False
+    await asyncio.gather(transport.run(), handle_transcriptions())
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
+    parser.add_argument(
+        "-u",
+        "--url",
+        type=str,
+        required=True,
+        help="URL of the Daily room to join")
+    parser.add_argument(
+        "-k",
+        "--apikey",
+        type=str,
+        required=True,
+        help="Daily API Key (needed to create token)",
+    )
+
+    args, unknown = parser.parse_known_args()
+
+    # Create a meeting token for the given room with an expiration 1 hour in
+    # the future.
+    room_name: str = urllib.parse.urlparse(args.url).path[1:]
+    expiration: float = time.time() + 60 * 60
+
+    res: requests.Response = requests.post(
+        f"https://api.daily.co/v1/meeting-tokens",
+        headers={
+            "Authorization": f"Bearer {args.apikey}"},
+        json={
+            "properties": {
+                "room_name": room_name,
+                "is_owner": True,
+                "exp": expiration}},
+    )
+
+    if res.status_code != 200:
+        raise Exception(
+            f"Failed to create meeting token: {res.status_code} {res.text}")
+
+    token: str = res.json()["token"]
+
+    asyncio.run(main(args.url, token))

examples/internal/11a-dial-out.py

@@ -0,0 +1,137 @@
+import aiohttp
+import asyncio
+import os
+import wave
+
+from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
+from dailyai.pipeline.aggregators import LLMContextAggregator
+from dailyai.services.ai_services import AIService, FrameLogger
+from dailyai.pipeline.frames import Frame, AudioFrame, LLMResponseEndFrame, LLMMessagesFrame
+from typing import AsyncGenerator
+
+from runner import configure
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+sounds = {}
+sound_files = [
+    'ding1.wav',
+    'ding2.wav'
+]
+
+script_dir = os.path.dirname(__file__)
+
+for file in sound_files:
+    # Build the full path to the image file
+    full_path = os.path.join(script_dir, "assets", file)
+    # Get the filename without the extension to use as the dictionary key
+    filename = os.path.splitext(os.path.basename(full_path))[0]
+    # Open the image and convert it to bytes
+    with wave.open(full_path) as audio_file:
+        sounds[file] = audio_file.readframes(-1)
+
+
+class OutboundSoundEffectWrapper(AIService):
+    def __init__(self):
+        pass
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, LLMResponseEndFrame):
+            yield AudioFrame(sounds["ding1.wav"])
+            # In case anything else up the stack needs it
+            yield frame
+        else:
+            yield frame
+
+
+class InboundSoundEffectWrapper(AIService):
+    def __init__(self):
+        pass
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, LLMMessagesFrame):
+            yield AudioFrame(sounds["ding2.wav"])
+            # In case anything else up the stack needs it
+            yield frame
+        else:
+            yield frame
+
+
+async def main(room_url: str, token, phone):
+    async with aiohttp.ClientSession() as session:
+
+        global transport
+        global llm
+        global tts
+
+        transport = DailyTransport(
+            room_url,
+            token,
+            "Respond bot",
+            300,
+        )
+        transport._mic_enabled = True
+        transport._mic_sample_rate = 16000
+        transport._camera_enabled = False
+
+        llm = AzureLLMService()
+        tts = AzureTTSService()
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            await tts.say("Hi, I'm listening!", transport.send_queue)
+            await transport.send_queue.put(AudioFrame(sounds["ding1.wav"]))
+
+        async def handle_transcriptions():
+            messages = [
+                {"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
+            ]
+
+            tma_in = LLMContextAggregator(
+                messages, "user", transport._my_participant_id
+            )
+            tma_out = LLMContextAggregator(
+                messages, "assistant", transport._my_participant_id
+            )
+            out_sound = OutboundSoundEffectWrapper()
+            in_sound = InboundSoundEffectWrapper()
+            fl = FrameLogger("LLM Out")
+            fl2 = FrameLogger("Transcription In")
+            await out_sound.run_to_queue(
+                transport.send_queue,
+                tts.run(
+                    tma_out.run(
+                        llm.run(
+                            fl2.run(
+                                in_sound.run(
+                                    tma_in.run(
+                                        transport.get_receive_frames()
+                                    )
+                                )
+                            )
+                        )
+                    )
+                )
+            )
+
+        @transport.event_handler("on_participant_joined")
+        async def pax_joined(transport, pax):
+            print(f"PARTICIPANT JOINED: {pax}")
+
+        @transport.event_handler("on_call_state_updated")
+        async def on_call_state_updated(transport, state):
+            if (state == "joined"):
+                if (phone):
+                    transport.start_recording()
+                    transport.dialout(phone)
+
+        transport.transcription_settings["extra"]["punctuate"] = True
+
+        await asyncio.gather(transport.run(), handle_transcriptions())
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/moondream-chatbot/.dockerignore

@@ -1,163 +0,0 @@
-# flyctl launch added from .gitignore
-# Byte-compiled / optimized / DLL files
-**/__pycache__
-**/*.py[cod]
-**/*$py.class
-
-# C extensions
-**/*.so
-
-# Distribution / packaging
-**/.Python
-**/build
-**/develop-eggs
-**/dist
-**/downloads
-**/eggs
-**/.eggs
-**/lib
-**/lib64
-**/parts
-**/sdist
-**/var
-**/wheels
-**/share/python-wheels
-**/*.egg-info
-**/.installed.cfg
-**/*.egg
-**/MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-**/*.manifest
-**/*.spec
-
-# Installer logs
-**/pip-log.txt
-**/pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-**/htmlcov
-**/.tox
-**/.nox
-**/.coverage
-**/.coverage.*
-**/.cache
-**/nosetests.xml
-**/coverage.xml
-**/*.cover
-**/*.py,cover
-**/.hypothesis
-**/.pytest_cache
-**/cover
-
-# Translations
-**/*.mo
-**/*.pot
-
-# Django stuff:
-**/*.log
-**/local_settings.py
-**/db.sqlite3
-**/db.sqlite3-journal
-
-# Flask stuff:
-**/instance
-**/.webassets-cache
-
-# Scrapy stuff:
-**/.scrapy
-
-# Sphinx documentation
-**/docs/_build
-
-# PyBuilder
-**/.pybuilder
-**/target
-
-# Jupyter Notebook
-**/.ipynb_checkpoints
-
-# IPython
-**/profile_default
-**/ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-**/.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-**/__pypackages__
-
-# Celery stuff
-**/celerybeat-schedule
-**/celerybeat.pid
-
-# SageMath parsed files
-**/*.sage.py
-
-# Environments
-**/.env
-**/.venv
-**/env
-**/venv
-**/ENV
-**/env.bak
-**/venv.bak
-
-# Spyder project settings
-**/.spyderproject
-**/.spyproject
-
-# Rope project settings
-**/.ropeproject
-
-# mkdocs documentation
-site
-
-# mypy
-**/.mypy_cache
-**/.dmypy.json
-**/dmypy.json
-
-# Pyre type checker
-**/.pyre
-
-# pytype static type analyzer
-**/.pytype
-
-# Cython debug symbols
-**/cython_debug
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-**/runpod.toml
-fly.toml

examples/moondream-chatbot/.gitignore

@@ -1,161 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-runpod.toml

examples/moondream-chatbot/Dockerfile

@@ -1,25 +0,0 @@
-FROM ubuntu:22.04
-
-RUN apt-get update && apt-get install -y wget
-RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
-RUN dpkg -i cuda-keyring_1.1-1_all.deb
-
-RUN echo "deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /" > /etc/apt/sources.list.d/cuda-ubuntu2204-x86_64.list
-
-RUN apt-get update && apt-get install -y python3 python3-pip
-RUN apt-get install -y cuda-nvcc-12-4 libcublas-12-4 libcudnn8
-
-RUN mkdir /app
-RUN mkdir /app/assets
-RUN mkdir /app/utils
-COPY *.py /app/
-COPY requirements.txt /app/
-copy assets/* /app/assets/
-copy utils/* /app/utils/
-
-WORKDIR /app
-RUN pip3 install -r requirements.txt
-
-EXPOSE 7860
-
-CMD ["python3", "server.py"]

examples/moondream-chatbot/Dockerfile.intel

@@ -1,76 +0,0 @@
-FROM ubuntu:22.04
-
-# environment variables for Intel OneAPI components
-ENV DPCPPROOT=/opt/intel/oneapi/compiler/latest
-ENV MKLROOT=/opt/intel/oneapi/mkl/latest
-ENV CCLROOT=/opt/intel/oneapi/ccl/latest
-ENV MPIROOT=/opt/intel/oneapi/mpi/latest
-
-# Install necessary dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    build-essential \
-    wget \
-    lsb-release \
-    pciutils \
-    gnupg2 \
-    python3-pip
-
-# Add Intel OneAPI repository and GPG key
-# Intel GPU repository and GPG key
-# Install Intel OneAPI components and source the environment scripts
-RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
-    echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \
-    /bin/bash -c ' \
-    . /etc/os-release && \
-    if [[ " jammy " =~ " ${VERSION_CODENAME} " ]]; then \
-        wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | gpg --dearmor --output /usr/share/keyrings/intel-graphics.gpg && \
-        echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu ${VERSION_CODENAME}/lts/2350 unified" | \
-        tee /etc/apt/sources.list.d/intel-gpu-${VERSION_CODENAME}.list && \
-        apt-get update && \
-        apt-get install -y --no-install-recommends intel-opencl-icd \
-            intel-level-zero-gpu level-zero intel-media-va-driver-non-free \
-            libmfx1 libmfxgen1 libvpl2 libegl-mesa0 libegl1-mesa \
-            libegl1-mesa-dev libgbm1 libgl1-mesa-dev libgl1-mesa-dri \
-            libglapi-mesa libgles2-mesa-dev libglx-mesa0 libigdgmm12 \
-            libxatracker2 mesa-va-drivers mesa-vdpau-drivers \
-            mesa-vulkan-drivers va-driver-all; \
-    else \
-        echo "Ubuntu version ${VERSION_CODENAME} not supported. Exiting..."; \
-        exit 1; \
-    fi' && \
-    apt-get update && apt-get install -y --no-install-recommends \
-    intel-oneapi-dpcpp-cpp-2024.1=2024.1.0-963 intel-oneapi-mkl-devel=2024.1.0-691 \
-    intel-oneapi-ccl-devel=2021.12.0-309 && \
-    apt-get clean && rm -rf /var/lib/apt/lists/* && \
-    groupadd -r render && usermod -aG render root && \
-    echo "source ${DPCPPROOT}/env/vars.sh" >> ~/.bashrc && \
-    echo "source ${MKLROOT}/env/vars.sh" >> ~/.bashrc && \
-    echo "source ${CCLROOT}/env/vars.sh" >> ~/.bashrc && \
-    echo "source ${MPIROOT}/env/vars.sh" >> ~/.bashrc && \
-    echo "export LD_LIBRARY_PATH=${MKLROOT}/lib:${DPCPPROOT}/linux/compiler/lib/intel64_lin:$LD_LIBRARY_PATH" >> ~/.bashrc
-
-WORKDIR /app
-COPY . /app
-RUN mkdir -p /app /app/assets /app/utils
-COPY *.py requirements.txt assets/* utils/* /app/
-
-# Install the Intel-specific versions of torch
-RUN python3 -m pip install --no-cache-dir -r requirements.txt && \
-    pip uninstall -y torch && \
-    pip freeze | grep 'nvidia-' | xargs pip uninstall -y && \
-    pip install --no-cache-dir --force-reinstall torch==2.1.0.post2 torchvision==0.16.0.post2 torchaudio==2.1.0.post2 \
-    intel-extension-for-pytorch==2.1.30+xpu oneccl_bind_pt==2.1.300+xpu \
-    --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/  
-
-RUN echo '#!/bin/bash\n\
-source ${DPCPPROOT}/env/vars.sh\n\
-source ${MKLROOT}/env/vars.sh\n\
-source ${CCLROOT}/env/vars.sh\n\
-source ${MPIROOT}/env/vars.sh\n\
-export LD_LIBRARY_PATH=${MKLROOT}/lib:${DPCPPROOT}/linux/compiler/lib/intel64_lin:$LD_LIBRARY_PATH\n\
-python3 server.py' > /usr/local/bin/run_app.sh && \
-    chmod +x /usr/local/bin/run_app.sh && \
-    find / -type d -name "__pycache__" -exec rm -rf {} + 
-
-EXPOSE 7860
-ENTRYPOINT ["/usr/local/bin/run_app.sh"]

examples/moondream-chatbot/README.md

@@ -1,44 +0,0 @@
-# Moondream Chatbot
-
-<img src="image.png" width="420px">
-
-
-This app connects you to a chatbot powered by GPT-4, complete with animations generated by Stable Video Diffusion. The chatbot also has vision powers thanks to [Moondream](https://moondream.ai) so you can ask it, for example, "what do you see?".
-
-ℹ️ The first time, things might take some time to get started since VAD (Voice Activity Detection) and vision models need to be downloaded.
-
-## Get started
-
-```python
-python3 -m venv venv
-source venv/bin/activate
-pip install -r requirements.txt
-
-cp env.example .env # and add your credentials
-
-```
-
-## Run the server
-
-```bash
-python server.py
-```
-
-Then, visit `http://localhost:7860/start` in your browser to start a chatbot
-session.
-
-## Build and test the Docker image
-
-```
-docker build -t moonbot .
-docker run --env-file .env -p 7860:7860 moonbot
-```
-
-### For Intel GPUs (Arc, Max and Flex series)
-
-```
-docker build -t moonbot -f Dockerfile.intel .
-docker run --env-file .env -p 7860:7860 --device /dev/dri moonbot
-```
-
-You can try to visit `http://localhost:7860/start` again.

examples/moondream-chatbot/bot.py

@@ -1,208 +0,0 @@
-import asyncio
-import aiohttp
-import os
-import sys
-
-from PIL import Image
-
-from pipecat.frames.frames import (
-    ImageRawFrame,
-    SpriteFrame,
-    Frame,
-    LLMMessagesFrame,
-    AudioRawFrame,
-    TTSStoppedFrame,
-    TextFrame,
-    UserImageRawFrame,
-    UserImageRequestFrame,
-)
-
-from pipecat.pipeline.parallel_pipeline import ParallelPipeline
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineTask
-from pipecat.processors.aggregators.llm_response import LLMUserResponseAggregator
-from pipecat.processors.aggregators.sentence import SentenceAggregator
-from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
-from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.moondream import MoondreamService
-from pipecat.services.openai import OpenAILLMService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-user_request_answer = "Let me take a look."
-
-sprites = []
-
-script_dir = os.path.dirname(__file__)
-
-for i in range(1, 26):
-    # Build the full path to the image file
-    full_path = os.path.join(script_dir, f"assets/robot0{i}.png")
-    # Get the filename without the extension to use as the dictionary key
-    # Open the image and convert it to bytes
-    with Image.open(full_path) as img:
-        sprites.append(ImageRawFrame(image=img.tobytes(), size=img.size, format=img.format))
-
-flipped = sprites[::-1]
-sprites.extend(flipped)
-
-# When the bot isn't talking, show a static image of the cat listening
-quiet_frame = sprites[0]
-talking_frame = SpriteFrame(images=sprites)
-
-
-class TalkingAnimation(FrameProcessor):
-    """
-    This class starts a talking animation when it receives an first AudioFrame,
-    and then returns to a "quiet" sprite when it sees a TTSStoppedFrame.
-    """
-
-    def __init__(self):
-        super().__init__()
-        self._is_talking = False
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        if isinstance(frame, AudioRawFrame):
-            if not self._is_talking:
-                await self.push_frame(talking_frame)
-                self._is_talking = True
-        elif isinstance(frame, TTSStoppedFrame):
-            await self.push_frame(quiet_frame)
-            self._is_talking = False
-        await self.push_frame(frame)
-
-
-class UserImageRequester(FrameProcessor):
-    def __init__(self):
-        super().__init__()
-        self.participant_id = None
-
-    def set_participant_id(self, participant_id: str):
-        self.participant_id = participant_id
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        if self.participant_id and isinstance(frame, TextFrame):
-            if frame.text == user_request_answer:
-                await self.push_frame(UserImageRequestFrame(self.participant_id), FrameDirection.UPSTREAM)
-                await self.push_frame(TextFrame("Describe the image in a short sentence."))
-        elif isinstance(frame, UserImageRawFrame):
-            await self.push_frame(frame)
-
-
-class TextFilterProcessor(FrameProcessor):
-    def __init__(self, text: str):
-        super().__init__()
-        self.text = text
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        if isinstance(frame, TextFrame):
-            if frame.text != self.text:
-                await self.push_frame(frame)
-        else:
-            await self.push_frame(frame)
-
-
-class ImageFilterProcessor(FrameProcessor):
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        await super().process_frame(frame, direction)
-
-        if not isinstance(frame, ImageRawFrame):
-            await self.push_frame(frame)
-
-
-async def main(room_url: str, token):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Chatbot",
-            DailyParams(
-                audio_out_enabled=True,
-                camera_out_enabled=True,
-                camera_out_width=1024,
-                camera_out_height=576,
-                transcription_enabled=True,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer()
-            )
-        )
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id="pNInz6obpgDQGcFmaJgB",
-        )
-
-        llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
-
-        ta = TalkingAnimation()
-
-        sa = SentenceAggregator()
-        ir = UserImageRequester()
-        va = VisionImageFrameAggregator()
-
-        # If you run into weird description, try with use_cpu=True
-        moondream = MoondreamService()
-
-        tf = TextFilterProcessor(user_request_answer)
-        imgf = ImageFilterProcessor()
-
-        messages = [
-            {
-                "role": "system",
-                "content": f"You are Chatbot, a friendly, helpful robot. Let the user know that you are capable of chatting or describing what you see. Your goal is to demonstrate your capabilities in a succinct way. Reply with only '{user_request_answer}' if the user asks you to describe what you see. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way, but keep your responses brief. Start by introducing yourself.",
-            },
-        ]
-
-        ura = LLMUserResponseAggregator(messages)
-
-        pipeline = Pipeline([
-            transport.input(),
-            ura,
-            llm,
-            ParallelPipeline(
-                [sa, ir, va, moondream],
-                [tf, imgf]),
-            tts,
-            ta,
-            transport.output()
-        ])
-
-        task = PipelineTask(pipeline)
-        await task.queue_frame(quiet_frame)
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            transport.capture_participant_transcription(participant["id"])
-            transport.capture_participant_video(participant["id"], framerate=0)
-            ir.set_participant_id(participant["id"])
-            await task.queue_frames([LLMMessagesFrame(messages)])
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/moondream-chatbot/env.example

@@ -1,4 +0,0 @@
-DAILY_SAMPLE_ROOM_URL=https://yourdomain.daily.co/yourroom # (for joining the bot to the same room repeatedly for local dev)
-DAILY_API_KEY=7df...
-OPENAI_API_KEY=sk-PL...
-ELEVENLABS_API_KEY=aeb...

examples/moondream-chatbot/requirements.txt

@@ -1,5 +0,0 @@
-python-dotenv
-requests
-fastapi[all]
-uvicorn
-pipecat-ai[daily,moondream,openai,silero]

examples/moondream-chatbot/server.py

@@ -1,124 +0,0 @@
-import os
-import argparse
-import subprocess
-import atexit
-
-from fastapi import FastAPI, Request, HTTPException
-from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import JSONResponse, RedirectResponse
-
-from utils.daily_helpers import create_room as _create_room, get_token
-
-MAX_BOTS_PER_ROOM = 1
-
-# Bot sub-process dict for status reporting and concurrency control
-bot_procs = {}
-
-
-def cleanup():
-    # Clean up function, just to be extra safe
-    for proc in bot_procs.values():
-        proc.terminate()
-        proc.wait()
-
-
-atexit.register(cleanup)
-
-
-app = FastAPI()
-
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-
-@app.get("/start")
-async def start_agent(request: Request):
-    print(f"!!! Creating room")
-    room_url, room_name = _create_room()
-    print(f"!!! Room URL: {room_url}")
-    # Ensure the room property is present
-    if not room_url:
-        raise HTTPException(
-            status_code=500,
-            detail="Missing 'room' property in request data. Cannot start agent without a target room!")
-
-    # Check if there is already an existing process running in this room
-    num_bots_in_room = sum(
-        1 for proc in bot_procs.values() if proc[1] == room_url and proc[0].poll() is None)
-    if num_bots_in_room >= MAX_BOTS_PER_ROOM:
-        raise HTTPException(
-            status_code=500, detail=f"Max bot limited reach for room: {room_url}")
-
-    # Get the token for the room
-    token = get_token(room_url)
-
-    if not token:
-        raise HTTPException(
-            status_code=500, detail=f"Failed to get token for room: {room_url}")
-
-    # Spawn a new agent, and join the user session
-    # Note: this is mostly for demonstration purposes (refer to 'deployment' in README)
-    try:
-        proc = subprocess.Popen(
-            [
-                f"python3 -m bot -u {room_url} -t {token}"
-            ],
-            shell=True,
-            bufsize=1,
-            cwd=os.path.dirname(os.path.abspath(__file__))
-        )
-        bot_procs[proc.pid] = (proc, room_url)
-    except Exception as e:
-        raise HTTPException(
-            status_code=500, detail=f"Failed to start subprocess: {e}")
-
-    return RedirectResponse(room_url)
-
-
-@app.get("/status/{pid}")
-def get_status(pid: int):
-    # Look up the subprocess
-    proc = bot_procs.get(pid)
-
-    # If the subprocess doesn't exist, return an error
-    if not proc:
-        raise HTTPException(
-            status_code=404, detail=f"Bot with process id: {pid} not found")
-
-    # Check the status of the subprocess
-    if proc[0].poll() is None:
-        status = "running"
-    else:
-        status = "finished"
-
-    return JSONResponse({"bot_id": pid, "status": status})
-
-
-if __name__ == "__main__":
-    import uvicorn
-
-    default_host = os.getenv("HOST", "0.0.0.0")
-    default_port = int(os.getenv("FAST_API_PORT", "7860"))
-
-    parser = argparse.ArgumentParser(
-        description="Daily Moondream FastAPI server")
-    parser.add_argument("--host", type=str,
-                        default=default_host, help="Host address")
-    parser.add_argument("--port", type=int,
-                        default=default_port, help="Port number")
-    parser.add_argument("--reload", action="store_true",
-                        help="Reload code on change")
-
-    config = parser.parse_args()
-
-    uvicorn.run(
-        "server:app",
-        host=config.host,
-        port=config.port,
-        reload=config.reload,
-    )

examples/moondream-chatbot/utils/daily_helpers.py

@@ -1,109 +0,0 @@
-
-import urllib.parse
-import os
-import time
-import urllib
-import requests
-
-from dotenv import load_dotenv
-load_dotenv()
-
-
-daily_api_path = os.getenv("DAILY_API_URL") or "api.daily.co/v1"
-daily_api_key = os.getenv("DAILY_API_KEY")
-
-
-def create_room() -> tuple[str, str]:
-    """
-    Helper function to create a Daily room.
-    # See: https://docs.daily.co/reference/rest-api/rooms
-
-    Returns:
-        tuple: A tuple containing the room URL and room name.
-
-    Raises:
-        Exception: If the request to create the room fails or if the response does not contain the room URL or room name.
-    """
-    room_props = {
-        "exp": time.time() + 60 * 60,  # 1 hour
-        "enable_chat": True,
-        "enable_emoji_reactions": True,
-        "eject_at_room_exp": True,
-        "enable_prejoin_ui": False,  # Important for the bot to be able to join headlessly
-    }
-    res = requests.post(
-        f"https://{daily_api_path}/rooms",
-        headers={"Authorization": f"Bearer {daily_api_key}"},
-        json={
-            "properties": room_props
-        },
-    )
-    if res.status_code != 200:
-        raise Exception(f"Unable to create room: {res.text}")
-
-    data = res.json()
-    room_url: str = data.get("url")
-    room_name: str = data.get("name")
-    if room_url is None or room_name is None:
-        raise Exception("Missing room URL or room name in response")
-
-    return room_url, room_name
-
-
-def get_name_from_url(room_url: str) -> str:
-    """
-    Extracts the name from a given room URL.
-
-    Args:
-        room_url (str): The URL of the room.
-
-    Returns:
-        str: The extracted name from the room URL.
-    """
-    return urllib.parse.urlparse(room_url).path[1:]
-
-
-def get_token(room_url: str) -> str:
-    """
-    Retrieves a meeting token for the specified Daily room URL.
-    # See: https://docs.daily.co/reference/rest-api/meeting-tokens
-
-    Args:
-        room_url (str): The URL of the Daily room.
-
-    Returns:
-        str: The meeting token.
-
-    Raises:
-        Exception: If no room URL is specified or if no Daily API key is specified.
-        Exception: If there is an error creating the meeting token.
-    """
-    if not room_url:
-        raise Exception(
-            "No Daily room specified. You must specify a Daily room in order a token to be generated.")
-
-    if not daily_api_key:
-        raise Exception(
-            "No Daily API key specified. set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers.")
-
-    expiration: float = time.time() + 60 * 60
-    room_name = get_name_from_url(room_url)
-
-    res: requests.Response = requests.post(
-        f"https://{daily_api_path}/meeting-tokens",
-        headers={
-            "Authorization": f"Bearer {daily_api_key}"},
-        json={
-            "properties": {
-                "room_name": room_name,
-                "is_owner": True,  # Owner tokens required for transcription
-                "exp": expiration}},
-    )
-
-    if res.status_code != 200:
-        raise Exception(
-            f"Failed to create meeting token: {res.status_code} {res.text}")
-
-    token: str = res.json()["token"]
-
-    return token

examples/patient-intake/Dockerfile

@@ -1,16 +0,0 @@
-FROM python:3.10-bullseye
-
-RUN mkdir /app
-RUN mkdir /app/assets
-RUN mkdir /app/utils
-COPY *.py /app/
-COPY requirements.txt /app/
-copy assets/* /app/assets/
-copy utils/* /app/utils/
-
-WORKDIR /app
-RUN pip3 install -r requirements.txt
-
-EXPOSE 7860
-
-CMD ["python3", "server.py"]

examples/patient-intake/README.md

@@ -1,37 +0,0 @@
-# Simple Chatbot
-
-<img src="image.png" width="420px">
-
-This app connects you to a chatbot powered by GPT-4, complete with animations generated by Stable Video Diffusion.
-
-See a video of it in action: https://x.com/kwindla/status/1778628911817183509
-
-And a quick video walkthrough of the code: https://www.loom.com/share/13df1967161f4d24ade054e7f8753416
-
-ℹ️ The first time, things might take extra time to get started since VAD (Voice Activity Detection) model needs to be downloaded.
-
-## Get started
-
-```python
-python3 -m venv venv
-source venv/bin/activate
-pip install -r requirements.txt
-
-cp env.example .env # and add your credentials
-
-```
-
-## Run the server
-
-```bash
-python server.py
-```
-
-Then, visit `http://localhost:7860/start` in your browser to start a chatbot session.
-
-## Build and test the Docker image
-
-```
-docker build -t chatbot .
-docker run --env-file .env -p 7860:7860 chatbot
-```

examples/patient-intake/bot.py

@@ -1,355 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import asyncio
-import aiohttp
-import os
-import sys
-import wave
-
-from typing import List
-
-from openai._types import NotGiven, NOT_GIVEN
-
-from openai.types.chat import (
-    ChatCompletionToolParam,
-)
-
-from pipecat.frames.frames import AudioRawFrame
-from pipecat.pipeline.pipeline import Pipeline
-from pipecat.pipeline.runner import PipelineRunner
-from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import LLMUserContextAggregator, LLMAssistantContextAggregator
-from pipecat.processors.logger import FrameLogger
-from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.elevenlabs import ElevenLabsTTSService
-from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame, OpenAILLMService
-from pipecat.services.ai_services import AIService
-from pipecat.transports.services.daily import DailyParams, DailyTransport
-from pipecat.vad.silero import SileroVADAnalyzer
-
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logger.remove(0)
-logger.add(sys.stderr, level="DEBUG")
-
-sounds = {}
-sound_files = [
-    "clack-short.wav",
-    "clack.wav",
-    "clack-short-quiet.wav",
-    "ding.wav",
-    "ding2.wav",
-]
-
-script_dir = os.path.dirname(__file__)
-
-for file in sound_files:
-    # Build the full path to the sound file
-    full_path = os.path.join(script_dir, "assets", file)
-    # Get the filename without the extension to use as the dictionary key
-    filename = os.path.splitext(os.path.basename(full_path))[0]
-    # Open the sound and convert it to bytes
-    with wave.open(full_path) as audio_file:
-        sounds[file] = AudioRawFrame(audio_file.readframes(-1),
-                                     audio_file.getframerate(), audio_file.getnchannels())
-
-
-class IntakeProcessor:
-    def __init__(
-        self,
-        context: OpenAILLMContext,
-        llm: AIService,
-        tools: List[ChatCompletionToolParam] | NotGiven = NOT_GIVEN,
-        *args,
-        **kwargs,
-    ):
-        super().__init__(*args, **kwargs)
-        self._context: OpenAILLMContext = context
-        self._llm = llm
-        print(f"Initializing context from IntakeProcessor")
-        self._context.add_message({"role": "system", "content": "You are Jessica, an agent for a company called Tri-County Health Services. Your job is to collect important information from the user before their doctor visit. You're talking to Chad Bailey. You should address the user by their first name and be polite and professional. You're not a medical professional, so you shouldn't provide any advice. Keep your responses short. Your job is to collect information to give to a doctor. Don't make assumptions about what values to plug into functions. Ask for clarification if a user response is ambiguous. Start by introducing yourself. Then, ask the user to confirm their identity by telling you their birthday, including the year. When they answer with their birthday, call the verify_birthday function."})
-        self._context.set_tools([
-            {
-                "type": "function",
-                "function": {
-                    "name": "verify_birthday",
-                    "description": "Use this function to verify the user has provided their correct birthday.",
-                    "parameters": {
-                        "type": "object",
-                        "properties": {
-                            "birthday": {
-                                "type": "string",
-                                "description": "The user's birthdate, including the year. The user can provide it in any format, but convert it to YYYY-MM-DD format to call this function.",
-                            }},
-                    },
-                },
-            }])
-        # Create an allowlist of functions that the LLM can call
-        self._functions = [
-            "verify_birthday",
-            "list_prescriptions",
-            "list_allergies",
-            "list_conditions",
-            "list_visit_reasons",
-        ]
-
-    async def verify_birthday(self, llm, args):
-        if args["birthday"] == "1983-01-01":
-            self._context.set_tools(
-                [
-                    {
-                        "type": "function",
-                        "function": {
-                            "name": "list_prescriptions",
-                            "description": "Once the user has provided a list of their prescription medications, call this function.",
-                            "parameters": {
-                                "type": "object",
-                                "properties": {
-                                    "prescriptions": {
-                                        "type": "array",
-                                        "items": {
-                                            "type": "object",
-                                            "properties": {
-                                                "medication": {
-                                                    "type": "string",
-                                                    "description": "The medication's name",
-                                                },
-                                                "dosage": {
-                                                    "type": "string",
-                                                    "description": "The prescription's dosage",
-                                                },
-                                            },
-                                        },
-                                    }},
-                            },
-                        },
-                    }])
-            # It's a bit weird to push this to the LLM, but it gets it into the pipeline
-            await llm.push_frame(sounds["ding2.wav"], FrameDirection.DOWNSTREAM)
-            # We don't need the function call in the context, so just return a new
-            # system message and let the framework re-prompt
-            return [{"role": "system", "content": "Next, thank the user for confirming their identity, then ask the user to list their current prescriptions. Each prescription needs to have a medication name and a dosage. Do not call the list_prescriptions function with any unknown dosages."}]
-        else:
-            # The user provided an incorrect birthday; ask them to try again
-            return [{"role": "system", "content": "The user provided an incorrect birthday. Ask them for their birthday again. When they answer, call the verify_birthday function."}]
-
-    async def start_prescriptions(self, llm):
-        print(f"!!! doing start prescriptions")
-        # Move on to allergies
-        self._context.set_tools(
-            [
-                {
-                    "type": "function",
-                    "function": {
-                        "name": "list_allergies",
-                        "description": "Once the user has provided a list of their allergies, call this function.",
-                        "parameters": {
-                            "type": "object",
-                            "properties": {
-                                "allergies": {
-                                    "type": "array",
-                                    "items": {
-                                        "type": "object",
-                                        "properties": {
-                                            "name": {
-                                                "type": "string",
-                                                "description": "What the user is allergic to",
-                                            }},
-                                    },
-                                }},
-                        },
-                    },
-                }])
-        self._context.add_message(
-            {
-                "role": "system",
-                "content": "Next, ask the user if they have any allergies. Once they have listed their allergies or confirmed they don't have any, call the list_allergies function."})
-        print(f"!!! about to await llm process frame in start prescrpitions")
-        await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM)
-        print(f"!!! past await process frame in start prescriptions")
-
-    async def start_allergies(self, llm):
-        print("!!! doing start allergies")
-        # Move on to conditions
-        self._context.set_tools(
-            [
-                {
-                    "type": "function",
-                    "function": {
-                        "name": "list_conditions",
-                        "description": "Once the user has provided a list of their medical conditions, call this function.",
-                        "parameters": {
-                            "type": "object",
-                            "properties": {
-                                "conditions": {
-                                    "type": "array",
-                                    "items": {
-                                        "type": "object",
-                                        "properties": {
-                                            "name": {
-                                                "type": "string",
-                                                "description": "The user's medical condition",
-                                            }},
-                                    },
-                                }},
-                        },
-                    },
-                },
-            ])
-        self._context.add_message(
-            {
-                "role": "system",
-                "content": "Now ask the user if they have any medical conditions the doctor should know about. Once they've answered the question, call the list_conditions function."})
-        await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM)
-
-    async def start_conditions(self, llm):
-        print("!!! doing start conditions")
-        # Move on to visit reasons
-        self._context.set_tools(
-            [
-                {
-                    "type": "function",
-                    "function": {
-                        "name": "list_visit_reasons",
-                        "description": "Once the user has provided a list of the reasons they are visiting a doctor today, call this function.",
-                        "parameters": {
-                            "type": "object",
-                            "properties": {
-                                "visit_reasons": {
-                                    "type": "array",
-                                    "items": {
-                                        "type": "object",
-                                        "properties": {
-                                            "name": {
-                                                "type": "string",
-                                                "description": "The user's reason for visiting the doctor",
-                                            }},
-                                    },
-                                }},
-                        },
-                    },
-                }])
-        self._context.add_message(
-            {"role": "system", "content": "Finally, ask the user the reason for their doctor visit today. Once they answer, call the list_visit_reasons function."})
-        await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM)
-
-    async def start_visit_reasons(self, llm):
-        print("!!! doing start visit reasons")
-        # move to finish call
-        self._context.set_tools([])
-        self._context.add_message({"role": "system",
-                                   "content": "Now, thank the user and end the conversation."})
-        await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM)
-
-    async def save_data(self, llm, args):
-        logger.info(f"!!! Saving data: {args}")
-        # Since this is supposed to be "async", returning None from the callback
-        # will prevent adding anything to context or re-prompting
-        return None
-
-
-async def main(room_url: str, token):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Chatbot",
-            DailyParams(
-                audio_out_enabled=True,
-                camera_out_enabled=True,
-                camera_out_width=1024,
-                camera_out_height=576,
-                vad_enabled=True,
-                vad_analyzer=SileroVADAnalyzer(),
-                transcription_enabled=True,
-                #
-                # Spanish
-                #
-                # transcription_settings=DailyTranscriptionSettings(
-                #     language="es",
-                #     tier="nova",
-                #     model="2-general"
-                # )
-            )
-        )
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            #
-            # English
-            #
-            voice_id="pNInz6obpgDQGcFmaJgB",
-
-            #
-            # Spanish
-            #
-            # model="eleven_multilingual_v2",
-            # voice_id="gD1IexrzCvsXPHUuT0s3",
-        )
-
-        llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
-
-        messages = []
-        context = OpenAILLMContext(messages=messages)
-        user_context = LLMUserContextAggregator(context)
-        assistant_context = LLMAssistantContextAggregator(context)
-
-        intake = IntakeProcessor(context, llm)
-        llm.register_function("verify_birthday", intake.verify_birthday)
-        llm.register_function(
-            "list_prescriptions",
-            intake.save_data,
-            start_callback=intake.start_prescriptions)
-        llm.register_function(
-            "list_allergies",
-            intake.save_data,
-            start_callback=intake.start_allergies)
-        llm.register_function(
-            "list_conditions",
-            intake.save_data,
-            start_callback=intake.start_conditions)
-        llm.register_function(
-            "list_visit_reasons",
-            intake.save_data,
-            start_callback=intake.start_visit_reasons)
-
-        fl = FrameLogger("LLM Output")
-
-        pipeline = Pipeline([
-            transport.input(),   # Transport input
-            user_context,        # User responses
-            llm,                 # LLM
-            fl,                  # Frame logger
-            tts,                 # TTS
-            transport.output(),  # Transport output
-            assistant_context,   # Assistant responses
-        ])
-
-        task = PipelineTask(pipeline, PipelineParams(allow_interruptions=False))
-
-        @transport.event_handler("on_first_participant_joined")
-        async def on_first_participant_joined(transport, participant):
-            transport.capture_participant_transcription(participant["id"])
-            print(f"Context is: {context}")
-            await task.queue_frames([OpenAILLMContextFrame(context)])
-
-        runner = PipelineRunner()
-
-        await runner.run(task)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/patient-intake/env.example

@@ -1,4 +0,0 @@
-DAILY_SAMPLE_ROOM_URL=https://yourdomain.daily.co/yourroom # (for joining the bot to the same room repeatedly for local dev)
-DAILY_API_KEY=7df...
-OPENAI_API_KEY=sk-PL...
-ELEVENLABS_API_KEY=aeb...

examples/patient-intake/requirements.txt

@@ -1,5 +0,0 @@
-python-dotenv
-requests
-fastapi[all]
-uvicorn
-pipecat-ai[daily,openai,silero]

examples/patient-intake/runner.py

@@ -1,58 +0,0 @@
-import argparse
-import os
-import time
-import urllib
-import requests
-
-
-def configure():
-    parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
-    parser.add_argument(
-        "-u",
-        "--url",
-        type=str,
-        required=False,
-        help="URL of the Daily room to join")
-    parser.add_argument(
-        "-k",
-        "--apikey",
-        type=str,
-        required=False,
-        help="Daily API Key (needed to create an owner token for the room)",
-    )
-
-    args, unknown = parser.parse_known_args()
-
-    url = args.url or os.getenv("DAILY_SAMPLE_ROOM_URL")
-    key = args.apikey or os.getenv("DAILY_API_KEY")
-
-    if not url:
-        raise Exception(
-            "No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL.")
-
-    if not key:
-        raise Exception("No Daily API key specified. use the -k/--apikey option from the command line, or set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers.")
-
-    # Create a meeting token for the given room with an expiration 1 hour in
-    # the future.
-    room_name: str = urllib.parse.urlparse(url).path[1:]
-    expiration: float = time.time() + 60 * 60
-
-    res: requests.Response = requests.post(
-        f"https://api.daily.co/v1/meeting-tokens",
-        headers={
-            "Authorization": f"Bearer {key}"},
-        json={
-            "properties": {
-                "room_name": room_name,
-                "is_owner": True,
-                "exp": expiration}},
-    )
-
-    if res.status_code != 200:
-        raise Exception(
-            f"Failed to create meeting token: {res.status_code} {res.text}")
-
-    token: str = res.json()["token"]
-
-    return (url, token)

examples/patient-intake/server.py

@@ -1,124 +0,0 @@
-import os
-import argparse
-import subprocess
-import atexit
-
-from fastapi import FastAPI, Request, HTTPException
-from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import JSONResponse, RedirectResponse
-
-from utils.daily_helpers import create_room as _create_room, get_token
-
-MAX_BOTS_PER_ROOM = 1
-
-# Bot sub-process dict for status reporting and concurrency control
-bot_procs = {}
-
-
-def cleanup():
-    # Clean up function, just to be extra safe
-    for proc in bot_procs.values():
-        proc.terminate()
-        proc.wait()
-
-
-atexit.register(cleanup)
-
-
-app = FastAPI()
-
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-
-@app.get("/start")
-async def start_agent(request: Request):
-    print(f"!!! Creating room")
-    room_url, room_name = _create_room()
-    print(f"!!! Room URL: {room_url}")
-    # Ensure the room property is present
-    if not room_url:
-        raise HTTPException(
-            status_code=500,
-            detail="Missing 'room' property in request data. Cannot start agent without a target room!")
-
-    # Check if there is already an existing process running in this room
-    num_bots_in_room = sum(
-        1 for proc in bot_procs.values() if proc[1] == room_url and proc[0].poll() is None)
-    if num_bots_in_room >= MAX_BOTS_PER_ROOM:
-        raise HTTPException(
-            status_code=500, detail=f"Max bot limited reach for room: {room_url}")
-
-    # Get the token for the room
-    token = get_token(room_url)
-
-    if not token:
-        raise HTTPException(
-            status_code=500, detail=f"Failed to get token for room: {room_url}")
-
-    # Spawn a new agent, and join the user session
-    # Note: this is mostly for demonstration purposes (refer to 'deployment' in README)
-    try:
-        proc = subprocess.Popen(
-            [
-                f"python3 -m bot -u {room_url} -t {token}"
-            ],
-            shell=True,
-            bufsize=1,
-            cwd=os.path.dirname(os.path.abspath(__file__))
-        )
-        bot_procs[proc.pid] = (proc, room_url)
-    except Exception as e:
-        raise HTTPException(
-            status_code=500, detail=f"Failed to start subprocess: {e}")
-
-    return RedirectResponse(room_url)
-
-
-@app.get("/status/{pid}")
-def get_status(pid: int):
-    # Look up the subprocess
-    proc = bot_procs.get(pid)
-
-    # If the subprocess doesn't exist, return an error
-    if not proc:
-        raise HTTPException(
-            status_code=404, detail=f"Bot with process id: {pid} not found")
-
-    # Check the status of the subprocess
-    if proc[0].poll() is None:
-        status = "running"
-    else:
-        status = "finished"
-
-    return JSONResponse({"bot_id": pid, "status": status})
-
-
-if __name__ == "__main__":
-    import uvicorn
-
-    default_host = os.getenv("HOST", "0.0.0.0")
-    default_port = int(os.getenv("FAST_API_PORT", "7860"))
-
-    parser = argparse.ArgumentParser(
-        description="Daily Storyteller FastAPI server")
-    parser.add_argument("--host", type=str,
-                        default=default_host, help="Host address")
-    parser.add_argument("--port", type=int,
-                        default=default_port, help="Port number")
-    parser.add_argument("--reload", action="store_true",
-                        help="Reload code on change")
-
-    config = parser.parse_args()
-    print(f"to join a test room, visit http://localhost:{config.port}/start")
-    uvicorn.run(
-        "server:app",
-        host=config.host,
-        port=config.port,
-        reload=config.reload,
-    )

examples/patient-intake/utils/daily_helpers.py

@@ -1,109 +0,0 @@
-
-import urllib.parse
-import os
-import time
-import urllib
-import requests
-
-from dotenv import load_dotenv
-load_dotenv()
-
-
-daily_api_path = os.getenv("DAILY_API_URL") or "api.daily.co/v1"
-daily_api_key = os.getenv("DAILY_API_KEY")
-
-
-def create_room() -> tuple[str, str]:
-    """
-    Helper function to create a Daily room.
-    # See: https://docs.daily.co/reference/rest-api/rooms
-
-    Returns:
-        tuple: A tuple containing the room URL and room name.
-
-    Raises:
-        Exception: If the request to create the room fails or if the response does not contain the room URL or room name.
-    """
-    room_props = {
-        "exp": time.time() + 60 * 60,  # 1 hour
-        "enable_chat": True,
-        "enable_emoji_reactions": True,
-        "eject_at_room_exp": True,
-        "enable_prejoin_ui": False,  # Important for the bot to be able to join headlessly
-    }
-    res = requests.post(
-        f"https://{daily_api_path}/rooms",
-        headers={"Authorization": f"Bearer {daily_api_key}"},
-        json={
-            "properties": room_props
-        },
-    )
-    if res.status_code != 200:
-        raise Exception(f"Unable to create room: {res.text}")
-
-    data = res.json()
-    room_url: str = data.get("url")
-    room_name: str = data.get("name")
-    if room_url is None or room_name is None:
-        raise Exception("Missing room URL or room name in response")
-
-    return room_url, room_name
-
-
-def get_name_from_url(room_url: str) -> str:
-    """
-    Extracts the name from a given room URL.
-
-    Args:
-        room_url (str): The URL of the room.
-
-    Returns:
-        str: The extracted name from the room URL.
-    """
-    return urllib.parse.urlparse(room_url).path[1:]
-
-
-def get_token(room_url: str) -> str:
-    """
-    Retrieves a meeting token for the specified Daily room URL.
-    # See: https://docs.daily.co/reference/rest-api/meeting-tokens
-
-    Args:
-        room_url (str): The URL of the Daily room.
-
-    Returns:
-        str: The meeting token.
-
-    Raises:
-        Exception: If no room URL is specified or if no Daily API key is specified.
-        Exception: If there is an error creating the meeting token.
-    """
-    if not room_url:
-        raise Exception(
-            "No Daily room specified. You must specify a Daily room in order a token to be generated.")
-
-    if not daily_api_key:
-        raise Exception(
-            "No Daily API key specified. set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers.")
-
-    expiration: float = time.time() + 60 * 60
-    room_name = get_name_from_url(room_url)
-
-    res: requests.Response = requests.post(
-        f"https://{daily_api_path}/meeting-tokens",
-        headers={
-            "Authorization": f"Bearer {daily_api_key}"},
-        json={
-            "properties": {
-                "room_name": room_name,
-                "is_owner": True,  # Owner tokens required for transcription
-                "exp": expiration}},
-    )
-
-    if res.status_code != 200:
-        raise Exception(
-            f"Failed to create meeting token: {res.status_code} {res.text}")
-
-    token: str = res.json()["token"]
-
-    return token

examples/server/Dockerfile

@@ -7,14 +7,13 @@ COPY *.py /app
 COPY pyproject.toml /app
 
 COPY src/ /app/src/
-COPY examples/ /app/examples/
 
 WORKDIR /app
 RUN ls --recursive /app/
 RUN pip3 install --upgrade -r requirements.txt
 RUN python -m build .
 RUN pip3 install .
-RUN pip3 install gunicorn
+
 # If running on Ubuntu, Azure TTS requires some extra config
 # https://learn.microsoft.com/en-us/azure/ai-services/speech-service/quickstarts/setup-platform?pivots=programming-language-python&tabs=linux%2Cubuntu%2Cdotnetcli%2Cdotnet%2Cjre%2Cmaven%2Cnodejs%2Cmac%2Cpypi
 
@@ -37,4 +36,4 @@ WORKDIR /app
 
 EXPOSE 8000
 # run
-CMD ["gunicorn", "--workers=2", "--log-level", "debug", "--chdir", "examples/server", "--capture-output", "daily-bot-manager:app", "--bind=0.0.0.0:8000"]
+CMD ["gunicorn", "--workers=2", "--log-level", "debug", "--capture-output", "daily-bot-manager:app", "--bind=0.0.0.0:8000"]