go back to using @dataclass since they can be inspected

rebased jpt/readme branch

.dockerignore

@@ -0,0 +1,30 @@
+# flyctl launch added from .gitignore
+**/.vscode
+**/env
+**/__pycache__
+**/*~
+**/venv
+#*#
+
+# Distribution / packaging
+**/.Python
+**/build
+**/develop-eggs
+**/dist
+**/downloads
+**/eggs
+**/.eggs
+**/lib
+**/lib64
+**/parts
+**/sdist
+**/var
+**/wheels
+**/share/python-wheels
+**/*.egg-info
+**/.installed.cfg
+**/*.egg
+**/MANIFEST
+**/.DS_Store
+**/.env
+fly.toml

.github/workflows/build.yaml

@@ -0,0 +1,44 @@
+name: build
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - "**"
+    paths-ignore:
+      - "docs/**"
+
+concurrency:
+  group: build-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: "Build and Install"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        id: setup_python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Setup virtual environment
+        run: |
+          python -m venv .venv
+      - name: Install basic Python dependencies
+        run: |
+          source .venv/bin/activate
+          python -m pip install --upgrade pip
+          pip install -r dev-requirements.txt
+      - name: Build project
+        run: |
+          source .venv/bin/activate
+          python -m build
+      - name: Install project and other Python dependencies
+        run: |
+          source .venv/bin/activate
+          pip install --editable .

.github/workflows/lint.yaml

@@ -0,0 +1,44 @@
+name: lint
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - "**"
+    paths-ignore:
+      - "docs/**"
+
+concurrency:
+  group: build-lint-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  autopep8:
+    name: "Formatting lints"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Setup virtual environment
+        run: |
+          python -m venv .venv
+      - name: Install development Python dependencies
+        run: |
+          source .venv/bin/activate
+          python -m pip install --upgrade pip
+          pip install -r dev-requirements.txt
+      - name: autopep8
+        id: autopep8
+        run: |
+          source .venv/bin/activate
+          autopep8 --max-line-length 100 --exit-code -r -d --exclude "*_pb2.py" -a -a src/
+      - name: Fail if autopep8 requires changes
+        if: steps.autopep8.outputs.exit-code == 2
+        run: exit 1

.github/workflows/publish.yaml

@@ -0,0 +1,84 @@
+name: publish
+
+on:
+  workflow_dispatch:
+    inputs:
+      gitref:
+        type: string
+        description: "what git ref to build"
+        required: true
+
+jobs:
+  build:
+    name: "Build and upload wheels"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.gitref }}
+      - name: Set up Python
+        id: setup_python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Setup virtual environment
+        run: |
+          python -m venv .venv
+      - name: Install basic Python dependencies
+        run: |
+          source .venv/bin/activate
+          python -m pip install --upgrade pip
+          pip install -r dev-requirements.txt
+      - name: Build project
+        run: |
+          source .venv/bin/activate
+          python -m build
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels
+          path: ./dist
+
+  publish-to-pypi:
+    name: "Publish to PyPI"
+    runs-on: ubuntu-latest
+    needs: [ build ]
+    environment:
+      name: pypi
+      url: https://pypi.org/p/pipecat-ai
+    permissions:
+      id-token: write
+    steps:
+      - name: Download wheels
+        uses: actions/download-artifact@v4
+        with:
+          name: wheels
+          path: ./dist
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          verbose: true
+          print-hash: true
+
+  publish-to-test-pypi:
+    name: "Publish to Test PyPI"
+    runs-on: ubuntu-latest
+    needs: [ build ]
+    environment:
+      name: testpypi
+      url: https://pypi.org/p/pipecat-ai
+    permissions:
+      id-token: write
+    steps:
+      - name: Download wheels
+        uses: actions/download-artifact@v4
+        with:
+          name: wheels
+          path: ./dist
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          verbose: true
+          print-hash: true
+          repository-url: https://test.pypi.org/legacy/

.github/workflows/publish_test.yaml

@@ -0,0 +1,63 @@
+name: publish-test
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    name: "Build and upload wheels"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.gitref }}
+          fetch-tags: true
+          fetch-depth: 100
+      - name: Set up Python
+        id: setup_python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Setup virtual environment
+        run: |
+          python -m venv .venv
+      - name: Install basic Python dependencies
+        run: |
+          source .venv/bin/activate
+          python -m pip install --upgrade pip
+          pip install -r dev-requirements.txt
+      - name: Build project
+        run: |
+          source .venv/bin/activate
+          python -m build
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels
+          path: ./dist
+
+  publish-to-pypi:
+    name: "Publish to Test PyPI"
+    runs-on: ubuntu-latest
+    needs: [ build ]
+    environment:
+      name: testpypi
+      url: https://pypi.org/p/pipecat-ai
+    permissions:
+      id-token: write
+    steps:
+      - name: Download wheels
+        uses: actions/download-artifact@v4
+        with:
+          name: wheels
+          path: ./dist
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          verbose: true
+          print-hash: true
+          repository-url: https://test.pypi.org/legacy/

.github/workflows/tests.yaml

@@ -0,0 +1,49 @@
+name: test
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - "**"
+    paths-ignore:
+      - "docs/**"
+
+concurrency:
+  group: build-test-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: "Unit and Integration Tests"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        id: setup_python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Cache virtual environment
+        uses: actions/cache@v3
+        with:
+          # We are hashing requirements-dev.txt and requirements-extra.txt which
+          # contain all dependencies needed to run the tests and examples.
+          key: venv-${{ runner.os }}-${{ steps.setup_python.outputs.python-version}}-${{ hashFiles('linux-py3.10-requirements.txt') }}-${{ hashFiles('dev-requirements.txt') }}
+          path: .venv
+      - name: Install system packages
+        run: sudo apt-get install -y portaudio19-dev
+      - name: Setup virtual environment
+        run: |
+          python -m venv .venv
+      - name: Install basic Python dependencies
+        run: |
+          source .venv/bin/activate
+          python -m pip install --upgrade pip
+          pip install -r linux-py3.10-requirements.txt -r dev-requirements.txt
+      - name: Test with pytest
+        run: |
+          source .venv/bin/activate
+          pytest --doctest-modules --ignore-glob="*to_be_updated*" src tests

.gitignore

@@ -3,6 +3,7 @@ env/
 __pycache__/
 *~
 venv
+.venv
 #*#
 
 # Distribution / packaging
@@ -26,3 +27,4 @@ share/python-wheels/
 MANIFEST
 .DS_Store
 .env
+fly.toml

Dockerfile

@@ -7,13 +7,14 @@ COPY *.py /app
 COPY pyproject.toml /app
 
 COPY src/ /app/src/
+COPY examples/ /app/examples/
 
 WORKDIR /app
 RUN ls --recursive /app/
 RUN pip3 install --upgrade -r requirements.txt
 RUN python -m build .
 RUN pip3 install .
-
+RUN pip3 install gunicorn
 # If running on Ubuntu, Azure TTS requires some extra config
 # https://learn.microsoft.com/en-us/azure/ai-services/speech-service/quickstarts/setup-platform?pivots=programming-language-python&tabs=linux%2Cubuntu%2Cdotnetcli%2Cdotnet%2Cjre%2Cmaven%2Cnodejs%2Cmac%2Cpypi
 
@@ -36,4 +37,4 @@ WORKDIR /app
 
 EXPOSE 8000
 # run
-CMD ["gunicorn", "--workers=2", "--log-level", "debug", "--capture-output", "daily-bot-manager:app", "--bind=0.0.0.0:8000"]
+CMD ["gunicorn", "--workers=2", "--log-level", "debug", "--chdir", "examples/server", "--capture-output", "daily-bot-manager:app", "--bind=0.0.0.0:8000"]

README.md

@@ -1,33 +1,156 @@
-# Daily AI SDK
+<div align="center">
+ <img alt="pipecat" width="300px" height="auto" src="pipecat.png">
+</div>
 
-Build conversational, multi-modal AI apps with real-time voice and video, like this:
+# Pipecat
 
-_Demo Video to come_
+[![PyPI](https://img.shields.io/pypi/v/pipecat-ai)](https://pypi.org/project/pipecat-ai) [![Discord](https://img.shields.io/discord/1239284677165056021
+)](https://discord.gg/pipecat)
 
-With built-in support for many of the best AI platforms (or [add your own](/docs)):
+`pipecat` is a framework for building voice (and multimodal) conversational agents. Things like personal coaches, meeting assistants, story-telling toys for kids, customer support bots, and snarky social companions.
 
-- Azure - DALL-E, ChatGPT, and Azure AI Text-to-Speech
-- Deepgram - Speech-to-text, and Aura text-to-speech
-- Eleven Labs text-to-speech
-- Fal.ai image generation
-- OpenAI DALL-E and ChatGPT
-- Whisper local speech-to-text
+Build things like this:
 
-## Step 1: Get Started
+[![AI-powered voice patient intake for healthcare](https://img.youtube.com/vi/lDevgsp9vn0/0.jpg)](https://www.youtube.com/watch?v=lDevgsp9vn0)
 
-## Build/Install
+## Getting started with voice agents
+
+You can get started with Pipecat running on your local machine, then move your agent processes to the cloud when you’re ready. You can also add a telephone number, image output, video input, use different LLMs, and more.
+
+```shell
+# install the module
+pip install pipecat-ai
+
+# set up an .env file with API keys
+cp dot-env.template .env
+```
+
+By default, in order to minimize dependencies, only the basic framework functionality is available. Some third-party AI services require additional dependencies that you can install with:
+
+```shell
+pip install "pipecat-ai[option,...]"
+```
+
+Your project may or may not need these, so they're made available as optional requirements. Here is a list:
+
+- **AI services**: `anthropic`, `azure`, `fal`, `moondream`, `openai`, `playht`, `silero`, `whisper`
+- **Transports**: `daily`, `local`, `websocket`
+
+## A simple voice agent running locally
+
+If you’re doing AI-related stuff, you probably have an OpenAI API key.
+
+To generate voice output, one service that’s easy to get started with is ElevenLabs. If you don’t already have an ElevenLabs developer account, you can sign up for one [here].
+
+So let’s run a really simple agent that’s just a GPT-4 prompt, wired up to voice input and speaker output.
+
+You can change the prompt, in the code. The current prompt is “Tell me something interesting about the Roman Empire.”
+
+`cd examples/getting-started` to run the following examples …
+
+```shell
+# Talk to a local pipecat process with your voice. Specify GPT-4 as the LLM.
+
+export OPENAI_API_KEY=...
+export ELEVENLABS_API_KEY=...
+python ./local-mic.py | ./pipecat-pipes-gpt-4.py | ./local-speaker.py
+```
+
+## WebSockets instead of pipes
+
+To run your agent in the cloud, you can switch the Pipecat transport layer to use a WebSocket instead of Unix pipes.
+
+```shell
+# Talk to a local pipecat process with your voice. Specify GPT-4 as the LLM.
+
+export OPENAI_API_KEY=...
+export ELEVENLABS_API_KEY=...
+python ./local-mic-and-speaker-wss.py wss://localhost:8088
+```
+
+## WebRTC for production use
+
+WebSockets are fine for server-to-server communication or for initial development. But for production use, you’ll need client-server audio to use a protocol designed for real-time media transport. (For an explanation of the difference between WebSockets and WebRTC, see [this post.])
+
+One way to get up and running quickly with WebRTC is to sign up for a Daily developer account. Daily gives you SDKs and global infrastructure for audio (and video) routing. Every account gets 10,000 audio/video/transcription minutes free each month.
+
+Sign up [here](https://dashboard.daily.co/u/signup) and [create a room](https://docs.daily.co/reference/rest-api/rooms) in the developer Dashboard. Then run the examples, this time connecting via WebRTC instead of a WebSocket.
+
+```shell
+# 1. Run the pipecat process. Provide your Daily API key and a Daily room
+export DAILY_API_KEY=...
+export OPENAI_API_KEY=...
+export ELEVENLABS_API_KEY=...
+python pipecat-daily-gpt-4.py --daily-room https://example.daily.co/pipecat
+
+# 2. Visit the Daily room link in any web browser to talk to the pipecat process.
+#    You'll want to use a Daily SDK to embed the client-side code into your own
+#    app. But visiting the room URL in a browser is a quick way to start building
+#    agents because you can focus on just the agent code at first.
+open -a "Google Chrome" https://example.daily.co/pipecat
+```
+
+## Deploy your agent to the cloud
+Now that you’ve decoupled client and server, and have a Pipecat process that can run anywhere you can run Python, you can deploy this example agent to the cloud.
+
+`TBC`
+
+## Taking it further
+
+### Add a telephone number
+Daily supports telephone connections in addition to WebRTC streams. You can add a telephone number to your Daily room with the following REST API call. Once you’ve done that, you can call your agent on the phone.
+
+You’ll need to add a credit card to your Daily account to enable telephone numbers.
+
+`TBC`
+
+
+### Add image output
+
+Daily supports telephone connections in addition to WebRTC streams. You can add a telephone number to your Daily room with the following REST API call. Once you’ve done that, you can call your agent on the phone.
+
+You’ll need to add a credit card to your Daily account to enable telephone numbers.
+
+`TBC`
+
+### Add video output
+
+
+`TBC`
+
+
+## Code examples
+
+There are two directories of examples:
+
+- [foundational](https://github.com/daily-co/pipecat/tree/main/examples/foundational) — examples that build on each other, introducing one or two concepts at a time
+- [starter apps](https://github.com/daily-co/pipecat/tree/main/examples/starter-apps) — complete applications that you can use as starting points for development
+
+Before running the examples you need to install the dependencies (which will install all the dependencies to run all of the examples):
+
+```
+pip install -r {env}-requirements.txt
+```
+
+To run the example below you need to sign up for a [free Daily account](https://dashboard.daily.co/u/signup) and create a Daily room (so you can hear the LLM talking). After that, join the room's URL directly from a browser tab and run:
+
+```
+python examples/foundational/02-llm-say-one-thing.py
+```
+
+## Hacking on the framework itself
 
 _Note that you may need to set up a virtual environment before following the instructions below. For instance, you might need to run the following from the root of the repo:_
 
 ```
-python3 -m venv env
-source env/bin/activate
+python3 -m venv venv
+source venv/bin/activate
 ```
 
 From the root of this repo, run the following:
 
 ```
-pip install -r requirements.txt
+pip install -r dev-requirements.txt -r {env}-requirements.txt
 python -m build
 ```
 
@@ -43,117 +166,54 @@ If you want to use this package from another directory, you can run:
 pip install path_to_this_repo
 ```
 
-## Running the samples
+### Running tests
 
-Tou can run the simple sample like so:
+From the root directory, run:
 
 ```
-python src/examples/theoretical-to-real/01-say-one-thing.py -u <url of your Daily meeting> -k <your Daily API Key>
-```
-## Overview
-
-The Daily AI SDK allows you to build applications that can participate in WebRTC sessions and interact with AI Services. Some examples of what you can build with this:
-
-- conversational bots that interact 1:1 with a user, using voice recognition and text-to-speech
-- assistant bots that aggregate transcriptions from multiple participants in a meeting and provide realtime summaries or other AI-generated output.
-- image-recognition bots
-- etc
-
-## Concepts
-
-### Transport Service
-
-The SDK provides one “transport service”, which is a wrapper around Daily’s `daily-python` client (tk add link). You can use this service to listen for events related to a WebRTC session, such as “a participant joined the meeting”.
-The transport service also exposes a send queue, and a receive queue. You can use the send queue to send audio and video to the WebRTC session, and you can listen to the receive queue to see audio, video and transcription data from the WebRTC session.
-
-### AI Services
-
-The AI Service classes provide wrappers around various AI providers, and allow you to query LLMs, convert text to speech and make images from text. The audio and images can then be placed on the transport service’s send queue, where they’ll be sent to the WebRTC session.
-
-### Queue Frames
-
-Communication between the transport service and AI services, and between various AI services, takes place in Queue Frames. These frames contain an indication of the type of data as well as the data itself.
-
-## Using Transports, AI Services and Frames
-
-AI Services all define a `.run` method. This method consumes and generates `QueueFrame` frames. The kind of frames that can be consumed and generated depend on the kind of service. For instance, an LLM AI Service consumes `LLM_MESSAGE` frames (which define a history of interaction with an LLM) and emit `TEXT` frames (the response from the LLM).
-
-The `.run` method is an `AsyncIterable`, and it takes an `iterable`, `AsyncIterable` or `asyncio.Queue` that produces QueueFrames as a parameter. This makes it easy to chain AI Services, and consume input from the Transport’s `receive_queue` .
-
-AI Services also have a `.run_to_queue` method. This method is not an AsyncIterable, but instead sends processed QueueFrames to a queue. This makes it easy to send the output of an AI Service to the Transport’s `send_queue`.
-
-AI Services also define convenience functions that let you bypass creating QueueFrames for some simple cases (eg. using the TTS service to convert a string to audio output and send that audio to the transport’s `send_queue`). See below for examples.
-
-## Examples
-
-### Say Something
-
-The base TTS AI service exposes a `.say` method. After creating a transport and TTS service, you can use this method like so:
-
-```
-transport = DailyTransportService(...)
-tts = AzureTTSService()
-await tts.say("hello world", transport.send_queue)
+pytest --doctest-modules --ignore-glob="*to_be_updated*" src tests
 ```
 
-This will call the TTS service to render the text to audio frames, then put the audio frames on the transport’s send queue. The transport will then send those frames along to the WebRTC session.
+## Setting up your editor
 
-### Speak an LLM response
+This project uses strict [PEP 8](https://peps.python.org/pep-0008/) formatting.
 
-Given a system prompt contained in a `messages` array, you can emit the LLM’s response as audio with a chain like this:
+### Emacs
 
-```
-transport = DailyTransportService(...) # setup parameters omitted
-tts = AzureTTSService()
-llm = AzureLLMService()
-messages = [...] # system prompt omitted for brevity
+You can use [use-package](https://github.com/jwiegley/use-package) to install [py-autopep8](https://codeberg.org/ideasman42/emacs-py-autopep8) package and configure `autopep8` arguments:
 
-await tts.run_to_queue(
-  transport.send_queue,
-  llm.run([QueueFrame.LLM_MESSAGES, messages])
-)
+```elisp
+(use-package py-autopep8
+  :ensure t
+  :defer t
+  :hook ((python-mode . py-autopep8-mode))
+  :config
+  (setq py-autopep8-options '("-a" "-a", "--max-line-length=100")))
 ```
 
-In this code, the LLM service object sends the messages to Azure’s OpenAI implementation, which streams chunks back asynchronously. Those chunks are aggregated by the TTS Service to ensure the best audio response (TTS works best when it gets complete sentence, so it can inflect correctly), then sent to Azure’s TTS service, converted to audio frames, and sent to the WebRTC session via the Daily transport.
+`autopep8` was installed in the `venv` environment described before, so you should be able to use [pyvenv-auto](https://github.com/ryotaro612/pyvenv-auto) to automatically load that environment inside Emacs.
 
-### Pre-cache an LLM response
-
-Sometimes LLMs can be slower than we’d like for natural-feeling communication. Here’s an example where we take advantage of the time it takes to speak some pre-defined text to get a head start on the LLM response:
-
-(TK link to 04- sample)
-
-In this sample, we set up a buffer queue to receive the audio frames from the LLM response before while we are joining the call and start an asynchronous task to start filling this buffer:
-
-```
-    buffer_queue = asyncio.Queue()
-    llm_response_task = asyncio.create_task(
-        elevenlabs_tts.run_to_queue(
-            buffer_queue,
-            llm.run([QueueFrame(FrameType.LLM_MESSAGE, messages)]),
-            True,
-        )
-    )
-```
-
-Then, when we’ve joined the call, we speak the static text:
-
-```
-        await azure_tts.say("My friend...", transport.send_queue)
-```
-
-As that text is being spoken, the asynchronous LLM task continues in the background. When the text is done, we pull the frames off the buffer queue and put them in the transport’s `send_queue`:
-
-```
-        async def buffer_to_send_queue():
-            while True:
-                frame = await buffer_queue.get()
-                await transport.send_queue.put(frame)
-                buffer_queue.task_done()
-                if frame.frame_type == FrameType.END_STREAM:
-                    break
-
-        await asyncio.gather(llm_response_task, buffer_to_send_queue())
+```elisp
+(use-package pyvenv-auto
+  :ensure t
+  :defer t
+  :hook ((python-mode . pyvenv-auto-run)))
 
 ```
 
-One thing to note here is the last parameter to `run_to_queue` in the first code clause above: this causes the `run_to_queue` method to send an `END_STREAM` frame when it’s done rendering. This lets us know when to stop our `buffer_to_send_queue` task above.
+### Visual Studio Code
+
+Install the
+[autopep8](https://marketplace.visualstudio.com/items?itemName=ms-python.autopep8) extension. Then edit the user settings (_Ctrl-Shift-P_ `Open User Settings (JSON)`) and set it as the default Python formatter, enable formatting on save and configure `autopep8` arguments:
+
+```json
+"[python]": {
+    "editor.defaultFormatter": "ms-python.autopep8",
+    "editor.formatOnSave": true
+},
+"autopep8.args": [
+    "-a",
+    "-a",
+    "--max-line-length=100"
+],
+```

dev-requirements.txt

@@ -0,0 +1,6 @@
+autopep8~=2.1.0
+build~=1.2.1
+pip-tools~=7.4.1
+pytest~=8.2.0
+setuptools~=69.5.1
+setuptools_scm~=8.1.0

docs/README.md

@@ -1,12 +1,16 @@
-# Daily AI SDK Docs
+# Pipecat Docs
 
 ## [Architecture Overview](architecture.md)
 
-Learn about the thinking behind the SDK's design.
+Learn about the thinking behind the framework's design.
+
+## [A Frame's Progress](frame-progress.md)
+
+See how a Frame is processed through a Transport, a Pipeline, and a series of Frame Processors.
 
 ## [Example Code](examples/)
 
-The repo includes several example apps in the `src/examples` directory. The docs explain how they work.
+The repository includes several example apps in the `examples` directory. The docs explain how they work.
 
 ## [API Reference](api/)
 

docs/architecture.md

@@ -1,2 +1,17 @@
-# Daily AI SDK Architecture Guide
+# Pipecat architecture guide
 
+## Frames
+
+Frames can represent discrete chunks of data, for instance a chunk of text, a chunk of audio, or an image. They can also be used to as control flow, for instance a frame that indicates that there is no more data available, or that a user started or stopped talking. They can also represent more complex data structures, such as a message array used for an LLM completion.
+
+## FrameProcessors
+
+Frame processors operate on frames. Every frame processor implements a `process_frame` method that consumes one frame and produces zero or more frames. Frame processors can do simple transforms, such as concatenating text fragments into sentences, or they can treat frames as input for an AI Service, and emit chat completions based on message arrays or transform text into audio or images.
+
+## Pipelines
+
+Pipelines are lists of frame processors linked together. Frame processors can push frames upstream or downstream to their peers. A very simple pipeline might chain an LLM frame processor to a text-to-speech frame processor, with a transport as an output.
+
+## Transports
+
+Transports provide input and output frame processors to receive or send frames respectively. For example, the `DailyTransport` does this with a WebRTC session joined to a Daily.co room.

docs/examples/01-say-one-thing.md

@@ -16,7 +16,7 @@ if __name__ == "__main__":
 
 ### `configure()`
 
-The `configure()` function comes from `src/examples/foundational/support/runner.py`, and it allows you to configure the examples from the command line directly, or using environment variables:
+The `configure()` function comes from `examples/foundational/support/runner.py`, and it allows you to configure the examples from the command line directly, or using environment variables:
 
 ```bash
 python 01-say-one-thing.py -u https://YOUR_DOMAIN.daily.co/YOUR_ROOM -k YOUR_API_KEY

docs/examples/README.md

@@ -1,5 +1,5 @@
 # Daily AI SDK Examples
 
-The docs in this folder pair with the example apps located in `src/examples/foundational`. They are designed to serve as a quick references for building different kinds of AI apps. But the examples also build on one another, so it can be really helpful to walk through them in order.
+The docs in this folder pair with the example apps located in `examples/foundational`. They are designed to serve as a quick references for building different kinds of AI apps. But the examples also build on one another, so it can be really helpful to walk through them in order.
 
 To start, you can learn about the overall structure of the examples in [01 - Say One Thing](01-say-one-thing.md).

docs/frame-progress.md

@@ -0,0 +1,46 @@
+# A Frame's Progress
+
+1. A user says “Hello, LLM” and the cloud transcription service delivers a transcription to the Transport.
+![A transcript frame arrives](images/frame-progress-01.png)
+
+2. The Transport places a Transcription frame in the Pipeline’s source queue.
+![Frame in source queue](images/frame-progress-02.png)
+
+3. The Pipeline passes the Transcription frame to the first Frame Processor in its list, the LLM User Message Aggregator.
+![To UMA](images/frame-progress-03.png)
+
+4. The LLM User Message Aggregator updates the LLM Context with a `{“user”: “Hello LLM”}` message.
+![Update context](images/frame-progress-04.png)
+
+5. The LLM User Message Aggregator yields an LLM Message Frame, containing the updated LLM Context. The Pipeline passes this frame to the LLM Frame Processor.
+![Update context](images/frame-progress-05.png)
+
+6. The LLM Frame Processor creates a streaming chat completion based on the LLM context and yields the first chunk of a response, Text Frame with the value “Hi, “. The Pipeline passes this frame to the TTS Frame Processor. The TTS Frame Processor aggregates this response but doesn’t yield anything, yet, because it’s waiting for a full sentence.
+![LLM yields Text](images/frame-progress-06.png)
+
+7. The LLM Frame Processor yields another Text Frame with the value “there.”. The Pipeline passes this frame to the TTS Frame Processor.
+![LLM yields more Text](images/frame-progress-07.png)
+
+8. The TTS Frame Processor now has a full sentence, so it starts streaming audio based on “Hi, there.” It yields the first chunk of streaming audio as an Audio frame, which the Pipeline passes to the LLM Assistant Message Aggregator.
+![TTS yields Audio](images/frame-progress-08.png)
+
+9. The LLM Assistant Message Aggregator doesn’t do anything with Audio frames, so it immediately yields the frame, unchanged. This is the convention for all Frame Processors: frames that the processor doesn’t process should be immediately yielded.
+![pass-through](images/frame-progress-09.png)
+
+10. The Pipeline places the first Audio frame in its sink queue, which is being watched by the Transport. Since the frame is now in a queue, the Pipeline can continue processing other frames. Note that the source and sink queues form a sort of “boundary of concurrent processing” between a Pipeline and the outside world. In a Pipeline, Frames are processed sequentially; once a Frame is on a queue it can be processed in parallel with the frames being processed by the Pipeline. TODO: link to a more in-depth section about this.
+![sink queue](images/frame-progress-10.png)
+
+11. The TTS Frame Processor yields another Audio frame as the Transport transmits the first Audio frame.
+![parallel audio](images/frame-progress-11.png)
+
+12. As before, the LLM Assistant Message Aggregator immediately yields the Audio frame and the Pipeline places the Audio frame in the sink queue.
+![sink queue 2](images/frame-progress-12.png)
+
+13. The TTS Frame Processor has no more frames to yield. The LLM Frame Processor emits an LLM Response End Frame, which the Pipeline passes to the TTS Frame Processor.
+![response end](images/frame-progress-13.png)
+
+14. The TTS Frame Processor immediately yields the LLM Response End Frame, so the Pipeline passes it along to the LLM Assistant Message Aggregator. The LLM Assistant Message Aggregator updates the LLM Context with the full response from the LLM. TODO TODO: I realized I forgot that the TSS Frame Processor also yields the Text frames that the LLM emitted so that the LLM Assistant Message Aggregator could accumulate them, arrggh.
+![response end](images/frame-progress-14.png)
+
+15. The system is quiet, and waiting for the next message from the Transport.
+![response end](images/frame-progress-15.png)

dot-env.template

@@ -0,0 +1,35 @@
+# Anthropic
+ANTHROPIC_API_KEY=...
+
+# Azure
+AZURE_SPEECH_REGION=...
+AZURE_SPEECH_API_KEY=...
+
+AZURE_CHATGPT_API_KEY=...
+AZURE_CHATGPT_ENDPOINT=https://...
+AZURE_CHATGPT_MODEL=...
+
+AZURE_DALLE_API_KEY=...
+AZURE_DALLE_ENDPOINT=https://...
+AZURE_DALLE_MODEL=...
+
+# Daily
+DAILY_API_KEY=...
+DAILY_SAMPLE_ROOM_URL=https://...
+
+# ElevenLabs
+ELEVENLABS_API_KEY=...
+ELEVENLABS_VOICE_ID=...
+
+# Fal
+FAL_KEY=...
+
+# Fireworks
+FIREWORKS_API_KEY=...
+
+# PlayHT
+PLAY_HT_USER_ID=...
+PLAY_HT_API_KEY=...
+
+# OpenAI
+OPENAI_API_KEY=...

examples/foundational/01-say-one-thing.py

@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import aiohttp
+import os
+import sys
+
+from pipecat.frames.frames import EndFrame, TextFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.task import PipelineTask
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.services.elevenlabs import ElevenLabsTTSService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+
+from runner import configure
+
+from loguru import logger
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+async def main(room_url):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransport(
+            room_url, None, "Say One Thing", DailyParams(audio_out_enabled=True))
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        runner = PipelineRunner()
+
+        task = PipelineTask(Pipeline([tts, transport.output()]))
+
+        # Register an event handler so we can play the audio when the
+        # participant joins.
+        @transport.event_handler("on_participant_joined")
+        async def on_new_participant_joined(transport, participant):
+            participant_name = participant["info"]["userName"] or ''
+            await task.queue_frames([TextFrame(f"Hello there, {participant_name}!"), EndFrame()])
+
+        await runner.run(task)
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url))

examples/foundational/01a-local-audio.py

@@ -0,0 +1,53 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import aiohttp
+import os
+import sys
+
+from pipecat.frames.frames import EndFrame, TextFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineTask
+from pipecat.services.elevenlabs import ElevenLabsTTSService
+from pipecat.transports.base_transport import TransportParams
+from pipecat.transports.local.audio import LocalAudioTransport
+
+from loguru import logger
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        transport = LocalAudioTransport(TransportParams(audio_out_enabled=True))
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        pipeline = Pipeline([tts, transport.output()])
+
+        task = PipelineTask(pipeline)
+
+        async def say_something():
+            await asyncio.sleep(1)
+            await task.queue_frames([TextFrame("Hello there!"), EndFrame()])
+
+        runner = PipelineRunner()
+
+        await asyncio.gather(runner.run(task), say_something())
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/foundational/02-llm-say-one-thing.py

@@ -0,0 +1,68 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import aiohttp
+import os
+import sys
+
+from pipecat.frames.frames import EndFrame, LLMMessagesFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineTask
+from pipecat.services.elevenlabs import ElevenLabsTTSService
+from pipecat.services.openai import OpenAILLMService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+
+from runner import configure
+
+from loguru import logger
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+async def main(room_url):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransport(
+            room_url,
+            None,
+            "Say One Thing From an LLM",
+            DailyParams(audio_out_enabled=True))
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        messages = [
+            {
+                "role": "system",
+                "content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world.",
+            }]
+
+        runner = PipelineRunner()
+
+        task = PipelineTask(Pipeline([llm, tts, transport.output()]))
+
+        @transport.event_handler("on_first_participant_joined")
+        async def on_first_participant_joined(transport, participant):
+            await task.queue_frames([LLMMessagesFrame(messages), EndFrame()])
+
+        await runner.run(task)
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url))

examples/foundational/03-still-frame.py

@@ -0,0 +1,68 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import aiohttp
+import os
+import sys
+
+from pipecat.frames.frames import TextFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineTask
+from pipecat.services.fal import FalImageGenService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+
+from runner import configure
+
+from loguru import logger
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+async def main(room_url):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransport(
+            room_url,
+            None,
+            "Show a still frame image",
+            DailyParams(
+                camera_out_enabled=True,
+                camera_out_width=1024,
+                camera_out_height=1024
+            )
+        )
+
+        imagegen = FalImageGenService(
+            params=FalImageGenService.InputParams(
+                image_size="square_hd"
+            ),
+            aiohttp_session=session,
+            key=os.getenv("FAL_KEY"),
+        )
+
+        runner = PipelineRunner()
+
+        task = PipelineTask(Pipeline([imagegen, transport.output()]))
+
+        @transport.event_handler("on_first_participant_joined")
+        async def on_first_participant_joined(transport, participant):
+            # Note that we do not put an EndFrame() item in the pipeline for this demo.
+            # This means that the bot will stay in the channel until it times out.
+            # An EndFrame() in the pipeline would cause the transport to shut
+            # down.
+            await task.queue_frames([TextFrame("a cat in the style of picasso")])
+
+        await runner.run(task)
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url))

examples/foundational/03a-local-still-frame.py

@@ -0,0 +1,68 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import aiohttp
+import os
+import sys
+
+import tkinter as tk
+
+from pipecat.frames.frames import TextFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineTask
+from pipecat.services.fal import FalImageGenService
+from pipecat.transports.base_transport import TransportParams
+from pipecat.transports.local.tk import TkLocalTransport
+
+from loguru import logger
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        tk_root = tk.Tk()
+        tk_root.title("Picasso Cat")
+
+        transport = TkLocalTransport(
+            tk_root,
+            TransportParams(
+                camera_out_enabled=True,
+                camera_out_width=1024,
+                camera_out_height=1024))
+
+        imagegen = FalImageGenService(
+            params=FalImageGenService.InputParams(
+                image_size="square_hd"
+            ),
+            aiohttp_session=session,
+            key=os.getenv("FAL_KEY"),
+        )
+
+        pipeline = Pipeline([imagegen, transport.output()])
+
+        task = PipelineTask(pipeline)
+        await task.queue_frames([TextFrame("a cat in the style of picasso")])
+
+        runner = PipelineRunner()
+
+        async def run_tk():
+            while runner.is_active():
+                tk_root.update()
+                tk_root.update_idletasks()
+                await asyncio.sleep(0.1)
+
+        await asyncio.gather(runner.run(task), run_tk())
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/foundational/04-utterance-and-speech.py

@@ -0,0 +1,86 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import aiohttp
+import asyncio
+import os
+import sys
+
+from pipecat.pipeline.merge_pipeline import SequentialMergePipeline
+from pipecat.pipeline.pipeline import Pipeline
+
+from pipecat.frames.frames import EndPipeFrame, LLMMessagesFrame, TextFrame
+from pipecat.pipeline.task import PipelineTask
+from pipecat.services.azure import AzureLLMService, AzureTTSService
+from pipecat.services.elevenlabs import ElevenLabsTTSService
+from pipecat.services.transport_services import TransportServiceOutput
+from pipecat.services.transports.daily_transport import DailyTransport
+
+from runner import configure
+
+from loguru import logger
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+async def main(room_url: str):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransport(room_url, None, "Static And Dynamic Speech")
+
+        meeting = TransportServiceOutput(transport, mic_enabled=True)
+
+        llm = AzureLLMService(
+            api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
+            endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
+            model=os.getenv("AZURE_CHATGPT_MODEL"),
+        )
+        azure_tts = AzureTTSService(
+            api_key=os.getenv("AZURE_SPEECH_API_KEY"),
+            region=os.getenv("AZURE_SPEECH_REGION"),
+        )
+
+        elevenlabs_tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        messages = [{"role": "system",
+                     "content": "tell the user a joke about llamas"}]
+
+        # Start a task to run the LLM to create a joke, and convert the LLM
+        # output to audio frames. This task will run in parallel with generating
+        # and speaking the audio for static text, so there's no delay to speak
+        # the LLM response.
+        llm_pipeline = Pipeline([llm, elevenlabs_tts])
+        llm_task = PipelineTask(llm_pipeline)
+        await llm_task.queue_frames([LLMMessagesFrame(messages), EndPipeFrame()])
+
+        simple_tts_pipeline = Pipeline([azure_tts])
+        await simple_tts_pipeline.queue_frames(
+            [
+                TextFrame("My friend the LLM is going to tell a joke about llamas."),
+                EndPipeFrame(),
+            ]
+        )
+
+        merge_pipeline = SequentialMergePipeline(
+            [simple_tts_pipeline, llm_pipeline])
+
+        await asyncio.gather(
+            transport.run(merge_pipeline),
+            simple_tts_pipeline.run_pipeline(),
+            llm_pipeline.run_pipeline(),
+        )
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url))

examples/foundational/05-sync-speech-and-image.py

@@ -0,0 +1,164 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import aiohttp
+import os
+import sys
+
+from dataclasses import dataclass
+
+from pipecat.frames.frames import (
+    AppFrame,
+    Frame,
+    ImageRawFrame,
+    TextFrame,
+    EndFrame,
+    LLMMessagesFrame,
+    LLMResponseStartFrame,
+)
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineTask
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.processors.aggregators.gated import GatedAggregator
+from pipecat.processors.aggregators.llm_response import LLMFullResponseAggregator
+from pipecat.processors.aggregators.sentence import SentenceAggregator
+from pipecat.processors.aggregators.parallel_task import ParallelTask
+from pipecat.services.openai import OpenAILLMService
+from pipecat.services.elevenlabs import ElevenLabsTTSService
+from pipecat.services.fal import FalImageGenService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+
+from runner import configure
+
+from loguru import logger
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+@dataclass
+class MonthFrame(AppFrame):
+    month: str
+
+    def __str__(self):
+        return f"{self.name}(month: {self.month})"
+
+
+class MonthPrepender(FrameProcessor):
+    def __init__(self):
+        super().__init__()
+        self.most_recent_month = "Placeholder, month frame not yet received"
+        self.prepend_to_next_text_frame = False
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        if isinstance(frame, MonthFrame):
+            self.most_recent_month = frame.month
+        elif self.prepend_to_next_text_frame and isinstance(frame, TextFrame):
+            await self.push_frame(TextFrame(f"{self.most_recent_month}: {frame.text}"))
+            self.prepend_to_next_text_frame = False
+        elif isinstance(frame, LLMResponseStartFrame):
+            self.prepend_to_next_text_frame = True
+            await self.push_frame(frame)
+        else:
+            await self.push_frame(frame, direction)
+
+
+async def main(room_url):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransport(
+            room_url,
+            None,
+            "Month Narration Bot",
+            DailyParams(
+                audio_out_enabled=True,
+                camera_out_enabled=True,
+                camera_out_width=1024,
+                camera_out_height=1024
+            )
+        )
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        imagegen = FalImageGenService(
+            params=FalImageGenService.InputParams(
+                image_size="square_hd"
+            ),
+            aiohttp_session=session,
+            key=os.getenv("FAL_KEY"),
+        )
+
+        gated_aggregator = GatedAggregator(
+            gate_open_fn=lambda frame: isinstance(frame, ImageRawFrame),
+            gate_close_fn=lambda frame: isinstance(frame, LLMResponseStartFrame),
+            start_open=False
+        )
+
+        sentence_aggregator = SentenceAggregator()
+        month_prepender = MonthPrepender()
+        llm_full_response_aggregator = LLMFullResponseAggregator()
+
+        pipeline = Pipeline([
+            llm,
+            sentence_aggregator,
+            ParallelTask(
+                [month_prepender, tts],
+                [llm_full_response_aggregator, imagegen]
+            ),
+            gated_aggregator,
+            transport.output()
+        ])
+
+        frames = []
+        for month in [
+            "January",
+            "February",
+            "March",
+            "April",
+            "May",
+            "June",
+            "July",
+            "August",
+            "September",
+            "October",
+            "November",
+            "December",
+        ]:
+            messages = [
+                {
+                    "role": "system",
+                    "content": f"Describe a nature photograph suitable for use in a calendar, for the month of {month}. Include only the image description with no preamble. Limit the description to one sentence, please.",
+                }
+            ]
+            frames.append(MonthFrame(month=month))
+            frames.append(LLMMessagesFrame(messages))
+
+        frames.append(EndFrame())
+
+        runner = PipelineRunner()
+
+        task = PipelineTask(pipeline)
+
+        await task.queue_frames(frames)
+
+        await runner.run(task)
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url))

examples/foundational/05a-local-sync-speech-and-image.py

@@ -0,0 +1,164 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import aiohttp
+import asyncio
+import os
+import sys
+
+import tkinter as tk
+
+from pipecat.frames.frames import AudioRawFrame, Frame, URLImageRawFrame, LLMMessagesFrame, TextFrame
+from pipecat.pipeline.parallel_pipeline import ParallelPipeline
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineTask
+from pipecat.processors.aggregators.llm_response import LLMFullResponseAggregator
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.services.openai import OpenAILLMService
+from pipecat.services.elevenlabs import ElevenLabsTTSService
+from pipecat.services.fal import FalImageGenService
+from pipecat.transports.base_transport import TransportParams
+from pipecat.transports.local.tk import TkLocalTransport
+
+from loguru import logger
+
+from dotenv import load_dotenv
+
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        tk_root = tk.Tk()
+        tk_root.title("Calendar")
+
+        runner = PipelineRunner()
+
+        async def get_month_data(month):
+            messages = [{"role": "system", "content": f"Describe a nature photograph suitable for use in a calendar, for the month of {month}. Include only the image description with no preamble. Limit the description to one sentence, please.", }]
+
+            class ImageDescription(FrameProcessor):
+                def __init__(self):
+                    super().__init__()
+                    self.text = ""
+
+                async def process_frame(self, frame: Frame, direction: FrameDirection):
+                    if isinstance(frame, TextFrame):
+                        self.text = frame.text
+                    await self.push_frame(frame, direction)
+
+            class AudioGrabber(FrameProcessor):
+                def __init__(self):
+                    super().__init__()
+                    self.audio = bytearray()
+
+                async def process_frame(self, frame: Frame, direction: FrameDirection):
+                    if isinstance(frame, AudioRawFrame):
+                        self.audio.extend(frame.audio)
+                        self.frame = AudioRawFrame(
+                            bytes(self.audio), frame.sample_rate, frame.num_channels)
+
+            class ImageGrabber(FrameProcessor):
+                def __init__(self):
+                    super().__init__()
+                    self.frame = None
+
+                async def process_frame(self, frame: Frame, direction: FrameDirection):
+                    if isinstance(frame, URLImageRawFrame):
+                        self.frame = frame
+
+            llm = OpenAILLMService(
+                api_key=os.getenv("OPENAI_API_KEY"),
+                model="gpt-4-turbo-preview")
+
+            tts = ElevenLabsTTSService(
+                aiohttp_session=session,
+                api_key=os.getenv("ELEVENLABS_API_KEY"),
+                voice_id=os.getenv("ELEVENLABS_VOICE_ID"))
+
+            imagegen = FalImageGenService(
+                params=FalImageGenService.InputParams(
+                    image_size="square_hd"
+                ),
+                aiohttp_session=session,
+                key=os.getenv("FAL_KEY"))
+
+            aggregator = LLMFullResponseAggregator()
+
+            description = ImageDescription()
+
+            audio_grabber = AudioGrabber()
+
+            image_grabber = ImageGrabber()
+
+            pipeline = Pipeline([llm, aggregator, description,
+                                 ParallelPipeline([tts, audio_grabber],
+                                                  [imagegen, image_grabber])])
+
+            task = PipelineTask(pipeline)
+            await task.queue_frame(LLMMessagesFrame(messages))
+            await task.stop_when_done()
+
+            await runner.run(task)
+
+            return {
+                "month": month,
+                "text": description.text,
+                "image": image_grabber.frame,
+                "audio": audio_grabber.frame,
+            }
+
+        transport = TkLocalTransport(
+            tk_root,
+            TransportParams(
+                audio_out_enabled=True,
+                camera_out_enabled=True,
+                camera_out_width=1024,
+                camera_out_height=1024))
+
+        pipeline = Pipeline([transport.output()])
+
+        task = PipelineTask(pipeline)
+
+        # We only specify 5 months as we create tasks all at once and we might
+        # get rate limited otherwise.
+        months: list[str] = [
+            "January",
+            "February",
+            # "March",
+            # "April",
+            # "May",
+        ]
+
+        # We create one task per month. This will be executed concurrently.
+        month_tasks = [asyncio.create_task(get_month_data(month)) for month in months]
+
+        # Now we wait for each month task in the order they're completed. The
+        # benefit is we'll have as little delay as possible before the first
+        # month, and likely no delay between months, but the months won't
+        # display in order.
+        async def show_images(month_tasks):
+            for month_data_task in asyncio.as_completed(month_tasks):
+                data = await month_data_task
+                await task.queue_frames([data["image"], data["audio"]])
+
+            await runner.stop_when_done()
+
+        async def run_tk():
+            while True:
+                tk_root.update()
+                tk_root.update_idletasks()
+                await asyncio.sleep(0.1)
+
+        await asyncio.gather(runner.run(task), show_images(month_tasks), run_tk())
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/foundational/06-listen-and-respond.py

@@ -0,0 +1,94 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import aiohttp
+import os
+import sys
+
+from pipecat.frames.frames import LLMMessagesFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineTask
+from pipecat.processors.aggregators.llm_response import (
+    LLMAssistantResponseAggregator,
+    LLMUserResponseAggregator,
+)
+from pipecat.processors.logger import FrameLogger
+from pipecat.services.elevenlabs import ElevenLabsTTSService
+from pipecat.services.openai import OpenAILLMService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+from pipecat.vad.silero import SileroVAD
+
+from runner import configure
+
+from loguru import logger
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransport(
+            room_url,
+            token,
+            "Respond bot",
+            DailyParams(
+                audio_in_enabled=True,  # This is so Silero VAD can get audio data
+                audio_out_enabled=True,
+                transcription_enabled=True
+            )
+        )
+
+        vad = SileroVAD()
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        fl_in = FrameLogger("Inner")
+        fl_out = FrameLogger("Outer")
+
+        messages = [
+            {
+                "role": "system",
+                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so it should not contain special characters. Respond to what the user said in a creative and helpful way.",
+            },
+        ]
+        tma_in = LLMUserResponseAggregator(messages)
+        tma_out = LLMAssistantResponseAggregator(messages)
+
+        pipeline = Pipeline([fl_in, transport.input(), vad, tma_in, llm,
+                            fl_out, tts, tma_out, transport.output()])
+
+        task = PipelineTask(pipeline)
+
+        @transport.event_handler("on_first_participant_joined")
+        async def on_first_participant_joined(transport, participant):
+            transport.capture_participant_transcription(participant["id"])
+            # Kick off the conversation.
+            messages.append(
+                {"role": "system", "content": "Please introduce yourself to the user."})
+            await task.queue_frames([LLMMessagesFrame(messages)])
+
+        runner = PipelineRunner()
+
+        await runner.run(task)
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/foundational/06a-image-sync.py

@@ -0,0 +1,116 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import aiohttp
+import os
+import sys
+
+from PIL import Image
+
+from pipecat.frames.frames import ImageRawFrame, Frame, SystemFrame, TextFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineTask
+from pipecat.processors.aggregators.llm_context import (
+    LLMAssistantContextAggregator,
+    LLMUserContextAggregator,
+)
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.services.openai import OpenAILLMService
+from pipecat.services.elevenlabs import ElevenLabsTTSService
+from pipecat.transports.services.daily import DailyTransport
+
+from pipecat.transports.services.daily import DailyParams
+from runner import configure
+
+from loguru import logger
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+class ImageSyncAggregator(FrameProcessor):
+    def __init__(self, speaking_path: str, waiting_path: str):
+        super().__init__()
+        self._speaking_image = Image.open(speaking_path)
+        self._speaking_image_format = self._speaking_image.format
+        self._speaking_image_bytes = self._speaking_image.tobytes()
+
+        self._waiting_image = Image.open(waiting_path)
+        self._waiting_image_format = self._waiting_image.format
+        self._waiting_image_bytes = self._waiting_image.tobytes()
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        if not isinstance(frame, SystemFrame):
+            await self.push_frame(ImageRawFrame(image=self._speaking_image_bytes, size=(1024, 1024), format=self._speaking_image_format))
+            await self.push_frame(frame)
+            await self.push_frame(ImageRawFrame(image=self._waiting_image_bytes, size=(1024, 1024), format=self._waiting_image_format))
+        else:
+            await self.push_frame(frame)
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransport(
+            room_url,
+            token,
+            "Respond bot",
+            DailyParams(
+                audio_out_enabled=True,
+                camera_out_width=1024,
+                camera_out_height=1024,
+                transcription_enabled=True
+            )
+        )
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        messages = [
+            {
+                "role": "system",
+                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so it should not contain special characters. Respond to what the user said in a creative and helpful way.",
+            },
+        ]
+
+        tma_in = LLMUserContextAggregator(messages)
+        tma_out = LLMAssistantContextAggregator(messages)
+
+        image_sync_aggregator = ImageSyncAggregator(
+            os.path.join(os.path.dirname(__file__), "assets", "speaking.png"),
+            os.path.join(os.path.dirname(__file__), "assets", "waiting.png"),
+        )
+
+        pipeline = Pipeline([transport.input(), image_sync_aggregator,
+                            tma_in, llm, tma_out, tts, transport.output()])
+
+        task = PipelineTask(pipeline)
+
+        @transport.event_handler("on_first_participant_joined")
+        async def on_first_participant_joined(transport, participant):
+            participant_name = participant["info"]["userName"] or ''
+            transport.capture_participant_transcription(participant["id"])
+            await task.queue_frames([TextFrame(f"Hi, this is {participant_name}.")])
+
+        runner = PipelineRunner()
+
+        await runner.run(task)
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/foundational/07-interruptible.py

@@ -0,0 +1,75 @@
+import asyncio
+import aiohttp
+import logging
+import os
+from pipecat.pipeline.aggregators import (
+    LLMAssistantResponseAggregator,
+    LLMUserResponseAggregator,
+)
+
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.services.ai_services import FrameLogger
+from pipecat.transports.daily_transport import DailyTransport
+from pipecat.services.open_ai_services import OpenAILLMService
+from pipecat.services.elevenlabs_ai_services import ElevenLabsTTSService
+
+from runner import configure
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("pipecat")
+logger.setLevel(logging.DEBUG)
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransport(
+            room_url,
+            token,
+            "Respond bot",
+            duration_minutes=5,
+            start_transcription=True,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            camera_enabled=False,
+            vad_enabled=True,
+        )
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        pipeline = Pipeline([FrameLogger(), llm, FrameLogger(), tts])
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport, participant):
+            await transport.say("Hi, I'm listening!", tts)
+
+        async def run_conversation():
+            messages = [
+                {
+                    "role": "system",
+                    "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
+                },
+            ]
+
+            await transport.run_interruptible_pipeline(
+                pipeline,
+                post_processor=LLMAssistantResponseAggregator(messages),
+                pre_processor=LLMUserResponseAggregator(messages),
+            )
+
+        await asyncio.gather(transport.run(), run_conversation())
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/foundational/08-bots-arguing.py

@@ -0,0 +1,148 @@
+from typing import Tuple
+import aiohttp
+import asyncio
+import logging
+import os
+from pipecat.pipeline.aggregators import SentenceAggregator
+from pipecat.pipeline.pipeline import Pipeline
+
+from pipecat.transports.daily_transport import DailyTransport
+from pipecat.services.azure_ai_services import AzureLLMService, AzureTTSService
+from pipecat.services.elevenlabs_ai_services import ElevenLabsTTSService
+from pipecat.services.fal_ai_services import FalImageGenService
+from pipecat.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame
+
+from runner import configure
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("pipecat")
+logger.setLevel(logging.DEBUG)
+
+
+async def main(room_url: str):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransport(
+            room_url,
+            None,
+            "Respond bot",
+            duration_minutes=10,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            camera_enabled=True,
+            camera_width=1024,
+            camera_height=1024,
+        )
+
+        llm = AzureLLMService(
+            api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
+            endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
+            model=os.getenv("AZURE_CHATGPT_MODEL"),
+        )
+        tts1 = AzureTTSService(
+            api_key=os.getenv("AZURE_SPEECH_API_KEY"),
+            region=os.getenv("AZURE_SPEECH_REGION"),
+        )
+        tts2 = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id="jBpfuIE2acCO8z3wKNLl",
+        )
+        dalle = FalImageGenService(
+            params=FalImageGenService.InputParams(
+                image_size="1024x1024"
+            ),
+            aiohttp_session=session,
+            key=os.getenv("FAL_KEY"),
+        )
+
+        bot1_messages = [
+            {
+                "role": "system",
+                "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long.",
+            },
+        ]
+        bot2_messages = [
+            {
+                "role": "system",
+                "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich.",
+            },
+        ]
+
+        async def get_text_and_audio(messages) -> Tuple[str, bytearray]:
+            """This function streams text from the LLM and uses the TTS service to convert
+             that text to speech as it's received. """
+            source_queue = asyncio.Queue()
+            sink_queue = asyncio.Queue()
+            sentence_aggregator = SentenceAggregator()
+            pipeline = Pipeline(
+                [llm, sentence_aggregator, tts1], source_queue, sink_queue
+            )
+
+            await source_queue.put(LLMMessagesFrame(messages))
+            await source_queue.put(EndFrame())
+            await pipeline.run_pipeline()
+
+            message = ""
+            all_audio = bytearray()
+            while sink_queue.qsize():
+                frame = sink_queue.get_nowait()
+                if isinstance(frame, TextFrame):
+                    message += frame.text
+                elif isinstance(frame, AudioFrame):
+                    all_audio.extend(frame.audio)
+
+            return (message, all_audio)
+
+        async def get_bot1_statement():
+            message, audio = await get_text_and_audio(bot1_messages)
+
+            bot1_messages.append({"role": "assistant", "content": message})
+            bot2_messages.append({"role": "user", "content": message})
+
+            return audio
+
+        async def get_bot2_statement():
+            message, audio = await get_text_and_audio(bot2_messages)
+
+            bot2_messages.append({"role": "assistant", "content": message})
+            bot1_messages.append({"role": "user", "content": message})
+
+            return audio
+
+        async def argue():
+            for i in range(100):
+                print(f"In iteration {i}")
+
+                bot1_description = "A woman conservatively dressed as a librarian in a library surrounded by books, cartoon, serious, highly detailed"
+
+                (audio1, image_data1) = await asyncio.gather(
+                    get_bot1_statement(), dalle.run_image_gen(bot1_description)
+                )
+                await transport.send_queue.put(
+                    [
+                        ImageFrame(image_data1[1], image_data1[2]),
+                        AudioFrame(audio1),
+                    ]
+                )
+
+                bot2_description = "A cat dressed in a hot dog costume, cartoon, bright colors, funny, highly detailed"
+
+                (audio2, image_data2) = await asyncio.gather(
+                    get_bot2_statement(), dalle.run_image_gen(bot2_description)
+                )
+                await transport.send_queue.put(
+                    [
+                        ImageFrame(image_data2[1], image_data2[2]),
+                        AudioFrame(audio2),
+                    ]
+                )
+
+        await asyncio.gather(transport.run(), argue())
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url))

examples/foundational/09-mirror.py

@@ -0,0 +1,62 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import sys
+
+from pipecat.frames.frames import AudioRawFrame, ImageRawFrame
+from pipecat.processors.filter import Filter
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineTask
+from pipecat.pipeline.parallel_pipeline import ParallelPipeline
+from pipecat.transports.services.daily import DailyTransport, DailyParams
+
+from runner import configure
+
+from loguru import logger
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+async def main(room_url, token):
+    transport = DailyTransport(
+        room_url, token, "Test",
+        DailyParams(
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+            camera_out_enabled=True,
+            camera_out_width=1280,
+            camera_out_height=720
+        )
+    )
+
+    @transport.event_handler("on_first_participant_joined")
+    async def on_first_participant_joined(transport, participant):
+        transport.capture_participant_video(participant["id"])
+
+    # The ParallelPipeline is not really necessary here but it shows how you
+    # would process audio and video concurrently in parallel pipelines.
+    pipeline = Pipeline([transport.input(),
+                         ParallelPipeline(
+                             [Filter([AudioRawFrame])],
+                             [Filter([ImageRawFrame])]),
+                         transport.output()])
+
+    runner = PipelineRunner()
+
+    task = PipelineTask(pipeline)
+
+    await runner.run(task)
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/foundational/09a-local-mirror.py

@@ -0,0 +1,65 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import sys
+
+import tkinter as tk
+
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineTask
+from pipecat.transports.base_transport import TransportParams
+from pipecat.transports.local.tk import TkLocalTransport
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+
+from runner import configure
+
+from loguru import logger
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+async def main(room_url, token):
+    tk_root = tk.Tk()
+    tk_root.title("Local Mirror")
+
+    daily_transport = DailyTransport(room_url, token, "Test", DailyParams(audio_in_enabled=True))
+
+    tk_transport = TkLocalTransport(
+        tk_root,
+        TransportParams(
+            audio_out_enabled=True,
+            camera_out_enabled=True,
+            camera_out_width=1280,
+            camera_out_height=720))
+
+    @daily_transport.event_handler("on_first_participant_joined")
+    async def on_first_participant_joined(transport, participant):
+        transport.capture_participant_video(participant["id"])
+
+    pipeline = Pipeline([daily_transport.input(), tk_transport.output()])
+
+    runner = PipelineRunner()
+
+    async def run_tk():
+        while runner.is_active():
+            tk_root.update()
+            tk_root.update_idletasks()
+            await asyncio.sleep(0.1)
+
+    task = PipelineTask(pipeline)
+
+    await asyncio.gather(runner.run(task), run_tk())
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/foundational/10-wake-word.py

@@ -0,0 +1,181 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import aiohttp
+import os
+import random
+import sys
+
+from PIL import Image
+
+from pipecat.frames.frames import (
+    Frame,
+    SystemFrame,
+    TextFrame,
+    ImageRawFrame,
+    SpriteFrame,
+    TranscriptionFrame,
+)
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineTask
+from pipecat.processors.aggregators.llm_context import (
+    LLMUserContextAggregator,
+    LLMAssistantContextAggregator,
+)
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.services.openai import OpenAILLMService
+from pipecat.services.elevenlabs import ElevenLabsTTSService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+
+from runner import configure
+
+from loguru import logger
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+sprites = {}
+image_files = [
+    "sc-default.png",
+    "sc-talk.png",
+    "sc-listen-1.png",
+    "sc-think-1.png",
+    "sc-think-2.png",
+    "sc-think-3.png",
+    "sc-think-4.png",
+]
+
+script_dir = os.path.dirname(__file__)
+
+for file in image_files:
+    # Build the full path to the image file
+    full_path = os.path.join(script_dir, "assets", file)
+    # Get the filename without the extension to use as the dictionary key
+    filename = os.path.splitext(os.path.basename(full_path))[0]
+    # Open the image and convert it to bytes
+    with Image.open(full_path) as img:
+        sprites[file] = ImageRawFrame(image=img.tobytes(), size=img.size, format=img.format)
+
+# When the bot isn't talking, show a static image of the cat listening
+quiet_frame = sprites["sc-listen-1.png"]
+
+# When the bot is talking, build an animation from two sprites
+talking_list = [sprites["sc-default.png"], sprites["sc-talk.png"]]
+talking = [random.choice(talking_list) for x in range(30)]
+talking_frame = SpriteFrame(talking)
+
+# TODO: Support "thinking" as soon as we get a valid transcript, while LLM
+# is processing
+thinking_list = [
+    sprites["sc-think-1.png"],
+    sprites["sc-think-2.png"],
+    sprites["sc-think-3.png"],
+    sprites["sc-think-4.png"],
+]
+thinking_frame = SpriteFrame(thinking_list)
+
+
+class NameCheckFilter(FrameProcessor):
+    def __init__(self, names: list[str]):
+        super().__init__()
+        self._names = names
+        self._sentence = ""
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        if isinstance(frame, SystemFrame):
+            await self.push_frame(frame, direction)
+            return
+
+        content: str = ""
+
+        # TODO: split up transcription by participant
+        if isinstance(frame, TranscriptionFrame):
+            content = frame.text
+            self._sentence += content
+            if self._sentence.endswith((".", "?", "!")):
+                if any(name in self._sentence for name in self._names):
+                    await self.push_frame(TextFrame(self._sentence))
+                    self._sentence = ""
+                else:
+                    self._sentence = ""
+        else:
+            await self.push_frame(frame, direction)
+
+
+class ImageSyncAggregator(FrameProcessor):
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await self.push_frame(talking_frame)
+        await self.push_frame(frame)
+        await self.push_frame(quiet_frame)
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransport(
+            room_url,
+            token,
+            "Santa Cat",
+            DailyParams(
+                audio_out_enabled=True,
+                camera_out_enabled=True,
+                camera_out_width=720,
+                camera_out_height=1280,
+                camera_out_framerate=10,
+                transcription_enabled=True
+            )
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id="jBpfuIE2acCO8z3wKNLl",
+        )
+        isa = ImageSyncAggregator()
+
+        messages = [
+            {
+                "role": "system",
+                "content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while.  Your responses should only be a few sentences long.",
+            },
+        ]
+
+        tma_in = LLMUserContextAggregator(messages)
+        tma_out = LLMAssistantContextAggregator(messages)
+        ncf = NameCheckFilter(["Santa Cat", "Santa"])
+
+        pipeline = Pipeline([transport.input(), isa, ncf, tma_in,
+                            llm, tma_out, tts, transport.output()])
+
+        @transport.event_handler("on_first_participant_joined")
+        async def on_first_participant_joined(transport, participant):
+            # Send some greeting at the beginning.
+            await tts.say("Hi! If you want to talk to me, just say 'hey Santa Cat'.")
+            transport.capture_participant_transcription(participant["id"])
+
+        async def starting_image():
+            await transport.send_image(quiet_frame)
+
+        runner = PipelineRunner()
+
+        task = PipelineTask(pipeline)
+
+        await asyncio.gather(runner.run(task), starting_image())
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/foundational/11-sound-effects.py

@@ -0,0 +1,132 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import aiohttp
+import asyncio
+import os
+import sys
+import wave
+
+from pipecat.frames.frames import (
+    Frame,
+    AudioRawFrame,
+    LLMResponseEndFrame,
+    LLMMessagesFrame,
+)
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineTask
+from pipecat.processors.aggregators.llm_context import (
+    LLMUserContextAggregator,
+    LLMAssistantContextAggregator,
+)
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.processors.logger import FrameLogger
+from pipecat.services.elevenlabs import ElevenLabsTTSService
+from pipecat.services.openai import OpenAILLMService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+
+from runner import configure
+
+from loguru import logger
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+sounds = {}
+sound_files = ["ding1.wav", "ding2.wav"]
+
+script_dir = os.path.dirname(__file__)
+
+for file in sound_files:
+    # Build the full path to the image file
+    full_path = os.path.join(script_dir, "assets", file)
+    # Get the filename without the extension to use as the dictionary key
+    filename = os.path.splitext(os.path.basename(full_path))[0]
+    # Open the image and convert it to bytes
+    with wave.open(full_path) as audio_file:
+        sounds[file] = AudioRawFrame(audio_file.readframes(-1),
+                                     audio_file.getframerate(), audio_file.getnchannels())
+
+
+class OutboundSoundEffectWrapper(FrameProcessor):
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        if isinstance(frame, LLMResponseEndFrame):
+            await self.push_frame(sounds["ding1.wav"])
+            # In case anything else downstream needs it
+            await self.push_frame(frame, direction)
+        else:
+            await self.push_frame(frame, direction)
+
+
+class InboundSoundEffectWrapper(FrameProcessor):
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        if isinstance(frame, LLMMessagesFrame):
+            await self.push_frame(sounds["ding2.wav"])
+            # In case anything else downstream needs it
+            await self.push_frame(frame, direction)
+        else:
+            await self.push_frame(frame, direction)
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransport(
+            room_url,
+            token,
+            "Respond bot",
+            DailyParams(audio_out_enabled=True, transcription_enabled=True)
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id="ErXwobaYiN019PkySvjV",
+        )
+
+        messages = [
+            {
+                "role": "system",
+                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
+            },
+        ]
+
+        tma_in = LLMUserContextAggregator(messages)
+        tma_out = LLMAssistantContextAggregator(messages)
+        out_sound = OutboundSoundEffectWrapper()
+        in_sound = InboundSoundEffectWrapper()
+        fl = FrameLogger("LLM Out")
+        fl2 = FrameLogger("Transcription In")
+
+        pipeline = Pipeline([transport.input(), tma_in, in_sound, fl2, llm,
+                            tma_out, fl, tts, out_sound, transport.output()])
+
+        @transport.event_handler("on_first_participant_joined")
+        async def on_first_participant_joined(transport, participant):
+            transport.capture_participant_transcription(participant["id"])
+            await tts.say("Hi, I'm listening!")
+            await transport.send_audio(sounds["ding1.wav"])
+
+        runner = PipelineRunner()
+
+        task = PipelineTask(pipeline)
+
+        await runner.run(task)
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/foundational/12-describe-video.py

@@ -0,0 +1,104 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import aiohttp
+import os
+import sys
+
+from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineTask
+from pipecat.processors.aggregators.user_response import UserResponseAggregator
+from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.services.elevenlabs import ElevenLabsTTSService
+from pipecat.services.moondream import MoondreamService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+from pipecat.vad.silero import SileroVAD
+
+from runner import configure
+
+from loguru import logger
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+class UserImageRequester(FrameProcessor):
+
+    def __init__(self, participant_id: str | None = None):
+        super().__init__()
+        self._participant_id = participant_id
+
+    def set_participant_id(self, participant_id: str):
+        self._participant_id = participant_id
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        if self._participant_id and isinstance(frame, TextFrame):
+            await self.push_frame(UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM)
+        await self.push_frame(frame, direction)
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransport(
+            room_url,
+            token,
+            "Describe participant video",
+            DailyParams(
+                audio_in_enabled=True,  # This is so Silero VAD can get audio data
+                audio_out_enabled=True,
+                transcription_enabled=True
+            )
+        )
+
+        vad = SileroVAD()
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        user_response = UserResponseAggregator()
+
+        image_requester = UserImageRequester()
+
+        vision_aggregator = VisionImageFrameAggregator()
+
+        # If you run into weird description, try with use_cpu=True
+        moondream = MoondreamService()
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        @transport.event_handler("on_first_participant_joined")
+        async def on_first_participant_joined(transport, participant):
+            await tts.say("Hi there! Feel free to ask me what I see.")
+            transport.capture_participant_video(participant["id"], framerate=0)
+            transport.capture_participant_transcription(participant["id"])
+            image_requester.set_participant_id(participant["id"])
+
+        pipeline = Pipeline([transport.input(), vad, user_response, image_requester,
+                             vision_aggregator, moondream, tts, transport.output()])
+
+        task = PipelineTask(pipeline)
+
+        runner = PipelineRunner()
+
+        await runner.run(task)
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

examples/foundational/13-whisper-transcription.py

@@ -0,0 +1,55 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import sys
+
+from pipecat.frames.frames import Frame, TranscriptionFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineTask
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.services.whisper import WhisperSTTService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+
+from runner import configure
+
+from loguru import logger
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+class TranscriptionLogger(FrameProcessor):
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        if isinstance(frame, TranscriptionFrame):
+            print(f"Transcription: {frame.text}")
+
+
+async def main(room_url: str):
+    transport = DailyTransport(room_url, None, "Transcription bot",
+                               DailyParams(audio_in_enabled=True))
+
+    stt = WhisperSTTService()
+
+    tl = TranscriptionLogger()
+
+    pipeline = Pipeline([transport.input(), stt, tl])
+
+    task = PipelineTask(pipeline)
+
+    runner = PipelineRunner()
+
+    await runner.run(task)
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url))

examples/foundational/13a-whisper-local.py

@@ -0,0 +1,55 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import sys
+
+from pipecat.frames.frames import Frame, TranscriptionFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineTask
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.services.whisper import WhisperSTTService
+from pipecat.transports.base_transport import TransportParams
+from pipecat.transports.local.audio import LocalAudioTransport
+
+from runner import configure
+
+from loguru import logger
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+class TranscriptionLogger(FrameProcessor):
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        if isinstance(frame, TranscriptionFrame):
+            print(f"Transcription: {frame.text}")
+
+
+async def main(room_url: str):
+    transport = LocalAudioTransport(TransportParams(audio_in_enabled=True))
+
+    stt = WhisperSTTService()
+
+    tl = TranscriptionLogger()
+
+    pipeline = Pipeline([transport.input(), stt, tl])
+
+    task = PipelineTask(pipeline)
+
+    runner = PipelineRunner()
+
+    await runner.run(task)
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url))

examples/foundational/runner.py

@@ -4,15 +4,15 @@ import time
 import urllib
 import requests
 
-from dotenv import load_dotenv
-load_dotenv()
-
 
 def configure():
     parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
     parser.add_argument(
-        "-u", "--url", type=str, required=False, help="URL of the Daily room to join"
-    )
+        "-u",
+        "--url",
+        type=str,
+        required=False,
+        help="URL of the Daily room to join")
     parser.add_argument(
         "-k",
         "--apikey",
@@ -33,20 +33,25 @@ def configure():
     if not key:
         raise Exception("No Daily API key specified. use the -k/--apikey option from the command line, or set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers.")
 
-    # Create a meeting token for the given room with an expiration 1 hour in the future.
+    # Create a meeting token for the given room with an expiration 1 hour in
+    # the future.
     room_name: str = urllib.parse.urlparse(url).path[1:]
     expiration: float = time.time() + 60 * 60
 
     res: requests.Response = requests.post(
         f"https://api.daily.co/v1/meeting-tokens",
-        headers={"Authorization": f"Bearer {key}"},
+        headers={
+            "Authorization": f"Bearer {key}"},
         json={
-            "properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
-        },
+            "properties": {
+                "room_name": room_name,
+                "is_owner": True,
+                "exp": expiration}},
     )
 
     if res.status_code != 200:
-        raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
+        raise Exception(
+            f"Failed to create meeting token: {res.status_code} {res.text}")
 
     token: str = res.json()["token"]
 

examples/foundational/websocket-server/frames.proto

@@ -0,0 +1,25 @@
+syntax = "proto3";
+
+package pipecat_proto;
+
+message TextFrame {
+    string text = 1;
+}
+
+message AudioFrame {
+    bytes audio = 1;
+}
+
+message TranscriptionFrame {
+    string text = 1;
+    string participant_id = 2;
+    string timestamp = 3;
+}
+
+message Frame {
+    oneof frame {
+        TextFrame text = 1;
+        AudioFrame audio = 2;
+        TranscriptionFrame transcription = 3;
+    }
+}

examples/foundational/websocket-server/index.html

@@ -0,0 +1,134 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <script src="//cdn.jsdelivr.net/npm/protobufjs@7.X.X/dist/protobuf.min.js"></script>
+    <title>WebSocket Audio Stream</title>
+</head>
+
+<body>
+    <h1>WebSocket Audio Stream</h1>
+    <button id="startAudioBtn">Start Audio</button>
+    <button id="stopAudioBtn">Stop Audio</button>
+    <script>
+        const SAMPLE_RATE = 16000;
+        const BUFFER_SIZE = 8192;
+        const MIN_AUDIO_SIZE = 6400;
+
+        let audioContext;
+        let microphoneStream;
+        let scriptProcessor;
+        let source;
+        let frame;
+        let audioChunks = [];
+        let isPlaying = false;
+        let ws;
+
+        const proto = protobuf.load("frames.proto", (err, root) => {
+            if (err) throw err;
+            frame = root.lookupType("pipecat_proto.Frame");
+        });
+
+        function initWebSocket() {
+            ws = new WebSocket('ws://localhost:8765');
+
+            ws.addEventListener('open', () => console.log('WebSocket connection established.'));
+            ws.addEventListener('message', handleWebSocketMessage);
+            ws.addEventListener('close', (event) => console.log("WebSocket connection closed.", event.code, event.reason));
+            ws.addEventListener('error', (event) => console.error('WebSocket error:', event));
+        }
+
+        async function handleWebSocketMessage(event) {
+            const arrayBuffer = await event.data.arrayBuffer();
+            enqueueAudioFromProto(arrayBuffer);
+        }
+
+        function enqueueAudioFromProto(arrayBuffer) {
+            const parsedFrame = frame.decode(new Uint8Array(arrayBuffer));
+            if (!parsedFrame?.audio) return false;
+
+            const frameCount = parsedFrame.audio.data.length / 2;
+            const audioOutBuffer = audioContext.createBuffer(1, frameCount, SAMPLE_RATE);
+            const nowBuffering = audioOutBuffer.getChannelData(0);
+            const view = new Int16Array(parsedFrame.audio.data.buffer);
+
+            for (let i = 0; i < frameCount; i++) {
+                const word = view[i];
+                nowBuffering[i] = ((word + 32768) % 65536 - 32768) / 32768.0;
+            }
+
+            audioChunks.push(audioOutBuffer);
+            if (!isPlaying) playNextChunk();
+        }
+
+        function playNextChunk() {
+            if (audioChunks.length === 0) {
+                isPlaying = false;
+                return;
+            }
+
+            isPlaying = true;
+            const audioOutBuffer = audioChunks.shift();
+            const source = audioContext.createBufferSource();
+            source.buffer = audioOutBuffer;
+            source.connect(audioContext.destination);
+            source.onended = playNextChunk;
+            source.start();
+        }
+
+        function startAudio() {
+            if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+                alert('getUserMedia is not supported in your browser.');
+                return;
+            }
+
+            navigator.mediaDevices.getUserMedia({ audio: true })
+                .then((stream) => {
+                    microphoneStream = stream;
+                    audioContext = new (window.AudioContext || window.webkitAudioContext)();
+                    scriptProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
+                    source = audioContext.createMediaStreamSource(stream);
+                    source.connect(scriptProcessor);
+                    scriptProcessor.connect(audioContext.destination);
+
+                    const audioBuffer = [];
+                    const skipRatio = Math.floor(audioContext.sampleRate / (SAMPLE_RATE * 2));
+
+                    scriptProcessor.onaudioprocess = (event) => {
+                        const rawLeftChannelData = event.inputBuffer.getChannelData(0);
+                        for (let i = 0; i < rawLeftChannelData.length; i += skipRatio) {
+                            const normalized = ((rawLeftChannelData[i] * 32768.0) + 32768) % 65536 - 32768;
+                            const swappedBytes = ((normalized & 0xff) << 8) | ((normalized >> 8) & 0xff);
+                            audioBuffer.push(swappedBytes);
+                        }
+
+                        if (audioBuffer.length >= MIN_AUDIO_SIZE) {
+                            const audioFrame = frame.create({ audio: { audio: audioBuffer.slice(0, MIN_AUDIO_SIZE) } });
+                            const encodedFrame = new Uint8Array(frame.encode(audioFrame).finish());
+                            ws.send(encodedFrame);
+                            audioBuffer.splice(0, MIN_AUDIO_SIZE);
+                        }
+                    };
+
+                    initWebSocket();
+                })
+                .catch((error) => console.error('Error accessing microphone:', error));
+        }
+
+        function stopAudio() {
+            if (ws) {
+                ws.close();
+                scriptProcessor.disconnect();
+                source.disconnect();
+                ws = undefined;
+            }
+        }
+
+        document.getElementById('startAudioBtn').addEventListener('click', startAudio);
+        document.getElementById('stopAudioBtn').addEventListener('click', stopAudio);
+    </script>
+</body>
+
+</html>

examples/foundational/websocket-server/sample.py

@@ -0,0 +1,50 @@
+import asyncio
+import aiohttp
+import logging
+import os
+from pipecat.pipeline.frame_processor import FrameProcessor
+from pipecat.pipeline.frames import TextFrame, TranscriptionFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.services.elevenlabs_ai_services import ElevenLabsTTSService
+from pipecat.transports.websocket_transport import WebsocketTransport
+from pipecat.services.whisper_ai_services import WhisperSTTService
+
+logging.basicConfig(format="%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("pipecat")
+logger.setLevel(logging.DEBUG)
+
+
+class WhisperTranscriber(FrameProcessor):
+    async def process_frame(self, frame):
+        if isinstance(frame, TranscriptionFrame):
+            print(f"Transcribed: {frame.text}")
+        else:
+            yield frame
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        transport = WebsocketTransport(
+            mic_enabled=True,
+            speaker_enabled=True,
+        )
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        pipeline = Pipeline([
+            WhisperSTTService(),
+            WhisperTranscriber(),
+            tts,
+        ])
+
+        @transport.on_connection
+        async def queue_frame():
+            await pipeline.queue_frames([TextFrame("Hello there!")])
+
+        await transport.run(pipeline)
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/image-gen.py

@@ -5,11 +5,10 @@ import time
 import urllib.parse
 import random
 
-from dailyai.services.daily_transport_service import DailyTransportService
-from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
-from dailyai.queue_frame import QueueFrame, FrameType
-from dailyai.services.fal_ai_services import FalImageGenService
-from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from pipecat.transports.daily_transport import DailyTransport
+from pipecat.services.azure_ai_services import AzureLLMService, AzureTTSService
+from pipecat.pipeline.frames import Frame
+from pipecat.services.fal_ai_services import FalImageGenService
 
 
 async def main(room_url: str, token):
@@ -17,7 +16,7 @@ async def main(room_url: str, token):
     global llm
     global tts
 
-    transport = DailyTransportService(
+    transport = DailyTransport(
         room_url,
         token,
         "Imagebot",
@@ -45,21 +44,24 @@ async def main(room_url: str, token):
             print(f"finder: {finder}")
             if finder >= 0:
                 async for audio in tts.run_tts(f"Resetting."):
-                    transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio))
+                    transport.output_queue.put(
+                        Frame(FrameType.AUDIO_FRAME, audio))
                 sentence = ""
                 continue
-            # todo: we could differentiate between transcriptions from different participants
+            # todo: we could differentiate between transcriptions from
+            # different participants
             sentence += f" {message['text']}"
             print(f"sentence is now: {sentence}")
             # TODO: Cache this audio
-            phrase = random.choice(["OK.", "Got it.", "Sure.", "You bet.", "Sure thing."])
+            phrase = random.choice(
+                ["OK.", "Got it.", "Sure.", "You bet.", "Sure thing."])
             async for audio in tts.run_tts(phrase):
-                transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio))
+                transport.output_queue.put(Frame(FrameType.AUDIO_FRAME, audio))
             img_result = img.run_image_gen(sentence, "1024x1024")
             awaited_img = await asyncio.gather(img_result)
             transport.output_queue.put(
                 [
-                    QueueFrame(FrameType.IMAGE_FRAME, awaited_img[0][1]),
+                    Frame(FrameType.IMAGE_FRAME, awaited_img[0][1]),
                 ]
             )
 
@@ -72,18 +74,19 @@ async def main(room_url: str, token):
             audio_generator = tts.run_tts(
                 f"Hello, {participant['info']['userName']}! Describe an image and I'll create it. To start over, just say 'start over'.")
             async for audio in audio_generator:
-                transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio))
+                transport.output_queue.put(Frame(FrameType.AUDIO_FRAME, audio))
 
-    transport.transcription_settings["extra"]["punctuate"] = False
-    transport.transcription_settings["extra"]["endpointing"] = False
     await asyncio.gather(transport.run(), handle_transcriptions())
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
     parser.add_argument(
-        "-u", "--url", type=str, required=True, help="URL of the Daily room to join"
-    )
+        "-u",
+        "--url",
+        type=str,
+        required=True,
+        help="URL of the Daily room to join")
     parser.add_argument(
         "-k",
         "--apikey",
@@ -94,20 +97,25 @@ if __name__ == "__main__":
 
     args, unknown = parser.parse_known_args()
 
-    # Create a meeting token for the given room with an expiration 1 hour in the future.
+    # Create a meeting token for the given room with an expiration 1 hour in
+    # the future.
     room_name: str = urllib.parse.urlparse(args.url).path[1:]
     expiration: float = time.time() + 60 * 60
 
     res: requests.Response = requests.post(
         f"https://api.daily.co/v1/meeting-tokens",
-        headers={"Authorization": f"Bearer {args.apikey}"},
+        headers={
+            "Authorization": f"Bearer {args.apikey}"},
         json={
-            "properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
-        },
+            "properties": {
+                "room_name": room_name,
+                "is_owner": True,
+                "exp": expiration}},
     )
 
     if res.status_code != 200:
-        raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
+        raise Exception(
+            f"Failed to create meeting token: {res.status_code} {res.text}")
 
     token: str = res.json()["token"]
 

examples/internal/11a-dial-out.py

@@ -3,14 +3,17 @@ import asyncio
 import os
 import wave
 
-from dailyai.services.daily_transport_service import DailyTransportService
-from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
-from dailyai.queue_aggregators import LLMContextAggregator
-from dailyai.services.ai_services import AIService, FrameLogger
-from dailyai.queue_frame import QueueFrame, AudioQueueFrame, LLMResponseEndQueueFrame, LLMMessagesQueueFrame
+from pipecat.transports.daily_transport import DailyTransport
+from pipecat.services.azure_ai_services import AzureLLMService, AzureTTSService
+from pipecat.pipeline.aggregators import LLMContextAggregator
+from pipecat.services.ai_services import AIService, FrameLogger
+from pipecat.pipeline.frames import Frame, AudioFrame, LLMResponseEndFrame, LLMMessagesFrame
 from typing import AsyncGenerator
 
-from examples.foundational.support.runner import configure
+from runner import configure
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
 
 sounds = {}
 sound_files = [
@@ -34,9 +37,9 @@ class OutboundSoundEffectWrapper(AIService):
     def __init__(self):
         pass
 
-    async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
-        if isinstance(frame, LLMResponseEndQueueFrame):
-            yield AudioQueueFrame(sounds["ding1.wav"])
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, LLMResponseEndFrame):
+            yield AudioFrame(sounds["ding1.wav"])
             # In case anything else up the stack needs it
             yield frame
         else:
@@ -47,9 +50,9 @@ class InboundSoundEffectWrapper(AIService):
     def __init__(self):
         pass
 
-    async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
-        if isinstance(frame, LLMMessagesQueueFrame):
-            yield AudioQueueFrame(sounds["ding2.wav"])
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, LLMMessagesFrame):
+            yield AudioFrame(sounds["ding2.wav"])
             # In case anything else up the stack needs it
             yield frame
         else:
@@ -63,7 +66,7 @@ async def main(room_url: str, token, phone):
         global llm
         global tts
 
-        transport = DailyTransportService(
+        transport = DailyTransport(
             room_url,
             token,
             "Respond bot",
@@ -77,9 +80,9 @@ async def main(room_url: str, token, phone):
         tts = AzureTTSService()
 
         @transport.event_handler("on_first_other_participant_joined")
-        async def on_first_other_participant_joined(transport):
+        async def on_first_other_participant_joined(transport, participant):
             await tts.say("Hi, I'm listening!", transport.send_queue)
-            await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"]))
+            await transport.send_queue.put(AudioFrame(sounds["ding1.wav"]))
 
         async def handle_transcriptions():
             messages = [
@@ -124,8 +127,6 @@ async def main(room_url: str, token, phone):
                     transport.start_recording()
                     transport.dialout(phone)
 
-        transport.transcription_settings["extra"]["punctuate"] = True
-
         await asyncio.gather(transport.run(), handle_transcriptions())
 
 

examples/server/README.md

@@ -0,0 +1,34 @@
+# Server Example
+
+Use this server app to quickly host a bot on the web:
+
+```
+flask --app daily-bot-manager.py --debug run
+```
+
+It's currently configured to serve example apps defined in the APPS constant in the server file:
+
+```
+chatbot
+patient-intake
+storybot
+translator
+```
+
+Once the server is started, you can create a bot instance by opening `http://127.0.0.1:5000/start/chatbot` in a browser, and the server will do the following:
+
+- Create a new, randomly-named Daily room with `DAILY_API_KEY` from your .env file or environment
+- Start an instance of `chatbot.py` and connect it to that room
+- 301 redirect your browser to the room
+
+### Options
+
+The server supports several options, which can be set in the body of a POST request, or as params in the URL of a GET request.
+
+- `room_url` (default: none): A room URL to join. If empty, the server will create a Daily room and return the URL in the response.
+  room_properties (none): A JSON object (URL encoded if included as a GET parameter) for overriding default room creation properties, as described here: https://docs.daily.co/reference/rest-api/rooms/create-room This will be ignored if a room_url is provided.
+- `token_properties` (none): A JSON object (URL encoded if included as a GET parameter) for overriding default token properties. By default, the server creates an owner token with an expiration time of one hour.
+- `duration` (7200 seconds, or two hours): Use this property to set a time limit for the bot, as well as an expiration time for the room (if the server is creating one). This will not add an expiration time to an existing room. Expiration times in `token_properties` or `room_properties` will also take precedence over this value. You can set this property to `0` to disable timeouts, but this isn't recommended.
+- `bot_args` (none): A string containing any additional command-line args to pass to the bot.
+- `wait_for_bot` (true): Whether to wait for the bot to successfully join the room before returning a response from the server. If true, the server will start the bot script, then poll the room for up to 5 seconds to confirm the bot has joined the room. If it doesn't, the server will stop the bot and return a 500 response. If set to `false`, the server will start the bot, but immediately return a 200 response. This can be useful if the server is creating rooms for you, and you need the room URL to join the user to the room.
+- `redirect` (true): Instead of returning a 200 for GET requests, the server will return a 301 redirect to the ROOM_URL. This is handy for testing by creating a bot with a GET request directly in the browser. POST requests will never return redirects. Set to `false` to get 200 responses with info in a JSON object even for GET requests.

examples/server/daily-bot-manager.py

@@ -0,0 +1,165 @@
+import os
+import requests
+import urllib
+import subprocess
+import time
+
+from flask import Flask, jsonify, redirect, request
+from flask_cors import CORS
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+app = Flask(__name__)
+CORS(app)
+
+APPS = {
+    "chatbot": "../starter-apps/chatbot.py",
+    "patient-intake": "../starter-apps/patient-intake.py",
+    "storybot": "../starter-apps/storybot.py",
+    "translator": "../starter-apps/translator.py"
+}
+
+daily_api_key = os.getenv("DAILY_API_KEY")
+api_path = os.getenv("DAILY_API_PATH") or "https://api.daily.co/v1"
+
+
+def get_room_name(room_url):
+    return urllib.parse.urlparse(room_url).path[1:]
+
+
+def create_room(room_properties, exp):
+    room_props = {
+        "exp": exp,
+        "enable_chat": True,
+        "enable_emoji_reactions": True,
+        "eject_at_room_exp": True,
+        "enable_prejoin_ui": False,
+        "enable_recording": "cloud"
+    }
+    if room_properties:
+        room_props |= room_properties
+
+    res = requests.post(
+        f"{api_path}/rooms",
+        headers={"Authorization": f"Bearer {daily_api_key}"},
+        json={
+            "properties": room_props
+        },
+    )
+    if res.status_code != 200:
+        raise Exception(f"Unable to create room: {res.text}")
+
+    room_url = res.json()["url"]
+    room_name = res.json()["name"]
+    return (room_url, room_name)
+
+
+def create_token(room_name, token_properties, exp):
+    token_props = {"exp": exp, "is_owner": True}
+    if token_properties:
+        token_props |= token_properties
+    # Force the token to be limited to the room
+    token_props |= {"room_name": room_name}
+    res = requests.post(
+        f'{api_path}/meeting-tokens',
+        headers={
+            'Authorization': f'Bearer {daily_api_key}'},
+        json={
+            'properties': token_props})
+    if res.status_code != 200:
+        if res.status_code != 200:
+            raise Exception(f"Unable to create meeting token: {res.text}")
+
+    meeting_token = res.json()['token']
+    return meeting_token
+
+
+def start_bot(*, bot_path, room_url, token, bot_args, wait_for_bot):
+
+    room_name = get_room_name(room_url)
+    proc = subprocess.Popen(
+        [f"python {bot_path} -u {room_url} -t {token} -k {daily_api_key} {bot_args}"],
+        shell=True,
+        bufsize=1,
+    )
+
+    if wait_for_bot:
+        # Don't return until the bot has joined the room, but wait for at most 5
+        # seconds.
+        attempts = 0
+        while attempts < 50:
+            time.sleep(0.1)
+            attempts += 1
+            res = requests.get(
+                f"{api_path}/rooms/{room_name}/get-session-data",
+                headers={"Authorization": f"Bearer {daily_api_key}"},
+            )
+            if res.status_code == 200:
+                print(f"Took {attempts} attempts to join room {room_name}")
+                return True
+
+        # If we don't break from the loop, that means we never found the bot in the room
+        raise Exception("The bot was unable to join the room. Please try again.")
+
+    return True
+
+
+@app.route("/start/<string:botname>", methods=["GET", "POST"])
+def start(botname):
+    try:
+        if botname not in APPS:
+            raise Exception(f"Bot '{botname}' is not in the allowlist.")
+
+        bot_path = APPS[botname]
+        props = {
+            "room_url": None,
+            "room_properties": None,
+            "token_properties": None,
+            "bot_args": None,
+            "wait_for_bot": True,
+            "duration": None,
+            "redirect": True
+        }
+        props |= request.values.to_dict()  # gets URL params as well as plaintext POST body
+        try:
+            props |= request.json
+        except BaseException:
+            pass
+        if props['redirect'] == "false":
+            props['redirect'] = False
+        if props['wait_for_bot'] == "false":
+            props['wait_for_bot'] = False
+
+        duration = int(os.getenv("DAILY_BOT_DURATION") or 7200)
+        if props['duration']:
+            duration = props['duration']
+        exp = time.time() + duration
+        if (props['room_url']):
+            room_url = props['room_url']
+            try:
+                room_name = get_room_name(room_url)
+            except ValueError:
+                raise Exception(
+                    "There was a problem detecting the room name. Please double-check the value of room_url.")
+        else:
+            room_url, room_name = create_room(props['room_properties'], exp)
+        token = create_token(room_name, props['token_properties'], exp)
+        bot = start_bot(
+            room_url=room_url,
+            bot_path=bot_path,
+            token=token,
+            bot_args=props['bot_args'],
+            wait_for_bot=props['wait_for_bot'])
+
+        if props['redirect'] and request.method == "GET":
+            return redirect(room_url, 302)
+        else:
+            return jsonify({"room_url": room_url, "token": token})
+    except BaseException as e:
+        return f"There was a problem starting the bot: {e}", 500
+
+
+@app.route("/healthz")
+def health_check():
+    return "ok", 200