added fuzz example

.dockerignore

@@ -1,30 +0,0 @@
-# flyctl launch added from .gitignore
-**/.vscode
-**/env
-**/__pycache__
-**/*~
-**/venv
-#*#
-
-# Distribution / packaging
-**/.Python
-**/build
-**/develop-eggs
-**/dist
-**/downloads
-**/eggs
-**/.eggs
-**/lib
-**/lib64
-**/parts
-**/sdist
-**/var
-**/wheels
-**/share/python-wheels
-**/*.egg-info
-**/.installed.cfg
-**/*.egg
-**/MANIFEST
-**/.DS_Store
-**/.env
-fly.toml

.github/workflows/build.yaml

@@ -1,44 +0,0 @@
-name: build
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - "**"
-    paths-ignore:
-      - "docs/**"
-
-concurrency:
-  group: build-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    name: "Build and Install"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python
-        id: setup_python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-      - name: Setup virtual environment
-        run: |
-          python -m venv .venv
-      - name: Install basic Python dependencies
-        run: |
-          source .venv/bin/activate
-          python -m pip install --upgrade pip
-          pip install -r dev-requirements.txt
-      - name: Build project
-        run: |
-          source .venv/bin/activate
-          python -m build
-      - name: Install project and other Python dependencies
-        run: |
-          source .venv/bin/activate
-          pip install --editable .

.github/workflows/lint.yaml

@@ -22,23 +22,11 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-      - name: Setup virtual environment
-        run: |
-          python -m venv .venv
-      - name: Install development Python dependencies
-        run: |
-          source .venv/bin/activate
-          python -m pip install --upgrade pip
-          pip install -r dev-requirements.txt
       - name: autopep8
         id: autopep8
-        run: |
-          source .venv/bin/activate
-          autopep8 --max-line-length 100 --exit-code -r -d --exclude "*_pb2.py" -a -a src/
+        uses: peter-evans/autopep8@v2
+        with:
+          args: --exit-code -r -d -a -a src/
       - name: Fail if autopep8 requires changes
         if: steps.autopep8.outputs.exit-code == 2
         run: exit 1

.github/workflows/publish.yaml

@@ -1,84 +0,0 @@
-name: publish
-
-on:
-  workflow_dispatch:
-    inputs:
-      gitref:
-        type: string
-        description: "what git ref to build"
-        required: true
-
-jobs:
-  build:
-    name: "Build and upload wheels"
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.inputs.gitref }}
-      - name: Set up Python
-        id: setup_python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-      - name: Setup virtual environment
-        run: |
-          python -m venv .venv
-      - name: Install basic Python dependencies
-        run: |
-          source .venv/bin/activate
-          python -m pip install --upgrade pip
-          pip install -r dev-requirements.txt
-      - name: Build project
-        run: |
-          source .venv/bin/activate
-          python -m build
-      - name: Upload wheels
-        uses: actions/upload-artifact@v4
-        with:
-          name: wheels
-          path: ./dist
-
-  publish-to-pypi:
-    name: "Publish to PyPI"
-    runs-on: ubuntu-latest
-    needs: [ build ]
-    environment:
-      name: pypi
-      url: https://pypi.org/p/dailyai
-    permissions:
-      id-token: write
-    steps:
-      - name: Download wheels
-        uses: actions/download-artifact@v4
-        with:
-          name: wheels
-          path: ./dist
-      - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          verbose: true
-          print-hash: true
-
-  publish-to-test-pypi:
-    name: "Publish to Test PyPI"
-    runs-on: ubuntu-latest
-    needs: [ build ]
-    environment:
-      name: testpypi
-      url: https://pypi.org/p/dailyai
-    permissions:
-      id-token: write
-    steps:
-      - name: Download wheels
-        uses: actions/download-artifact@v4
-        with:
-          name: wheels
-          path: ./dist
-      - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          verbose: true
-          print-hash: true
-          repository-url: https://test.pypi.org/legacy/

.github/workflows/publish_test.yaml

@@ -1,63 +0,0 @@
-name: publish-test
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-
-jobs:
-  build:
-    name: "Build and upload wheels"
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.inputs.gitref }}
-          fetch-tags: true
-          fetch-depth: 100
-      - name: Set up Python
-        id: setup_python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-      - name: Setup virtual environment
-        run: |
-          python -m venv .venv
-      - name: Install basic Python dependencies
-        run: |
-          source .venv/bin/activate
-          python -m pip install --upgrade pip
-          pip install -r dev-requirements.txt
-      - name: Build project
-        run: |
-          source .venv/bin/activate
-          python -m build
-      - name: Upload wheels
-        uses: actions/upload-artifact@v4
-        with:
-          name: wheels
-          path: ./dist
-
-  publish-to-pypi:
-    name: "Publish to Test PyPI"
-    runs-on: ubuntu-latest
-    needs: [ build ]
-    environment:
-      name: testpypi
-      url: https://pypi.org/p/dailyai
-    permissions:
-      id-token: write
-    steps:
-      - name: Download wheels
-        uses: actions/download-artifact@v4
-        with:
-          name: wheels
-          path: ./dist
-      - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          verbose: true
-          print-hash: true
-          repository-url: https://test.pypi.org/legacy/

.github/workflows/tests.yaml

@@ -1,49 +0,0 @@
-name: test
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - "**"
-    paths-ignore:
-      - "docs/**"
-
-concurrency:
-  group: build-test-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  test:
-    name: "Unit and Integration Tests"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python
-        id: setup_python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-      - name: Cache virtual environment
-        uses: actions/cache@v3
-        with:
-          # We are hashing requirements-dev.txt and requirements-extra.txt which
-          # contain all dependencies needed to run the tests and examples.
-          key: venv-${{ runner.os }}-${{ steps.setup_python.outputs.python-version}}-${{ hashFiles('linux-py3.10-requirements.txt') }}-${{ hashFiles('dev-requirements.txt') }}
-          path: .venv
-      - name: Install system packages
-        run: sudo apt-get install -y portaudio19-dev
-      - name: Setup virtual environment
-        run: |
-          python -m venv .venv
-      - name: Install basic Python dependencies
-        run: |
-          source .venv/bin/activate
-          python -m pip install --upgrade pip
-          pip install -r linux-py3.10-requirements.txt -r dev-requirements.txt
-      - name: Test with pytest
-        run: |
-          source .venv/bin/activate
-          pytest --doctest-modules --ignore-glob="*to_be_updated*" src tests

.gitignore

@@ -3,7 +3,6 @@ env/
 __pycache__/
 *~
 venv
-.venv
 #*#
 
 # Distribution / packaging
@@ -27,4 +26,3 @@ share/python-wheels/
 MANIFEST
 .DS_Store
 .env
-fly.toml

README.md

@@ -4,8 +4,12 @@ Build things like this:
 
 [![AI-powered voice patient intake for healthcare](https://img.youtube.com/vi/lDevgsp9vn0/0.jpg)](https://www.youtube.com/watch?v=lDevgsp9vn0)
 
+
+
+
 **`dailyai` started as a toolkit for implementing generative AI voice bots.** Things like personal coaches, meeting assistants, story-telling toys for kids, customer support bots, and snarky social companions.
 
+
 In 2023 a *lot* of us got excited about the possibility of having open-ended conversations with LLMs. It became clear pretty quickly that we were all solving the same [low-level problems](https://www.daily.co/blog/how-to-talk-to-an-llm-with-your-voice/):
 - low-latency, reliable audio transport
 - echo cancellation
@@ -39,14 +43,12 @@ Currently implemented services:
 - Transport
   - Daily
   - Local (in progress, intended as a quick start example service)
-- Vision
-  - Moondream
 
 If you'd like to [implement a service]((https://github.com/daily-co/daily-ai-sdk/tree/main/src/dailyai/services)), we welcome PRs! Our goal is to support lots of services in all of the above categories, plus new categories (like real-time video) as they emerge.
 
-## Getting started
+## Step 1: Get started
 
-Today, the easiest way to get started with `dailyai` is to use [Daily](https://www.daily.co/) as your transport service. This toolkit started life as an internal SDK at Daily and millions of minutes of AI conversation have been served using it and its earlier prototype incarnations. (The [transport base class](https://github.com/daily-co/daily-ai-sdk/blob/main/src/dailyai/transports/abstract_transport.py) is easy to extend, though, so feel free to submit PRs if you'd like to implement another transport service.)
+Today, the easiest way to get started with `dailyai` is to use [Daily](https://www.daily.co/) as your transport service. This toolkit started life as an internal SDK at Daily and millions of minutes of AI conversation have been served using it and its earlier prototype incarnations. (The [transport base class](https://github.com/daily-co/daily-ai-sdk/blob/main/src/dailyai/services/base_transport_service.py) is easy to extend, though, so feel free to submit PRs if you'd like to implement another transport service.)
 
 ```
 # install the module
@@ -54,52 +56,35 @@ pip install dailyai
 
 # set up an .env file with API keys
 cp dot-env.template .env
+
+# sign up for a free Daily account, if you don't already have one, and
+# join the Daily room URL directly from a browser tab, then run one of the
+# samples
+python src/examples/foundational/02-llm-say-one-thing.py
 ```
 
-By default, in order to minimize dependencies, only the basic framework functionality is available. Some third-party AI services require additional
-dependencies that you can install with:
-
-```
-pip install "dailyai[option,...]"
-```
-
-Your project may or may not need these, so they're made available as optional requirements. Here is a list:
-
-- **AI services**: `anthropic`, `azure`, `fal`, `moondream`, `openai`, `playht`, `silero`, `whisper`
-- **Transports**: `daily`, `local`, `websocket`
-
 ## Code examples
 
 There are two directories of examples:
 
-- [foundational](https://github.com/daily-co/daily-ai-sdk/tree/main/examples/foundational) — demos that build on each other, introducing one or two concepts at a time
-- [starter apps](https://github.com/daily-co/daily-ai-sdk/tree/main/examples/starter-apps) — complete applications that you can use as starting points for development
+- [foundational](https://github.com/daily-co/daily-ai-sdk/tree/main/src/examples/foundational) — demos that build on each other, introducing one or two concepts at a time
+- [starter apps](https://github.com/daily-co/daily-ai-sdk/tree/main/src/examples/starter-apps) — complete applications that you can use as starting points for development
 
-Before running the examples you need to install the dependencies (which will install all the dependencies to run all of the examples):
 
-```
-pip install -r {env}-requirements.txt
-```
-
-To run the example below you need to sign up for a [free Daily account](https://dashboard.daily.co/u/signup) and create a Daily room (so you can hear the LLM talking). After that, join the room's URL directly from a browser tab and run:
-
-```
-python examples/foundational/02-llm-say-one-thing.py
-```
 
 ## Hacking on the framework itself
 
 _Note that you may need to set up a virtual environment before following the instructions below. For instance, you might need to run the following from the root of the repo:_
 
 ```
-python3 -m venv venv
-source venv/bin/activate
+python3 -m venv env
+source env/bin/activate
 ```
 
 From the root of this repo, run the following:
 
 ```
-pip install -r {env}-requirements.txt -r dev-requirements.txt
+pip install -r requirements.txt
 python -m build
 ```
 
@@ -114,55 +99,3 @@ If you want to use this package from another directory, you can run:
 ```
 pip install path_to_this_repo
 ```
-
-### Running tests
-
-From the root directory, run:
-
-```
-pytest --doctest-modules --ignore-glob="*to_be_updated*" src tests
-```
-
-## Setting up your editor
-
-This project uses strict [PEP 8](https://peps.python.org/pep-0008/) formatting.
-
-### Emacs
-
-You can use [use-package](https://github.com/jwiegley/use-package) to install [py-autopep8](https://codeberg.org/ideasman42/emacs-py-autopep8) package and configure `autopep8` arguments:
-
-```elisp
-(use-package py-autopep8
-  :ensure t
-  :defer t
-  :hook ((python-mode . py-autopep8-mode))
-  :config
-  (setq py-autopep8-options '("-a" "-a", "--max-line-length=100")))
-```
-
-`autopep8` was installed in the `venv` environment described before, so you should be able to use [pyvenv-auto](https://github.com/ryotaro612/pyvenv-auto) to automatically load that environment inside Emacs.
-
-```elisp
-(use-package pyvenv-auto
-  :ensure t
-  :defer t
-  :hook ((python-mode . pyvenv-auto-run)))
-
-```
-
-### Visual Studio Code
-
-Install the
-[autopep8](https://marketplace.visualstudio.com/items?itemName=ms-python.autopep8) extension. Then edit the user settings (_Ctrl-Shift-P_ `Open User Settings (JSON)`) and set it as the default Python formatter, enable formatting on save and configure `autopep8` arguments:
-
-```json
-"[python]": {
-    "editor.defaultFormatter": "ms-python.autopep8",
-    "editor.formatOnSave": true
-},
-"autopep8.args": [
-    "-a",
-    "-a",
-    "--max-line-length=100"
-],
-```

dev-requirements.txt

@@ -1,6 +0,0 @@
-autopep8==2.0.4
-build==1.0.3
-pip-tools==7.4.1
-pytest==8.1.1
-setuptools==69.2.0
-setuptools_scm==8.0.4

docs/README.md

@@ -4,13 +4,9 @@
 
 Learn about the thinking behind the SDK's design.
 
-## [A Frame's Progress](frame-progress.md)
-
-See how a Frame is processed through a Transport, a Pipeline, and a series of Frame Processors.
-
 ## [Example Code](examples/)
 
-The repo includes several example apps in the `examples` directory. The docs explain how they work.
+The repo includes several example apps in the `src/examples` directory. The docs explain how they work.
 
 ## [API Reference](api/)
 

docs/examples/01-say-one-thing.md

@@ -16,7 +16,7 @@ if __name__ == "__main__":
 
 ### `configure()`
 
-The `configure()` function comes from `examples/foundational/support/runner.py`, and it allows you to configure the examples from the command line directly, or using environment variables:
+The `configure()` function comes from `src/examples/foundational/support/runner.py`, and it allows you to configure the examples from the command line directly, or using environment variables:
 
 ```bash
 python 01-say-one-thing.py -u https://YOUR_DOMAIN.daily.co/YOUR_ROOM -k YOUR_API_KEY

docs/examples/README.md

@@ -1,5 +1,5 @@
 # Daily AI SDK Examples
 
-The docs in this folder pair with the example apps located in `examples/foundational`. They are designed to serve as a quick references for building different kinds of AI apps. But the examples also build on one another, so it can be really helpful to walk through them in order.
+The docs in this folder pair with the example apps located in `src/examples/foundational`. They are designed to serve as a quick references for building different kinds of AI apps. But the examples also build on one another, so it can be really helpful to walk through them in order.
 
 To start, you can learn about the overall structure of the examples in [01 - Say One Thing](01-say-one-thing.md).

docs/frame-progress.md

@@ -1,46 +0,0 @@
-# A Frame's Progress
-
-1. A user says “Hello, LLM” and the cloud transcription service delivers a transcription to the Transport.
-![A transcript frame arrives](images/frame-progress-01.png)
-
-2. The Transport places a Transcription frame in the Pipeline’s source queue.
-![Frame in source queue](images/frame-progress-02.png)
-
-3. The Pipeline passes the Transcription frame to the first Frame Processor in its list, the LLM User Message Aggregator.
-![To UMA](images/frame-progress-03.png)
-
-4. The LLM User Message Aggregator updates the LLM Context with a `{“user”: “Hello LLM”}` message.
-![Update context](images/frame-progress-04.png)
-
-5. The LLM User Message Aggregator yields an LLM Message Frame, containing the updated LLM Context. The Pipeline passes this frame to the LLM Frame Processor.
-![Update context](images/frame-progress-05.png)
-
-6. The LLM Frame Processor creates a streaming chat completion based on the LLM context and yields the first chunk of a response, Text Frame with the value “Hi, “. The Pipeline passes this frame to the TTS Frame Processor. The TTS Frame Processor aggregates this response but doesn’t yield anything, yet, because it’s waiting for a full sentence.
-![LLM yields Text](images/frame-progress-06.png)
-
-7. The LLM Frame Processor yields another Text Frame with the value “there.”. The Pipeline passes this frame to the TTS Frame Processor.
-![LLM yields more Text](images/frame-progress-07.png)
-
-8. The TTS Frame Processor now has a full sentence, so it starts streaming audio based on “Hi, there.” It yields the first chunk of streaming audio as an Audio frame, which the Pipeline passes to the LLM Assistant Message Aggregator.
-![TTS yields Audio](images/frame-progress-08.png)
-
-9. The LLM Assistant Message Aggregator doesn’t do anything with Audio frames, so it immediately yields the frame, unchanged. This is the convention for all Frame Processors: frames that the processor doesn’t process should be immediately yielded.
-![pass-through](images/frame-progress-09.png)
-
-10. The Pipeline places the first Audio frame in its sink queue, which is being watched by the Transport. Since the frame is now in a queue, the Pipeline can continue processing other frames. Note that the source and sink queues form a sort of “boundary of concurrent processing” between a Pipeline and the outside world. In a Pipeline, Frames are processed sequentially; once a Frame is on a queue it can be processed in parallel with the frames being processed by the Pipeline. TODO: link to a more in-depth section about this.
-![sink queue](images/frame-progress-10.png)
-
-11. The TTS Frame Processor yields another Audio frame as the Transport transmits the first Audio frame.
-![parallel audio](images/frame-progress-11.png)
-
-12. As before, the LLM Assistant Message Aggregator immediately yields the Audio frame and the Pipeline places the Audio frame in the sink queue.
-![sink queue 2](images/frame-progress-12.png)
-
-13. The TTS Frame Processor has no more frames to yield. The LLM Frame Processor emits an LLM Response End Frame, which the Pipeline passes to the TTS Frame Processor.
-![response end](images/frame-progress-13.png)
-
-14. The TTS Frame Processor immediately yields the LLM Response End Frame, so the Pipeline passes it along to the LLM Assistant Message Aggregator. The LLM Assistant Message Aggregator updates the LLM Context with the full response from the LLM. TODO TODO: I realized I forgot that the TSS Frame Processor also yields the Text frames that the LLM emitted so that the LLM Assistant Message Aggregator could accumulate them, arrggh.
-![response end](images/frame-progress-14.png)
-
-15. The system is quiet, and waiting for the next message from the Transport.
-![response end](images/frame-progress-15.png)

dot-env.template

@@ -1,33 +1,5 @@
-# Anthropic
-ANTHROPIC_API_KEY=...
-
-# Azure
-AZURE_SPEECH_REGION=...
-AZURE_SPEECH_API_KEY=...
-
-AZURE_CHATGPT_API_KEY=...
-AZURE_CHATGPT_ENDPOINT=https://...
-AZURE_CHATGPT_MODEL=...
-
-AZURE_DALLE_API_KEY=...
-AZURE_DALLE_ENDPOINT=https://...
-AZURE_DALLE_MODEL=...
-
-# Daily
-DAILY_API_KEY=...
-DAILY_SAMPLE_ROOM_URL=https://...
-
-# ElevenLabs
+OPENAI_API_KEY=...
 ELEVENLABS_API_KEY=...
 ELEVENLABS_VOICE_ID=...
-
-# Fal
-FAL_KEY_ID=...
-FAL_KEY_SECRET=...
-
-# PlayHT
-PLAY_HT_USER_ID=...
-PLAY_HT_API_KEY=...
-
-# OpenAI
-OPENAI_API_KEY=...
+DAILY_API_KEY=...
+DAILY_SAMPLE_ROOM_URL=https://...

examples/foundational/06a-image-sync.py

@@ -1,96 +0,0 @@
-import asyncio
-import os
-import logging
-from typing import AsyncGenerator
-import aiohttp
-from PIL import Image
-
-from dailyai.pipeline.frames import ImageFrame, Frame, TextFrame
-from dailyai.pipeline.pipeline import Pipeline
-from dailyai.transports.daily_transport import DailyTransport
-from dailyai.services.ai_services import AIService
-from dailyai.pipeline.aggregators import (
-    LLMAssistantContextAggregator,
-    LLMUserContextAggregator,
-)
-from dailyai.services.open_ai_services import OpenAILLMService
-from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
-
-from runner import configure
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
-logger = logging.getLogger("dailyai")
-logger.setLevel(logging.DEBUG)
-
-
-class ImageSyncAggregator(AIService):
-    def __init__(self, speaking_path: str, waiting_path: str):
-        self._speaking_image = Image.open(speaking_path)
-        self._speaking_image_bytes = self._speaking_image.tobytes()
-
-        self._waiting_image = Image.open(waiting_path)
-        self._waiting_image_bytes = self._waiting_image.tobytes()
-
-    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
-        yield ImageFrame(self._speaking_image_bytes, (1024, 1024))
-        yield frame
-        yield ImageFrame(self._waiting_image_bytes, (1024, 1024))
-
-
-async def main(room_url: str, token):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Respond bot",
-            5,
-        )
-        transport._camera_enabled = True
-        transport._camera_width = 1024
-        transport._camera_height = 1024
-        transport._mic_enabled = True
-        transport._mic_sample_rate = 16000
-        transport.transcription_settings["extra"]["punctuate"] = True
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
-        )
-
-        llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4-turbo-preview")
-
-        messages = [
-            {
-                "role": "system",
-                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so it should not include any special characters. Respond to what the user said in a creative and helpful way.",
-            },
-        ]
-
-        tma_in = LLMUserContextAggregator(
-            messages, transport._my_participant_id)
-        tma_out = LLMAssistantContextAggregator(
-            messages, transport._my_participant_id
-        )
-        image_sync_aggregator = ImageSyncAggregator(
-            os.path.join(os.path.dirname(__file__), "assets", "speaking.png"),
-            os.path.join(os.path.dirname(__file__), "assets", "waiting.png"),
-        )
-
-        pipeline = Pipeline([image_sync_aggregator, tma_in, llm, tma_out, tts])
-
-        @transport.event_handler("on_first_other_participant_joined")
-        async def on_first_other_participant_joined(transport, participant):
-            await pipeline.queue_frames([TextFrame("Hi, I'm listening!")])
-
-        await transport.run(pipeline)
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/12-describe-video.py

@@ -1,84 +0,0 @@
-import asyncio
-import aiohttp
-import logging
-import os
-
-from typing import AsyncGenerator
-
-from dailyai.pipeline.aggregators import FrameProcessor, UserResponseAggregator, VisionImageFrameAggregator
-
-from dailyai.pipeline.frames import Frame, TextFrame, UserImageRequestFrame
-from dailyai.pipeline.pipeline import Pipeline
-from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
-from dailyai.services.moondream_ai_service import MoondreamService
-from dailyai.transports.daily_transport import DailyTransport
-
-from runner import configure
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
-logger = logging.getLogger("dailyai")
-logger.setLevel(logging.DEBUG)
-
-
-class UserImageRequester(FrameProcessor):
-    participant_id: str
-
-    def set_participant_id(self, participant_id: str):
-        self.participant_id = participant_id
-
-    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
-        if self.participant_id and isinstance(frame, TextFrame):
-            yield UserImageRequestFrame(self.participant_id)
-        yield frame
-
-
-async def main(room_url: str, token):
-    async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
-            room_url,
-            token,
-            "Describe participant video",
-            duration_minutes=5,
-            mic_enabled=True,
-            mic_sample_rate=16000,
-            vad_enabled=True,
-            start_transcription=True,
-            video_rendering_enabled=True
-        )
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
-        )
-
-        user_response = UserResponseAggregator()
-
-        image_requester = UserImageRequester()
-
-        vision_aggregator = VisionImageFrameAggregator()
-
-        moondream = MoondreamService()
-
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
-        )
-
-        @transport.event_handler("on_first_other_participant_joined")
-        async def on_first_other_participant_joined(transport, participant):
-            await transport.say("Hi there! Feel free to ask me what I see.", tts)
-            transport.render_participant_video(participant["id"], framerate=0)
-            image_requester.set_participant_id(participant["id"])
-
-        pipeline = Pipeline([user_response, image_requester, vision_aggregator, moondream, tts])
-
-        await transport.run(pipeline)
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/13-whisper-transcription.py

@@ -1,58 +0,0 @@
-import asyncio
-import logging
-
-from dailyai.pipeline.frames import EndFrame, TranscriptionFrame
-from dailyai.transports.daily_transport import DailyTransport
-from dailyai.services.whisper_ai_services import WhisperSTTService
-from dailyai.pipeline.pipeline import Pipeline
-
-from runner import configure
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
-logger = logging.getLogger("dailyai")
-logger.setLevel(logging.DEBUG)
-
-
-async def main(room_url: str):
-    transport = DailyTransport(
-        room_url,
-        None,
-        "Transcription bot",
-        start_transcription=False,
-        mic_enabled=False,
-        camera_enabled=False,
-        speaker_enabled=True,
-    )
-
-    stt = WhisperSTTService()
-
-    transcription_output_queue = asyncio.Queue()
-    transport_done = asyncio.Event()
-
-    pipeline = Pipeline([stt], source=transport.receive_queue, sink=transcription_output_queue)
-
-    async def handle_transcription():
-        print("`````````TRANSCRIPTION`````````")
-        while not transport_done.is_set():
-            item = await transcription_output_queue.get()
-            print("got item from queue", item)
-            if isinstance(item, TranscriptionFrame):
-                print(item.text)
-            elif isinstance(item, EndFrame):
-                break
-        print("handle_transcription done")
-
-    async def run_until_done():
-        await transport.run()
-        transport_done.set()
-        print("run_until_done done")
-
-    await asyncio.gather(run_until_done(), pipeline.run_pipeline(), handle_transcription())
-
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url))

examples/foundational/14-render-remote-participant.py

@@ -1,52 +0,0 @@
-import asyncio
-import logging
-
-from typing import AsyncGenerator
-
-from dailyai.pipeline.aggregators import FrameProcessor
-
-from dailyai.pipeline.frames import ImageFrame, Frame, UserImageFrame
-from dailyai.pipeline.pipeline import Pipeline
-from dailyai.transports.daily_transport import DailyTransport
-
-from runner import configure
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
-logger = logging.getLogger("dailyai")
-logger.setLevel(logging.DEBUG)
-
-
-class UserImageProcessor(FrameProcessor):
-
-    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
-        if isinstance(frame, UserImageFrame):
-            yield ImageFrame(frame.image, frame.size)
-        else:
-            yield frame
-
-
-async def main(room_url: str, token):
-    transport = DailyTransport(
-        room_url,
-        token,
-        "Render participant video",
-        camera_width=1280,
-        camera_height=720,
-        camera_enabled=True,
-        video_rendering_enabled=True
-    )
-
-    @ transport.event_handler("on_first_other_participant_joined")
-    async def on_first_other_participant_joined(transport, participant):
-        transport.render_participant_video(participant["id"])
-
-    pipeline = Pipeline([UserImageProcessor()])
-
-    await asyncio.gather(transport.run(pipeline))
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/14a-local-render-remote-participant.py

@@ -1,71 +0,0 @@
-import asyncio
-import logging
-import tkinter as tk
-
-from typing import AsyncGenerator
-
-from dailyai.pipeline.aggregators import FrameProcessor
-
-from dailyai.pipeline.frames import ImageFrame, Frame, UserImageFrame
-from dailyai.pipeline.pipeline import Pipeline
-from dailyai.transports.daily_transport import DailyTransport
-
-from dailyai.transports.local_transport import LocalTransport
-from runner import configure
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
-logger = logging.getLogger("dailyai")
-logger.setLevel(logging.DEBUG)
-
-
-class UserImageProcessor(FrameProcessor):
-
-    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
-        if isinstance(frame, UserImageFrame):
-            yield ImageFrame(frame.image, frame.size)
-        else:
-            yield frame
-
-
-async def main(room_url: str, token):
-    tk_root = tk.Tk()
-    tk_root.title("dailyai")
-
-    local_transport = LocalTransport(
-        tk_root=tk_root,
-        camera_enabled=True,
-        camera_width=1280,
-        camera_height=720
-    )
-
-    transport = DailyTransport(
-        room_url,
-        token,
-        "Render participant video",
-        video_rendering_enabled=True
-    )
-
-    @transport.event_handler("on_first_other_participant_joined")
-    async def on_first_other_participant_joined(transport, participant):
-        transport.render_participant_video(participant["id"])
-
-    async def run_tk():
-        while not transport._stop_threads.is_set():
-            tk_root.update()
-            tk_root.update_idletasks()
-            await asyncio.sleep(0.1)
-
-    local_pipeline = Pipeline([UserImageProcessor()], source=transport.receive_queue)
-
-    await asyncio.gather(
-        transport.run(),
-        local_transport.run(local_pipeline, override_pipeline_source_queue=False),
-        run_tk()
-    )
-
-if __name__ == "__main__":
-    (url, token) = configure()
-    asyncio.run(main(url, token))

examples/foundational/runner.py

@@ -1,58 +0,0 @@
-import argparse
-import os
-import time
-import urllib
-import requests
-
-
-def configure():
-    parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
-    parser.add_argument(
-        "-u",
-        "--url",
-        type=str,
-        required=False,
-        help="URL of the Daily room to join")
-    parser.add_argument(
-        "-k",
-        "--apikey",
-        type=str,
-        required=False,
-        help="Daily API Key (needed to create an owner token for the room)",
-    )
-
-    args, unknown = parser.parse_known_args()
-
-    url = args.url or os.getenv("DAILY_SAMPLE_ROOM_URL")
-    key = args.apikey or os.getenv("DAILY_API_KEY")
-
-    if not url:
-        raise Exception(
-            "No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL.")
-
-    if not key:
-        raise Exception("No Daily API key specified. use the -k/--apikey option from the command line, or set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers.")
-
-    # Create a meeting token for the given room with an expiration 1 hour in
-    # the future.
-    room_name: str = urllib.parse.urlparse(url).path[1:]
-    expiration: float = time.time() + 60 * 60
-
-    res: requests.Response = requests.post(
-        f"https://api.daily.co/v1/meeting-tokens",
-        headers={
-            "Authorization": f"Bearer {key}"},
-        json={
-            "properties": {
-                "room_name": room_name,
-                "is_owner": True,
-                "exp": expiration}},
-    )
-
-    if res.status_code != 200:
-        raise Exception(
-            f"Failed to create meeting token: {res.status_code} {res.text}")
-
-    token: str = res.json()["token"]
-
-    return (url, token)

examples/foundational/websocket-server/frames.proto

@@ -1,25 +0,0 @@
-syntax = "proto3";
-
-package dailyai_proto;
-
-message TextFrame {
-    string text = 1;
-}
-
-message AudioFrame {
-    bytes audio = 1;
-}
-
-message TranscriptionFrame {
-    string text = 1;
-    string participant_id = 2;
-    string timestamp = 3;
-}
-
-message Frame {
-    oneof frame {
-        TextFrame text = 1;
-        AudioFrame audio = 2;
-        TranscriptionFrame transcription = 3;
-    }
-}

examples/foundational/websocket-server/index.html

@@ -1,134 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <script src="//cdn.jsdelivr.net/npm/protobufjs@7.X.X/dist/protobuf.min.js"></script>
-    <title>WebSocket Audio Stream</title>
-</head>
-
-<body>
-    <h1>WebSocket Audio Stream</h1>
-    <button id="startAudioBtn">Start Audio</button>
-    <button id="stopAudioBtn">Stop Audio</button>
-    <script>
-        const SAMPLE_RATE = 16000;
-        const BUFFER_SIZE = 8192;
-        const MIN_AUDIO_SIZE = 6400;
-
-        let audioContext;
-        let microphoneStream;
-        let scriptProcessor;
-        let source;
-        let frame;
-        let audioChunks = [];
-        let isPlaying = false;
-        let ws;
-
-        const proto = protobuf.load("frames.proto", (err, root) => {
-            if (err) throw err;
-            frame = root.lookupType("dailyai_proto.Frame");
-        });
-
-        function initWebSocket() {
-            ws = new WebSocket('ws://localhost:8765');
-
-            ws.addEventListener('open', () => console.log('WebSocket connection established.'));
-            ws.addEventListener('message', handleWebSocketMessage);
-            ws.addEventListener('close', (event) => console.log("WebSocket connection closed.", event.code, event.reason));
-            ws.addEventListener('error', (event) => console.error('WebSocket error:', event));
-        }
-
-        async function handleWebSocketMessage(event) {
-            const arrayBuffer = await event.data.arrayBuffer();
-            enqueueAudioFromProto(arrayBuffer);
-        }
-
-        function enqueueAudioFromProto(arrayBuffer) {
-            const parsedFrame = frame.decode(new Uint8Array(arrayBuffer));
-            if (!parsedFrame?.audio) return false;
-
-            const frameCount = parsedFrame.audio.data.length / 2;
-            const audioOutBuffer = audioContext.createBuffer(1, frameCount, SAMPLE_RATE);
-            const nowBuffering = audioOutBuffer.getChannelData(0);
-            const view = new Int16Array(parsedFrame.audio.data.buffer);
-
-            for (let i = 0; i < frameCount; i++) {
-                const word = view[i];
-                nowBuffering[i] = ((word + 32768) % 65536 - 32768) / 32768.0;
-            }
-
-            audioChunks.push(audioOutBuffer);
-            if (!isPlaying) playNextChunk();
-        }
-
-        function playNextChunk() {
-            if (audioChunks.length === 0) {
-                isPlaying = false;
-                return;
-            }
-
-            isPlaying = true;
-            const audioOutBuffer = audioChunks.shift();
-            const source = audioContext.createBufferSource();
-            source.buffer = audioOutBuffer;
-            source.connect(audioContext.destination);
-            source.onended = playNextChunk;
-            source.start();
-        }
-
-        function startAudio() {
-            if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
-                alert('getUserMedia is not supported in your browser.');
-                return;
-            }
-
-            navigator.mediaDevices.getUserMedia({ audio: true })
-                .then((stream) => {
-                    microphoneStream = stream;
-                    audioContext = new (window.AudioContext || window.webkitAudioContext)();
-                    scriptProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
-                    source = audioContext.createMediaStreamSource(stream);
-                    source.connect(scriptProcessor);
-                    scriptProcessor.connect(audioContext.destination);
-
-                    const audioBuffer = [];
-                    const skipRatio = Math.floor(audioContext.sampleRate / (SAMPLE_RATE * 2));
-
-                    scriptProcessor.onaudioprocess = (event) => {
-                        const rawLeftChannelData = event.inputBuffer.getChannelData(0);
-                        for (let i = 0; i < rawLeftChannelData.length; i += skipRatio) {
-                            const normalized = ((rawLeftChannelData[i] * 32768.0) + 32768) % 65536 - 32768;
-                            const swappedBytes = ((normalized & 0xff) << 8) | ((normalized >> 8) & 0xff);
-                            audioBuffer.push(swappedBytes);
-                        }
-
-                        if (audioBuffer.length >= MIN_AUDIO_SIZE) {
-                            const audioFrame = frame.create({ audio: { audio: audioBuffer.slice(0, MIN_AUDIO_SIZE) } });
-                            const encodedFrame = new Uint8Array(frame.encode(audioFrame).finish());
-                            ws.send(encodedFrame);
-                            audioBuffer.splice(0, MIN_AUDIO_SIZE);
-                        }
-                    };
-
-                    initWebSocket();
-                })
-                .catch((error) => console.error('Error accessing microphone:', error));
-        }
-
-        function stopAudio() {
-            if (ws) {
-                ws.close();
-                scriptProcessor.disconnect();
-                source.disconnect();
-                ws = undefined;
-            }
-        }
-
-        document.getElementById('startAudioBtn').addEventListener('click', startAudio);
-        document.getElementById('stopAudioBtn').addEventListener('click', stopAudio);
-    </script>
-</body>
-
-</html>

examples/foundational/websocket-server/sample.py

@@ -1,50 +0,0 @@
-import asyncio
-import aiohttp
-import logging
-import os
-from dailyai.pipeline.frame_processor import FrameProcessor
-from dailyai.pipeline.frames import TextFrame, TranscriptionFrame
-from dailyai.pipeline.pipeline import Pipeline
-from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
-from dailyai.transports.websocket_transport import WebsocketTransport
-from dailyai.services.whisper_ai_services import WhisperSTTService
-
-logging.basicConfig(format="%(levelno)s %(asctime)s %(message)s")
-logger = logging.getLogger("dailyai")
-logger.setLevel(logging.DEBUG)
-
-
-class WhisperTranscriber(FrameProcessor):
-    async def process_frame(self, frame):
-        if isinstance(frame, TranscriptionFrame):
-            print(f"Transcribed: {frame.text}")
-        else:
-            yield frame
-
-
-async def main():
-    async with aiohttp.ClientSession() as session:
-        transport = WebsocketTransport(
-            mic_enabled=True,
-            speaker_enabled=True,
-        )
-        tts = ElevenLabsTTSService(
-            aiohttp_session=session,
-            api_key=os.getenv("ELEVENLABS_API_KEY"),
-            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
-        )
-
-        pipeline = Pipeline([
-            WhisperSTTService(),
-            WhisperTranscriber(),
-            tts,
-        ])
-
-        @transport.on_connection
-        async def queue_frame():
-            await pipeline.queue_frames([TextFrame("Hello there!")])
-
-        await transport.run(pipeline)
-
-if __name__ == "__main__":
-    asyncio.run(main())

examples/server/README.md

@@ -1,34 +0,0 @@
-# Server Example
-
-Use this server app to quickly host a bot on the web:
-
-```
-flask --app daily-bot-manager.py --debug run
-```
-
-It's currently configured to serve example apps defined in the APPS constant in the server file:
-
-```
-chatbot
-patient-intake
-storybot
-translator
-```
-
-Once the server is started, you can create a bot instance by opening `http://127.0.0.1:5000/start/chatbot` in a browser, and the server will do the following:
-
-- Create a new, randomly-named Daily room with `DAILY_API_KEY` from your .env file or environment
-- Start an instance of `chatbot.py` and connect it to that room
-- 301 redirect your browser to the room
-
-### Options
-
-The server supports several options, which can be set in the body of a POST request, or as params in the URL of a GET request.
-
-- `room_url` (default: none): A room URL to join. If empty, the server will create a Daily room and return the URL in the response.
-  room_properties (none): A JSON object (URL encoded if included as a GET parameter) for overriding default room creation properties, as described here: https://docs.daily.co/reference/rest-api/rooms/create-room This will be ignored if a room_url is provided.
-- `token_properties` (none): A JSON object (URL encoded if included as a GET parameter) for overriding default token properties. By default, the server creates an owner token with an expiration time of one hour.
-- `duration` (7200 seconds, or two hours): Use this property to set a time limit for the bot, as well as an expiration time for the room (if the server is creating one). This will not add an expiration time to an existing room. Expiration times in `token_properties` or `room_properties` will also take precedence over this value. You can set this property to `0` to disable timeouts, but this isn't recommended.
-- `bot_args` (none): A string containing any additional command-line args to pass to the bot.
-- `wait_for_bot` (true): Whether to wait for the bot to successfully join the room before returning a response from the server. If true, the server will start the bot script, then poll the room for up to 5 seconds to confirm the bot has joined the room. If it doesn't, the server will stop the bot and return a 500 response. If set to `false`, the server will start the bot, but immediately return a 200 response. This can be useful if the server is creating rooms for you, and you need the room URL to join the user to the room.
-- `redirect` (true): Instead of returning a 200 for GET requests, the server will return a 301 redirect to the ROOM_URL. This is handy for testing by creating a bot with a GET request directly in the browser. POST requests will never return redirects. Set to `false` to get 200 responses with info in a JSON object even for GET requests.

examples/server/daily-bot-manager.py

@@ -1,165 +0,0 @@
-import os
-import requests
-import urllib
-import subprocess
-import time
-
-from flask import Flask, jsonify, redirect, request
-from flask_cors import CORS
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
-
-app = Flask(__name__)
-CORS(app)
-
-APPS = {
-    "chatbot": "../starter-apps/chatbot.py",
-    "patient-intake": "../starter-apps/patient-intake.py",
-    "storybot": "../starter-apps/storybot.py",
-    "translator": "../starter-apps/translator.py"
-}
-
-daily_api_key = os.getenv("DAILY_API_KEY")
-api_path = os.getenv("DAILY_API_PATH") or "https://api.daily.co/v1"
-
-
-def get_room_name(room_url):
-    return urllib.parse.urlparse(room_url).path[1:]
-
-
-def create_room(room_properties, exp):
-    room_props = {
-        "exp": exp,
-        "enable_chat": True,
-        "enable_emoji_reactions": True,
-        "eject_at_room_exp": True,
-        "enable_prejoin_ui": False,
-        "enable_recording": "cloud"
-    }
-    if room_properties:
-        room_props |= room_properties
-
-    res = requests.post(
-        f"{api_path}/rooms",
-        headers={"Authorization": f"Bearer {daily_api_key}"},
-        json={
-            "properties": room_props
-        },
-    )
-    if res.status_code != 200:
-        raise Exception(f"Unable to create room: {res.text}")
-
-    room_url = res.json()["url"]
-    room_name = res.json()["name"]
-    return (room_url, room_name)
-
-
-def create_token(room_name, token_properties, exp):
-    token_props = {"exp": exp, "is_owner": True}
-    if token_properties:
-        token_props |= token_properties
-    # Force the token to be limited to the room
-    token_props |= {"room_name": room_name}
-    res = requests.post(
-        f'{api_path}/meeting-tokens',
-        headers={
-            'Authorization': f'Bearer {daily_api_key}'},
-        json={
-            'properties': token_props})
-    if res.status_code != 200:
-        if res.status_code != 200:
-            raise Exception(f"Unable to create meeting token: {res.text}")
-
-    meeting_token = res.json()['token']
-    return meeting_token
-
-
-def start_bot(*, bot_path, room_url, token, bot_args, wait_for_bot):
-
-    room_name = get_room_name(room_url)
-    proc = subprocess.Popen(
-        [f"python {bot_path} -u {room_url} -t {token} -k {daily_api_key} {bot_args}"],
-        shell=True,
-        bufsize=1,
-    )
-
-    if wait_for_bot:
-        # Don't return until the bot has joined the room, but wait for at most 5
-        # seconds.
-        attempts = 0
-        while attempts < 50:
-            time.sleep(0.1)
-            attempts += 1
-            res = requests.get(
-                f"{api_path}/rooms/{room_name}/get-session-data",
-                headers={"Authorization": f"Bearer {daily_api_key}"},
-            )
-            if res.status_code == 200:
-                print(f"Took {attempts} attempts to join room {room_name}")
-                return True
-
-        # If we don't break from the loop, that means we never found the bot in the room
-        raise Exception("The bot was unable to join the room. Please try again.")
-
-    return True
-
-
-@app.route("/start/<string:botname>", methods=["GET", "POST"])
-def start(botname):
-    try:
-        if botname not in APPS:
-            raise Exception(f"Bot '{botname}' is not in the allowlist.")
-
-        bot_path = APPS[botname]
-        props = {
-            "room_url": None,
-            "room_properties": None,
-            "token_properties": None,
-            "bot_args": None,
-            "wait_for_bot": True,
-            "duration": None,
-            "redirect": True
-        }
-        props |= request.values.to_dict()  # gets URL params as well as plaintext POST body
-        try:
-            props |= request.json
-        except BaseException:
-            pass
-        if props['redirect'] == "false":
-            props['redirect'] = False
-        if props['wait_for_bot'] == "false":
-            props['wait_for_bot'] = False
-
-        duration = int(os.getenv("DAILY_BOT_DURATION") or 7200)
-        if props['duration']:
-            duration = props['duration']
-        exp = time.time() + duration
-        if (props['room_url']):
-            room_url = props['room_url']
-            try:
-                room_name = get_room_name(room_url)
-            except ValueError:
-                raise Exception(
-                    "There was a problem detecting the room name. Please double-check the value of room_url.")
-        else:
-            room_url, room_name = create_room(props['room_properties'], exp)
-        token = create_token(room_name, props['token_properties'], exp)
-        bot = start_bot(
-            room_url=room_url,
-            bot_path=bot_path,
-            token=token,
-            bot_args=props['bot_args'],
-            wait_for_bot=props['wait_for_bot'])
-
-        if props['redirect'] and request.method == "GET":
-            return redirect(room_url, 302)
-        else:
-            return jsonify({"room_url": room_url, "token": token})
-    except BaseException as e:
-        return f"There was a problem starting the bot: {e}", 500
-
-
-@app.route("/healthz")
-def health_check():
-    return "ok", 200

linux-py3.10-requirements.txt

@@ -1,359 +0,0 @@
-#
-# This file is autogenerated by pip-compile with Python 3.10
-# by the following command:
-#
-#    pip-compile --all-extras pyproject.toml
-#
-aiohttp==3.9.3
-    # via dailyai (pyproject.toml)
-aiosignal==1.3.1
-    # via aiohttp
-anthropic==0.20.0
-    # via dailyai (pyproject.toml)
-anyio==4.3.0
-    # via
-    #   anthropic
-    #   httpx
-    #   openai
-    #   starlette
-async-timeout==4.0.3
-    # via aiohttp
-attrs==23.2.0
-    # via
-    #   aiohttp
-    #   fal
-av==11.0.0
-    # via faster-whisper
-azure-cognitiveservices-speech==1.36.0
-    # via dailyai (pyproject.toml)
-blinker==1.7.0
-    # via flask
-certifi==2024.2.2
-    # via
-    #   httpcore
-    #   httpx
-    #   requests
-cffi==1.16.0
-    # via cryptography
-charset-normalizer==3.3.2
-    # via requests
-click==8.1.7
-    # via
-    #   fal
-    #   flask
-    #   rich-click
-colorama==0.4.6
-    # via fal
-coloredlogs==15.0.1
-    # via onnxruntime
-cryptography==42.0.5
-    # via pyjwt
-ctranslate2==4.1.0
-    # via faster-whisper
-daily-python==0.7.3
-    # via dailyai (pyproject.toml)
-deprecated==1.2.14
-    # via opentelemetry-api
-dill==0.3.7
-    # via fal
-distlib==0.3.8
-    # via virtualenv
-distro==1.9.0
-    # via
-    #   anthropic
-    #   openai
-einops==0.7.0
-    # via dailyai (pyproject.toml)
-exceptiongroup==1.2.0
-    # via anyio
-fal==0.12.7
-    # via dailyai (pyproject.toml)
-fastapi==0.99.1
-    # via fal
-faster-whisper==1.0.1
-    # via dailyai (pyproject.toml)
-filelock==3.13.4
-    # via
-    #   huggingface-hub
-    #   pyht
-    #   torch
-    #   transformers
-    #   triton
-    #   virtualenv
-flask==3.0.3
-    # via
-    #   dailyai (pyproject.toml)
-    #   flask-cors
-flask-cors==4.0.0
-    # via dailyai (pyproject.toml)
-flatbuffers==24.3.25
-    # via onnxruntime
-frozenlist==1.4.1
-    # via
-    #   aiohttp
-    #   aiosignal
-fsspec==2024.3.1
-    # via
-    #   huggingface-hub
-    #   torch
-grpc-interceptor==0.15.4
-    # via fal
-grpcio==1.62.1
-    # via
-    #   fal
-    #   grpc-interceptor
-    #   isolate
-    #   isolate-proto
-    #   pyht
-h11==0.14.0
-    # via httpcore
-httpcore==1.0.5
-    # via httpx
-httpx==0.27.0
-    # via
-    #   anthropic
-    #   fal
-    #   openai
-huggingface-hub==0.22.2
-    # via
-    #   faster-whisper
-    #   timm
-    #   tokenizers
-    #   transformers
-humanfriendly==10.0
-    # via coloredlogs
-idna==3.6
-    # via
-    #   anyio
-    #   httpx
-    #   requests
-    #   yarl
-importlib-metadata==7.0.0
-    # via opentelemetry-api
-isolate[build]==0.12.7
-    # via
-    #   fal
-    #   isolate-proto
-isolate-proto==0.3.4
-    # via fal
-itsdangerous==2.1.2
-    # via flask
-jinja2==3.1.3
-    # via
-    #   flask
-    #   torch
-markdown-it-py==3.0.0
-    # via rich
-markupsafe==2.1.5
-    # via
-    #   jinja2
-    #   werkzeug
-mdurl==0.1.2
-    # via markdown-it-py
-mpmath==1.3.0
-    # via sympy
-msgpack==1.0.8
-    # via fal
-multidict==6.0.5
-    # via
-    #   aiohttp
-    #   yarl
-networkx==3.3
-    # via torch
-numpy==1.26.4
-    # via
-    #   ctranslate2
-    #   dailyai (pyproject.toml)
-    #   onnxruntime
-    #   torchvision
-    #   transformers
-nvidia-cublas-cu12==12.1.3.1
-    # via
-    #   nvidia-cudnn-cu12
-    #   nvidia-cusolver-cu12
-    #   torch
-nvidia-cuda-cupti-cu12==12.1.105
-    # via torch
-nvidia-cuda-nvrtc-cu12==12.1.105
-    # via torch
-nvidia-cuda-runtime-cu12==12.1.105
-    # via torch
-nvidia-cudnn-cu12==8.9.2.26
-    # via torch
-nvidia-cufft-cu12==11.0.2.54
-    # via torch
-nvidia-curand-cu12==10.3.2.106
-    # via torch
-nvidia-cusolver-cu12==11.4.5.107
-    # via torch
-nvidia-cusparse-cu12==12.1.0.106
-    # via
-    #   nvidia-cusolver-cu12
-    #   torch
-nvidia-nccl-cu12==2.19.3
-    # via torch
-nvidia-nvjitlink-cu12==12.4.127
-    # via
-    #   nvidia-cusolver-cu12
-    #   nvidia-cusparse-cu12
-nvidia-nvtx-cu12==12.1.105
-    # via torch
-onnxruntime==1.17.1
-    # via faster-whisper
-openai==1.14.3
-    # via dailyai (pyproject.toml)
-opentelemetry-api==1.24.0
-    # via
-    #   fal
-    #   opentelemetry-sdk
-opentelemetry-sdk==1.24.0
-    # via fal
-opentelemetry-semantic-conventions==0.45b0
-    # via opentelemetry-sdk
-packaging==24.0
-    # via
-    #   fal
-    #   huggingface-hub
-    #   onnxruntime
-    #   transformers
-pathspec==0.11.2
-    # via fal
-pillow==10.2.0
-    # via
-    #   dailyai (pyproject.toml)
-    #   fal
-    #   torchvision
-platformdirs==4.2.0
-    # via
-    #   isolate
-    #   virtualenv
-portalocker==2.8.2
-    # via fal
-protobuf==4.25.3
-    # via
-    #   isolate
-    #   isolate-proto
-    #   onnxruntime
-    #   pyht
-pyaudio==0.2.14
-    # via dailyai (pyproject.toml)
-pycparser==2.22
-    # via cffi
-pydantic==1.10.15
-    # via
-    #   anthropic
-    #   fal
-    #   fastapi
-    #   openai
-pygments==2.17.2
-    # via rich
-pyht==0.0.27
-    # via dailyai (pyproject.toml)
-pyjwt[crypto]==2.8.0
-    # via fal
-python-dateutil==2.9.0.post0
-    # via fal
-python-dotenv==1.0.1
-    # via dailyai (pyproject.toml)
-pyyaml==6.0.1
-    # via
-    #   ctranslate2
-    #   huggingface-hub
-    #   isolate
-    #   timm
-    #   transformers
-regex==2023.12.25
-    # via transformers
-requests==2.31.0
-    # via
-    #   huggingface-hub
-    #   pyht
-    #   transformers
-rich==13.7.1
-    # via
-    #   fal
-    #   rich-click
-rich-click==1.7.4
-    # via fal
-safetensors==0.4.2
-    # via
-    #   timm
-    #   transformers
-six==1.16.0
-    # via python-dateutil
-sniffio==1.3.1
-    # via
-    #   anthropic
-    #   anyio
-    #   httpx
-    #   openai
-starlette==0.27.0
-    # via fastapi
-structlog==22.3.0
-    # via fal
-sympy==1.12
-    # via
-    #   onnxruntime
-    #   torch
-tblib==3.0.0
-    # via isolate
-timm==0.9.16
-    # via dailyai (pyproject.toml)
-tokenizers==0.15.2
-    # via
-    #   anthropic
-    #   faster-whisper
-    #   transformers
-torch==2.2.2
-    # via
-    #   dailyai (pyproject.toml)
-    #   timm
-    #   torchaudio
-    #   torchvision
-torchaudio==2.2.2
-    # via dailyai (pyproject.toml)
-torchvision==0.17.2
-    # via timm
-tqdm==4.66.2
-    # via
-    #   huggingface-hub
-    #   openai
-    #   transformers
-transformers==4.39.3
-    # via dailyai (pyproject.toml)
-triton==2.2.0
-    # via torch
-types-python-dateutil==2.9.0.20240316
-    # via fal
-typing-extensions==4.10.0
-    # via
-    #   anthropic
-    #   anyio
-    #   dailyai (pyproject.toml)
-    #   fal
-    #   fastapi
-    #   huggingface-hub
-    #   openai
-    #   opentelemetry-sdk
-    #   pydantic
-    #   rich-click
-    #   torch
-urllib3==2.2.1
-    # via requests
-virtualenv==20.25.1
-    # via isolate
-websockets==12.0
-    # via
-    #   dailyai (pyproject.toml)
-    #   fal
-werkzeug==3.0.2
-    # via flask
-wrapt==1.16.0
-    # via deprecated
-yarl==1.9.4
-    # via aiohttp
-zipp==3.18.1
-    # via importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools

macos-py3.10-requirements.txt

@@ -1,325 +0,0 @@
-#
-# This file is autogenerated by pip-compile with Python 3.10
-# by the following command:
-#
-#    pip-compile --all-extras pyproject.toml
-#
-aiohttp==3.9.3
-    # via dailyai (pyproject.toml)
-aiosignal==1.3.1
-    # via aiohttp
-anthropic==0.20.0
-    # via dailyai (pyproject.toml)
-anyio==4.3.0
-    # via
-    #   anthropic
-    #   httpx
-    #   openai
-    #   starlette
-async-timeout==4.0.3
-    # via aiohttp
-attrs==23.2.0
-    # via
-    #   aiohttp
-    #   fal
-av==11.0.0
-    # via faster-whisper
-azure-cognitiveservices-speech==1.36.0
-    # via dailyai (pyproject.toml)
-blinker==1.7.0
-    # via flask
-certifi==2024.2.2
-    # via
-    #   httpcore
-    #   httpx
-    #   requests
-cffi==1.16.0
-    # via cryptography
-charset-normalizer==3.3.2
-    # via requests
-click==8.1.7
-    # via
-    #   fal
-    #   flask
-    #   rich-click
-colorama==0.4.6
-    # via fal
-coloredlogs==15.0.1
-    # via onnxruntime
-cryptography==42.0.5
-    # via pyjwt
-ctranslate2==4.1.0
-    # via faster-whisper
-daily-python==0.7.3
-    # via dailyai (pyproject.toml)
-deprecated==1.2.14
-    # via opentelemetry-api
-dill==0.3.7
-    # via fal
-distlib==0.3.8
-    # via virtualenv
-distro==1.9.0
-    # via
-    #   anthropic
-    #   openai
-einops==0.7.0
-    # via dailyai (pyproject.toml)
-exceptiongroup==1.2.0
-    # via anyio
-fal==0.12.7
-    # via dailyai (pyproject.toml)
-fastapi==0.99.1
-    # via fal
-faster-whisper==1.0.1
-    # via dailyai (pyproject.toml)
-filelock==3.13.4
-    # via
-    #   huggingface-hub
-    #   pyht
-    #   torch
-    #   transformers
-    #   virtualenv
-flask==3.0.3
-    # via
-    #   dailyai (pyproject.toml)
-    #   flask-cors
-flask-cors==4.0.0
-    # via dailyai (pyproject.toml)
-flatbuffers==24.3.25
-    # via onnxruntime
-frozenlist==1.4.1
-    # via
-    #   aiohttp
-    #   aiosignal
-fsspec==2024.3.1
-    # via
-    #   huggingface-hub
-    #   torch
-grpc-interceptor==0.15.4
-    # via fal
-grpcio==1.62.1
-    # via
-    #   fal
-    #   grpc-interceptor
-    #   isolate
-    #   isolate-proto
-    #   pyht
-h11==0.14.0
-    # via httpcore
-httpcore==1.0.5
-    # via httpx
-httpx==0.27.0
-    # via
-    #   anthropic
-    #   fal
-    #   openai
-huggingface-hub==0.22.2
-    # via
-    #   faster-whisper
-    #   timm
-    #   tokenizers
-    #   transformers
-humanfriendly==10.0
-    # via coloredlogs
-idna==3.6
-    # via
-    #   anyio
-    #   httpx
-    #   requests
-    #   yarl
-importlib-metadata==7.0.0
-    # via opentelemetry-api
-isolate[build]==0.12.7
-    # via
-    #   fal
-    #   isolate-proto
-isolate-proto==0.3.4
-    # via fal
-itsdangerous==2.1.2
-    # via flask
-jinja2==3.1.3
-    # via
-    #   flask
-    #   torch
-markdown-it-py==3.0.0
-    # via rich
-markupsafe==2.1.5
-    # via
-    #   jinja2
-    #   werkzeug
-mdurl==0.1.2
-    # via markdown-it-py
-mpmath==1.3.0
-    # via sympy
-msgpack==1.0.8
-    # via fal
-multidict==6.0.5
-    # via
-    #   aiohttp
-    #   yarl
-networkx==3.3
-    # via torch
-numpy==1.26.4
-    # via
-    #   ctranslate2
-    #   dailyai (pyproject.toml)
-    #   onnxruntime
-    #   torchvision
-    #   transformers
-onnxruntime==1.17.1
-    # via faster-whisper
-openai==1.14.3
-    # via dailyai (pyproject.toml)
-opentelemetry-api==1.24.0
-    # via
-    #   fal
-    #   opentelemetry-sdk
-opentelemetry-sdk==1.24.0
-    # via fal
-opentelemetry-semantic-conventions==0.45b0
-    # via opentelemetry-sdk
-packaging==24.0
-    # via
-    #   fal
-    #   huggingface-hub
-    #   onnxruntime
-    #   transformers
-pathspec==0.11.2
-    # via fal
-pillow==10.2.0
-    # via
-    #   dailyai (pyproject.toml)
-    #   fal
-    #   torchvision
-platformdirs==4.2.0
-    # via
-    #   isolate
-    #   virtualenv
-portalocker==2.8.2
-    # via fal
-protobuf==4.25.3
-    # via
-    #   isolate
-    #   isolate-proto
-    #   onnxruntime
-    #   pyht
-pyaudio==0.2.14
-    # via dailyai (pyproject.toml)
-pycparser==2.22
-    # via cffi
-pydantic==1.10.15
-    # via
-    #   anthropic
-    #   fal
-    #   fastapi
-    #   openai
-pygments==2.17.2
-    # via rich
-pyht==0.0.27
-    # via dailyai (pyproject.toml)
-pyjwt[crypto]==2.8.0
-    # via fal
-python-dateutil==2.9.0.post0
-    # via fal
-python-dotenv==1.0.1
-    # via dailyai (pyproject.toml)
-pyyaml==6.0.1
-    # via
-    #   ctranslate2
-    #   huggingface-hub
-    #   isolate
-    #   timm
-    #   transformers
-regex==2023.12.25
-    # via transformers
-requests==2.31.0
-    # via
-    #   huggingface-hub
-    #   pyht
-    #   transformers
-rich==13.7.1
-    # via
-    #   fal
-    #   rich-click
-rich-click==1.7.4
-    # via fal
-safetensors==0.4.2
-    # via
-    #   timm
-    #   transformers
-six==1.16.0
-    # via python-dateutil
-sniffio==1.3.1
-    # via
-    #   anthropic
-    #   anyio
-    #   httpx
-    #   openai
-starlette==0.27.0
-    # via fastapi
-structlog==22.3.0
-    # via fal
-sympy==1.12
-    # via
-    #   onnxruntime
-    #   torch
-tblib==3.0.0
-    # via isolate
-timm==0.9.16
-    # via dailyai (pyproject.toml)
-tokenizers==0.15.2
-    # via
-    #   anthropic
-    #   faster-whisper
-    #   transformers
-torch==2.2.2
-    # via
-    #   dailyai (pyproject.toml)
-    #   timm
-    #   torchaudio
-    #   torchvision
-torchaudio==2.2.2
-    # via dailyai (pyproject.toml)
-torchvision==0.17.2
-    # via timm
-tqdm==4.66.2
-    # via
-    #   huggingface-hub
-    #   openai
-    #   transformers
-transformers==4.39.3
-    # via dailyai (pyproject.toml)
-types-python-dateutil==2.9.0.20240316
-    # via fal
-typing-extensions==4.10.0
-    # via
-    #   anthropic
-    #   anyio
-    #   dailyai (pyproject.toml)
-    #   fal
-    #   fastapi
-    #   huggingface-hub
-    #   openai
-    #   opentelemetry-sdk
-    #   pydantic
-    #   rich-click
-    #   torch
-urllib3==2.2.1
-    # via requests
-virtualenv==20.25.1
-    # via isolate
-websockets==12.0
-    # via
-    #   dailyai (pyproject.toml)
-    #   fal
-werkzeug==3.0.2
-    # via flask
-wrapt==1.16.0
-    # via deprecated
-yarl==1.9.4
-    # via aiohttp
-zipp==3.18.1
-    # via importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools

pyproject.toml

@@ -1,10 +1,10 @@
 [build-system]
-requires = ["setuptools>=64", "setuptools_scm>=8"]
+requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "dailyai"
-dynamic = ["version"]
+version = "0.0.3.1"
 description = "An open source framework for real-time, multi-modal, conversational AI applications"
 license = { text = "BSD 2-Clause License" }
 readme = "README.md"
@@ -20,36 +20,28 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Artificial Intelligence"
 ]
 dependencies = [
-    "aiohttp~=3.9.0",
-    "numpy~=1.26.0",
-    "Pillow~=10.2.0",
-    "typing-extensions~=4.10.0",
+    "aiohttp",
+    "anthropic",
+    "azure-cognitiveservices-speech",
+    "daily-python",
+    "fal",
+    "faster_whisper",
+    "google-cloud-texttospeech",
+    "numpy",
+    "openai",
+    "Pillow",
+    "pyht",
+    "python-dotenv",
+    "torch",
+    "torchaudio",
+    "pyaudio",
+    "typing-extensions"
 ]
 
 [project.urls]
-Source = "https://github.com/daily-co/dailyai"
+Source = "https://github.com/daily-co/daily-ai-sdk"
 Website = "https://daily.co"
 
-[project.optional-dependencies]
-anthropic = [ "anthropic~=0.20.0" ]
-azure = [ "azure-cognitiveservices-speech~=1.36.0" ]
-daily = [ "daily-python~=0.7.0" ]
-examples = [ "python-dotenv~=1.0.0", "flask~=3.0.0", "flask_cors~=4.0.0" ]
-fal = [ "fal~=0.12.0" ]
-local = [ "pyaudio~=0.2.0" ]
-moondream = [ "einops~=0.7.0", "timm~=0.9.0", "transformers~=4.39.0" ]
-openai = [ "openai~=1.14.0" ]
-playht = [ "pyht~=0.0.26" ]
-silero = [ "torch~=2.2.0", "torchaudio~=2.2.0" ]
-websocket = [ "websockets~=12.0" ]
-whisper = [ "faster_whisper~=1.0.0" ]
-
 [tool.setuptools.packages.find]
 # All the following settings are optional:
 where = ["src"]
-
-[tool.pytest.ini_options]
-pythonpath = ["src"]
-
-[tool.setuptools_scm]
-local_scheme = "no-local-version"

requirements.txt

@@ -0,0 +1,4 @@
+autopep8==2.0.4
+build==1.0.3
+packaging==23.2
+pyproject_hooks==1.0.0

src/dailyai/pipeline/aggregators.py

@@ -5,98 +5,27 @@ from dailyai.pipeline.frame_processor import FrameProcessor
 
 from dailyai.pipeline.frames import (
     EndFrame,
+    AudioFrame,
     EndPipeFrame,
     Frame,
     ImageFrame,
-    LLMMessagesFrame,
+    LLMMessagesQueueFrame,
     LLMResponseEndFrame,
     LLMResponseStartFrame,
     TextFrame,
-    TranscriptionFrame,
+    TranscriptionQueueFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
-    VisionImageFrame,
 )
 from dailyai.pipeline.pipeline import Pipeline
 from dailyai.services.ai_services import AIService
 
-from typing import AsyncGenerator, Coroutine, List
+from typing import AsyncGenerator, Callable, Coroutine, List
+
+from dailyai.services.openai_llm_context import OpenAILLMContext
 
 
 class ResponseAggregator(FrameProcessor):
-    """This frame processor aggregates frames between a start and an end frame
-    into complete text frame sentences.
-
-    For example, frame input/output:
-        UserStartedSpeakingFrame() -> None
-        TranscriptionFrame("Hello,") -> None
-        TranscriptionFrame(" world.") -> None
-        UserStoppedSpeakingFrame() -> TextFrame("Hello world.")
-
-    Doctest:
-    >>> async def print_frames(aggregator, frame):
-    ...     async for frame in aggregator.process_frame(frame):
-    ...         if isinstance(frame, TextFrame):
-    ...             print(frame.text)
-
-    >>> aggregator = ResponseAggregator(start_frame = UserStartedSpeakingFrame,
-    ...                                 end_frame=UserStoppedSpeakingFrame,
-    ...                                 accumulator_frame=TranscriptionFrame,
-    ...                                 pass_through=False)
-    >>> asyncio.run(print_frames(aggregator, UserStartedSpeakingFrame()))
-    >>> asyncio.run(print_frames(aggregator, TranscriptionFrame("Hello,", 1, 1)))
-    >>> asyncio.run(print_frames(aggregator, TranscriptionFrame("world.",  1, 2)))
-    >>> asyncio.run(print_frames(aggregator, UserStoppedSpeakingFrame()))
-    Hello, world.
-
-    """
-
-    def __init__(
-        self,
-        *,
-        start_frame,
-        end_frame,
-        accumulator_frame,
-        pass_through=True,
-    ):
-        self.aggregation = ""
-        self.aggregating = False
-        self._start_frame = start_frame
-        self._end_frame = end_frame
-        self._accumulator_frame = accumulator_frame
-        self._pass_through = pass_through
-
-    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
-        if isinstance(frame, self._start_frame):
-            self.aggregating = True
-        elif isinstance(frame, self._end_frame):
-            self.aggregating = False
-            # Sometimes VAD triggers quickly on and off. If we don't get any transcription,
-            # it creates empty LLM message queue frames
-            if len(self.aggregation) > 0:
-                output = self.aggregation
-                self.aggregation = ""
-                yield self._end_frame()
-                yield TextFrame(output.strip())
-        elif isinstance(frame, self._accumulator_frame) and self.aggregating:
-            self.aggregation += f" {frame.text}"
-            if self._pass_through:
-                yield frame
-        else:
-            yield frame
-
-
-class UserResponseAggregator(ResponseAggregator):
-    def __init__(self):
-        super().__init__(
-            start_frame=UserStartedSpeakingFrame,
-            end_frame=UserStoppedSpeakingFrame,
-            accumulator_frame=TranscriptionFrame,
-            pass_through=False,
-        )
-
-
-class LLMResponseAggregator(FrameProcessor):
 
     def __init__(
         self,
@@ -132,7 +61,7 @@ class LLMResponseAggregator(FrameProcessor):
                     {"role": self._role, "content": self.aggregation})
                 self.aggregation = ""
                 yield self._end_frame()
-                yield LLMMessagesFrame(self.messages)
+                yield LLMMessagesQueueFrame(self.messages)
         elif isinstance(frame, self._accumulator_frame) and self.aggregating:
             self.aggregation += f" {frame.text}"
             if self._pass_through:
@@ -141,7 +70,7 @@ class LLMResponseAggregator(FrameProcessor):
             yield frame
 
 
-class LLMAssistantResponseAggregator(LLMResponseAggregator):
+class LLMResponseAggregator(ResponseAggregator):
     def __init__(self, messages: list[dict]):
         super().__init__(
             messages=messages,
@@ -152,14 +81,14 @@ class LLMAssistantResponseAggregator(LLMResponseAggregator):
         )
 
 
-class LLMUserResponseAggregator(LLMResponseAggregator):
+class UserResponseAggregator(ResponseAggregator):
     def __init__(self, messages: list[dict]):
         super().__init__(
             messages=messages,
             role="user",
             start_frame=UserStartedSpeakingFrame,
             end_frame=UserStoppedSpeakingFrame,
-            accumulator_frame=TranscriptionFrame,
+            accumulator_frame=TranscriptionQueueFrame,
             pass_through=False,
         )
 
@@ -189,7 +118,7 @@ class LLMContextAggregator(AIService):
             return
 
         # Ignore transcription frames from the bot
-        if isinstance(frame, TranscriptionFrame):
+        if isinstance(frame, TranscriptionQueueFrame):
             if frame.participantId == self.bot_participant_id:
                 return
 
@@ -201,19 +130,19 @@ class LLMContextAggregator(AIService):
 
         # TODO: split up transcription by participant
         if self.complete_sentences:
-            # type: ignore -- the linter thinks this isn't a TextFrame, even
+            # type: ignore -- the linter thinks this isn't a TextQueueFrame, even
             # though we check it above
             self.sentence += frame.text
             if self.sentence.endswith((".", "?", "!")):
                 self.messages.append(
                     {"role": self.role, "content": self.sentence})
                 self.sentence = ""
-                yield LLMMessagesFrame(self.messages)
+                yield LLMMessagesQueueFrame(self.messages)
         else:
-            # type: ignore -- the linter thinks this isn't a TextFrame, even
+            # type: ignore -- the linter thinks this isn't a TextQueueFrame, even
             # though we check it above
             self.messages.append({"role": self.role, "content": frame.text})
-            yield LLMMessagesFrame(self.messages)
+            yield LLMMessagesQueueFrame(self.messages)
 
 
 class LLMUserContextAggregator(LLMContextAggregator):
@@ -323,9 +252,15 @@ class LLMFullResponseAggregator(FrameProcessor):
         self.aggregation = ""
 
     async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if not isinstance(frame, AudioFrame):
+            print(f"^^^ LFRA got frame: {frame}")
         if isinstance(frame, TextFrame):
             self.aggregation += frame.text
+            print(
+                f"^^^ LFRA got textframe. aggregation is now {self.aggregation}")
         elif isinstance(frame, LLMResponseEndFrame):
+            print(
+                f"^^^ LFRA got an llmresponseendframe. About to yield aggregation: {self.aggregation}")
             yield TextFrame(self.aggregation)
             yield frame
             self.aggregation = ""
@@ -409,7 +344,7 @@ class ParallelPipeline(FrameProcessor):
                 continue
             seen_ids.add(id(frame))
 
-            # Skip passing along EndPipeFrame, because we use them
+            # Skip passing along EndParallelPipeQueueFrame, because we use them
             # for our own flow control.
             if not isinstance(frame, EndPipeFrame):
                 yield frame
@@ -420,8 +355,6 @@ class GatedAggregator(FrameProcessor):
     Yields gate-opening frame before any accumulated frames, then ensuing frames
     until and not including the gate-closed frame.
 
-    >>> from dailyai.pipeline.frames import ImageFrame
-
     >>> async def print_frames(aggregator, frame):
     ...     async for frame in aggregator.process_frame(frame):
     ...         if isinstance(frame, TextFrame):
@@ -435,7 +368,7 @@ class GatedAggregator(FrameProcessor):
     ...     start_open=False)
     >>> asyncio.run(print_frames(aggregator, TextFrame("Hello")))
     >>> asyncio.run(print_frames(aggregator, TextFrame("Hello again.")))
-    >>> asyncio.run(print_frames(aggregator, ImageFrame(image=bytes([]), size=(0, 0))))
+    >>> asyncio.run(print_frames(aggregator, ImageFrame(url='', image=bytes([]))))
     ImageFrame
     Hello
     Hello again.
@@ -465,37 +398,3 @@ class GatedAggregator(FrameProcessor):
             self.accumulator = []
         else:
             self.accumulator.append(frame)
-
-
-class VisionImageFrameAggregator(FrameProcessor):
-    """This aggregator waits for a consecutive TextFrame and an
-    ImageFrame. After the ImageFrame arrives it will output a VisionImageFrame.
-
-    >>> from dailyai.pipeline.frames import ImageFrame
-
-    >>> async def print_frames(aggregator, frame):
-    ...     async for frame in aggregator.process_frame(frame):
-    ...         print(frame)
-
-    >>> aggregator = VisionImageFrameAggregator()
-    >>> asyncio.run(print_frames(aggregator, TextFrame("What do you see?")))
-    >>> asyncio.run(print_frames(aggregator, ImageFrame(image=bytes([]), size=(0, 0))))
-    VisionImageFrame, text: What do you see?, image size: 0x0, buffer size: 0 B
-
-    """
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self._describe_text = None
-
-    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
-        if isinstance(frame, TextFrame):
-            self._describe_text = frame.text
-        elif isinstance(frame, ImageFrame):
-            if self._describe_text:
-                yield VisionImageFrame(self._describe_text, frame.image, frame.size)
-                self._describe_text = None
-            else:
-                yield frame
-        else:
-            yield frame

src/dailyai/pipeline/frame_processor.py

@@ -12,9 +12,9 @@ class FrameProcessor:
 
     By convention, FrameProcessors should immediately yield any frames they don't process.
 
-    Stateful FrameProcessors should watch for the EndFrame and finalize their
+    Stateful FrameProcessors should watch for the EndStreamQueueFrame and finalize their
     output, eg. yielding an unfinished sentence if they're aggregating LLM output to full
-    sentences. EndFrame is also a chance to clean up any services that need to
+    sentences. EndStreamQueueFrame is also a chance to clean up any services that need to
     be closed, del'd, etc.
     """
 
@@ -23,12 +23,11 @@ class FrameProcessor:
         self, frame: Frame
     ) -> AsyncGenerator[Frame, None]:
         """Process a single frame and yield 0 or more frames."""
+        if isinstance(frame, ControlFrame):
+            yield frame
         yield frame
 
     @abstractmethod
     async def interrupted(self) -> None:
         """Handle any cleanup if the pipeline was interrupted."""
         pass
-
-    def __str__(self):
-        return self.__class__.__name__

src/dailyai/pipeline/frames.proto

@@ -1,25 +0,0 @@
-syntax = "proto3";
-
-package dailyai_proto;
-
-message TextFrame {
-    string text = 1;
-}
-
-message AudioFrame {
-    bytes data = 1;
-}
-
-message TranscriptionFrame {
-    string text = 1;
-    string participantId = 2;
-    string timestamp = 3;
-}
-
-message Frame {
-    oneof frame {
-        TextFrame text = 1;
-        AudioFrame audio = 2;
-        TranscriptionFrame transcription = 3;
-    }
-}

src/dailyai/pipeline/frames.py

@@ -1,6 +1,8 @@
 from dataclasses import dataclass
 from typing import Any, List
 
+from dailyai.services.openai_llm_context import OpenAILLMContext
+
 
 class Frame:
     def __str__(self):
@@ -70,66 +72,11 @@ class AudioFrame(Frame):
 class ImageFrame(Frame):
     """An image. Will be shown by the transport if the transport's camera is
     enabled."""
-    image: bytes
-    size: tuple[int, int]
-
-    def __str__(self):
-        return f"{self.__class__.__name__}, image size: {self.size[0]}x{self.size[1]} buffer size: {len(self.image)} B"
-
-
-@dataclass()
-class URLImageFrame(ImageFrame):
-    """An image with an associated URL. Will be shown by the transport if the
-    transport's camera is enabled.
-
-    """
     url: str | None
-
-    def __init__(self, url, image, size):
-        super().__init__(image, size)
-        self.url = url
+    image: bytes
 
     def __str__(self):
-        return f"{self.__class__.__name__}, url: {self.url}, image size: {self.size[0]}x{self.size[1]}, buffer size: {len(self.image)} B"
-
-
-@dataclass()
-class VisionImageFrame(ImageFrame):
-    """An image with an associated text to ask for a description of it. Will be shown by the
-    transport if the transport's camera is enabled.
-
-    """
-    text: str | None
-
-    def __init__(self, text, image, size):
-        super().__init__(image, size)
-        self.text = text
-
-    def __str__(self):
-        return f"{self.__class__.__name__}, text: {self.text}, image size: {self.size[0]}x{self.size[1]}, buffer size: {len(self.image)} B"
-
-
-@dataclass()
-class UserImageFrame(ImageFrame):
-    """An image associated to a user. Will be shown by the transport if the transport's camera is
-    enabled."""
-    user_id: str
-
-    def __init__(self, user_id, image, size):
-        super().__init__(image, size)
-        self.user_id = user_id
-
-    def __str__(self):
-        return f"{self.__class__.__name__}, user: {self.user_id}, image size: {self.size[0]}x{self.size[1]}, buffer size: {len(self.image)} B"
-
-
-@dataclass()
-class UserImageRequestFrame(Frame):
-    """A frame user to request an image from the given user."""
-    user_id: str
-
-    def __str__(self):
-        return f"{self.__class__.__name__}, user: {self.user_id}"
+        return f"{self.__class__.__name__}, url: {self.url}, image size: {len(self.image)} B"
 
 
 @dataclass()
@@ -154,31 +101,15 @@ class TextFrame(Frame):
 
 
 @dataclass()
-class TranscriptionFrame(TextFrame):
+class TranscriptionQueueFrame(TextFrame):
     """A text frame with transcription-specific data. Will be placed in the
     transport's receive queue when a participant speaks."""
     participantId: str
     timestamp: str
 
-    def __str__(self):
-        return f"{self.__class__.__name__}, text: '{self.text}' participantId: {self.participantId}, timestamp: {self.timestamp}"
-
-
-class TTSStartFrame(ControlFrame):
-    """Used to indicate the beginning of a TTS response. Following AudioFrames
-    are part of the TTS response until an TTEndFrame. These frames can be used
-    for aggregating audio frames in a transport to optimize the size of frames
-    sent to the session, without needing to control this in the TTS service."""
-    pass
-
-
-class TTSEndFrame(ControlFrame):
-    """Indicates the end of a TTS response."""
-    pass
-
 
 @dataclass()
-class LLMMessagesFrame(Frame):
+class LLMMessagesQueueFrame(Frame):
     """A frame containing a list of LLM messages. Used to signal that an LLM
     service should run a chat completion and emit an LLMStartFrames, TextFrames
     and an LLMEndFrame.
@@ -187,6 +118,14 @@ class LLMMessagesFrame(Frame):
     messages: List[dict]
 
 
+@dataclass()
+class OpenAILLMContextFrame(Frame):
+    """Like an LLMMessagesQueueFrame, but with extra context specific to the
+    OpenAI API. The context in this message is also mutable, and will be
+    changed by the OpenAIContextAggregator frame processor."""
+    context: OpenAILLMContext
+
+
 @dataclass()
 class ReceivedAppMessageFrame(Frame):
     message: Any
@@ -199,10 +138,10 @@ class ReceivedAppMessageFrame(Frame):
 @dataclass()
 class SendAppMessageFrame(Frame):
     message: Any
-    participant_id: str | None
+    participantId: str | None
 
     def __str__(self):
-        return f"SendAppMessageFrame: participant: {self.participant_id}, message: {self.message}"
+        return f"SendAppMessageFrame: participantId: {self.participantId}, message: {self.message}"
 
 
 class UserStartedSpeakingFrame(Frame):
@@ -240,3 +179,33 @@ class LLMFunctionCallFrame(Frame):
     """Emitted when the LLM has received an entire function call completion."""
     function_name: str
     arguments: str
+
+
+@dataclass()
+class VideoImageFrame(Frame):
+    """Contains a still image from a partcipant's video stream."""
+    participantId: str
+    image: bytes
+
+    # def __str__(self):
+    #     return f"{self.__class__.__name__}, participantId: {self.participantId}, image size: {len(self.image)} B"
+
+
+class TelestratorImageFrame(ImageFrame):
+    pass
+
+
+@dataclass()
+class VisionFrame(Frame):
+    prompt: str
+    image: bytes
+
+    # def __str__(self):
+    #     return f"{self.__class__.__name__}, prompt: {self.prompt}, image size: {len(self.image)} B"
+
+
+@dataclass()
+class RequestVideoImageFrame(Frame):
+    """Send to the transport to request a new video image from a specific participant. Leave participantId
+    empty to request a frame from all participants."""
+    participantId: str | None

src/dailyai/pipeline/merge_pipeline.py

@@ -12,7 +12,7 @@ class SequentialMergePipeline(Pipeline):
         self.pipelines = pipelines
 
     async def run_pipeline(self):
-        for idx, pipeline in enumerate(self.pipelines):
+        for pipeline in self.pipelines:
             while True:
                 frame = await pipeline.sink.get()
                 if isinstance(

src/dailyai/pipeline/opeanai_llm_aggregator.py

@@ -1,24 +1,18 @@
-from typing import AsyncGenerator, Callable
+from typing import Any, AsyncGenerator, Callable
 from dailyai.pipeline.frame_processor import FrameProcessor
 from dailyai.pipeline.frames import (
     Frame,
     LLMResponseEndFrame,
     LLMResponseStartFrame,
+    OpenAILLMContextFrame,
     TextFrame,
-    TranscriptionFrame,
+    TranscriptionQueueFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
 )
-from dailyai.pipeline.openai_frames import OpenAILLMContextFrame
 from dailyai.services.openai_llm_context import OpenAILLMContext
 
-try:
-    from openai.types.chat import ChatCompletionRole
-except ModuleNotFoundError as e:
-    print(f"Exception: {e}")
-    print(
-        "In order to use OpenAI, you need to `pip install dailyai[openai]`. Also, set `OPENAI_API_KEY` environment variable.")
-    raise Exception(f"Missing module: {e}")
+from openai.types.chat import ChatCompletionRole
 
 
 class OpenAIContextAggregator(FrameProcessor):
@@ -96,7 +90,7 @@ class OpenAIUserContextAggregator(OpenAIContextAggregator):
             role="user",
             start_frame=UserStartedSpeakingFrame,
             end_frame=UserStoppedSpeakingFrame,
-            accumulator_frame=TranscriptionFrame,
+            accumulator_frame=TranscriptionQueueFrame,
             pass_through=False,
         )
 

src/dailyai/pipeline/openai_frames.py

@@ -1,12 +0,0 @@
-from dataclasses import dataclass
-
-from dailyai.pipeline.frames import Frame
-from dailyai.services.openai_llm_context import OpenAILLMContext
-
-
-@dataclass()
-class OpenAILLMContextFrame(Frame):
-    """Like an LLMMessagesFrame, but with extra context specific to the
-    OpenAI API. The context in this message is also mutable, and will be
-    changed by the OpenAIContextAggregator frame processor."""
-    context: OpenAILLMContext

src/dailyai/pipeline/pipeline.py

@@ -1,9 +1,8 @@
 import asyncio
-import logging
 from typing import AsyncGenerator, AsyncIterable, Iterable, List
 from dailyai.pipeline.frame_processor import FrameProcessor
 
-from dailyai.pipeline.frames import AudioFrame, EndPipeFrame, EndFrame, Frame
+from dailyai.pipeline.frames import EndPipeFrame, EndFrame, Frame
 
 
 class Pipeline:
@@ -18,24 +17,18 @@ class Pipeline:
         self,
         processors: List[FrameProcessor],
         source: asyncio.Queue | None = None,
-        sink: asyncio.Queue[Frame] | None = None,
-        name: str | None = None,
+        sink: asyncio.Queue[Frame] | None = None
     ):
         """Create a new pipeline. By default we create the sink and source queues
         if they're not provided, but these can be overridden to point to other
         queues. If this pipeline is run by a transport, its sink and source queues
         will be overridden.
         """
-        self._processors: List[FrameProcessor] = processors
+        self.processors: List[FrameProcessor] = processors
 
         self.source: asyncio.Queue[Frame] = source or asyncio.Queue()
         self.sink: asyncio.Queue[Frame] = sink or asyncio.Queue()
 
-        self._logger = logging.getLogger("dailyai.pipeline")
-        self._last_log_line = ""
-        self._shown_repeated_log = False
-        self._name = name or str(id(self))
-
     def set_source(self, source: asyncio.Queue[Frame]):
         """Set the source queue for this pipeline. Frames from this queue
         will be processed by each frame_processor in the pipeline, or order
@@ -47,9 +40,6 @@ class Pipeline:
         has processed a frame, its output will be placed on this queue."""
         self.sink = sink
 
-    def add_processor(self, processor: FrameProcessor):
-        self._processors.append(processor)
-
     async def get_next_source_frame(self) -> AsyncGenerator[Frame, None]:
         """Convenience function to get the next frame from the source queue. This
         lets us consistently have an AsyncGenerator yield frames, from either the
@@ -81,8 +71,8 @@ class Pipeline:
 
         The source and sink queues must be set before calling this method.
 
-        This method will exit when an EndFrame is placed on the sink queue.
-        No more frames will be placed on the sink queue after an EndFrame, even
+        This method will exit when an EndStreamQueueFrame is placed on the sink queue.
+        No more frames will be placed on the sink queue after an EndStreamQueueFrame, even
         if it's not the last frame yielded by the last frame_processor in the pipeline..
         """
 
@@ -90,9 +80,8 @@ class Pipeline:
             while True:
                 initial_frame = await self.source.get()
                 async for frame in self._run_pipeline_recursively(
-                    initial_frame, self._processors
+                    initial_frame, self.processors
                 ):
-                    self._log_frame(frame, len(self._processors) + 1)
                     await self.sink.put(frame)
 
                 if isinstance(initial_frame, EndFrame) or isinstance(
@@ -102,48 +91,20 @@ class Pipeline:
         except asyncio.CancelledError:
             # this means there's been an interruption, do any cleanup necessary
             # here.
-            for processor in self._processors:
+            for processor in self.processors:
                 await processor.interrupted()
+            pass
 
     async def _run_pipeline_recursively(
-        self, initial_frame: Frame, processors: List[FrameProcessor], depth=1
+        self, initial_frame: Frame, processors: List[FrameProcessor]
     ) -> AsyncGenerator[Frame, None]:
         """Internal function to add frames to the pipeline as they're yielded
         by each processor."""
         if processors:
-            self._log_frame(initial_frame, depth)
             async for frame in processors[0].process_frame(initial_frame):
                 async for final_frame in self._run_pipeline_recursively(
-                    frame, processors[1:], depth + 1
+                    frame, processors[1:]
                 ):
                     yield final_frame
         else:
             yield initial_frame
-
-    def _log_frame(self, frame: Frame, depth: int):
-        """Log a frame as it moves through the pipeline. This is useful for debugging.
-        Note that this function inherits the logging level from the "dailyai" logger.
-        If you want debug output from dailyai in general but not this function (it is
-        noisy) you can silence this function by doing something like this:
-
-        # enable debug logging for the dailyai package.
-        logger = logging.getLogger("dailyai")
-        logger.setLevel(logging.DEBUG)
-
-        # silence the pipeline logging
-        logger = logging.getLogger("dailyai.pipeline")
-        logger.setLevel(logging.WARNING)
-        """
-        source = str(self._processors[depth - 2]) if depth > 1 else "source"
-        dest = str(self._processors[depth - 1]) if depth < (len(self._processors) + 1) else "sink"
-        prefix = self._name + "  " * depth
-        logline = prefix + " -> ".join([source, frame.__class__.__name__, dest])
-        if logline == self._last_log_line:
-            if self._shown_repeated_log:
-                return
-            self._shown_repeated_log = True
-            self._logger.debug(prefix + "... repeated")
-        else:
-            self._shown_repeated_log = False
-            self._last_log_line = logline
-            self._logger.debug(logline)

src/dailyai/pipeline/protobufs/frames_pb2.py

@@ -1,32 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: frames.proto
-# Protobuf Python Version: 4.25.3
-"""Generated protocol buffer code."""
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import descriptor_pool as _descriptor_pool
-from google.protobuf import symbol_database as _symbol_database
-from google.protobuf.internal import builder as _builder
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-
-
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x66rames.proto\x12\rdailyai_proto\"\x19\n\tTextFrame\x12\x0c\n\x04text\x18\x01 \x01(\t\"\x1a\n\nAudioFrame\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\"L\n\x12TranscriptionFrame\x12\x0c\n\x04text\x18\x01 \x01(\t\x12\x15\n\rparticipantId\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"\xa2\x01\n\x05\x46rame\x12(\n\x04text\x18\x01 \x01(\x0b\x32\x18.dailyai_proto.TextFrameH\x00\x12*\n\x05\x61udio\x18\x02 \x01(\x0b\x32\x19.dailyai_proto.AudioFrameH\x00\x12:\n\rtranscription\x18\x03 \x01(\x0b\x32!.dailyai_proto.TranscriptionFrameH\x00\x42\x07\n\x05\x66rameb\x06proto3')
-
-_globals = globals()
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'frames_pb2', _globals)
-if _descriptor._USE_C_DESCRIPTORS == False:
-  DESCRIPTOR._options = None
-  _globals['_TEXTFRAME']._serialized_start=31
-  _globals['_TEXTFRAME']._serialized_end=56
-  _globals['_AUDIOFRAME']._serialized_start=58
-  _globals['_AUDIOFRAME']._serialized_end=84
-  _globals['_TRANSCRIPTIONFRAME']._serialized_start=86
-  _globals['_TRANSCRIPTIONFRAME']._serialized_end=162
-  _globals['_FRAME']._serialized_start=165
-  _globals['_FRAME']._serialized_end=327
-# @@protoc_insertion_point(module_scope)

src/dailyai/serializers/abstract_frame_serializer.py

@@ -1,16 +0,0 @@
-from abc import abstractmethod
-
-from dailyai.pipeline.frames import Frame
-
-
-class FrameSerializer:
-    def __init__(self):
-        pass
-
-    @abstractmethod
-    def serialize(self, frame: Frame) -> bytes:
-        raise NotImplementedError()
-
-    @abstractmethod
-    def deserialize(self, data: bytes) -> Frame:
-        raise NotImplementedError

src/dailyai/serializers/protobuf_serializer.py

@@ -1,64 +0,0 @@
-import dataclasses
-from typing import Text
-from dailyai.pipeline.frames import AudioFrame, Frame, TextFrame, TranscriptionFrame
-import dailyai.pipeline.protobufs.frames_pb2 as frame_protos
-from dailyai.serializers.abstract_frame_serializer import FrameSerializer
-
-
-class ProtobufFrameSerializer(FrameSerializer):
-    SERIALIZABLE_TYPES = {
-        TextFrame: "text",
-        AudioFrame: "audio",
-        TranscriptionFrame: "transcription"
-    }
-
-    SERIALIZABLE_FIELDS = {v: k for k, v in SERIALIZABLE_TYPES.items()}
-
-    def __init__(self):
-        pass
-
-    def serialize(self, frame: Frame) -> bytes:
-        proto_frame = frame_protos.Frame()
-        if type(frame) not in self.SERIALIZABLE_TYPES:
-            raise ValueError(
-                f"Frame type {type(frame)} is not serializable. You may need to add it to ProtobufFrameSerializer.SERIALIZABLE_FIELDS.")
-
-        # ignoring linter errors; we check that type(frame) is in this dict above
-        proto_optional_name = self.SERIALIZABLE_TYPES[type(frame)]  # type: ignore
-        for field in dataclasses.fields(frame):  # type: ignore
-            setattr(getattr(proto_frame, proto_optional_name), field.name,
-                    getattr(frame, field.name))
-
-        return proto_frame.SerializeToString()
-
-    def deserialize(self, data: bytes) -> Frame:
-        """Returns a Frame object from a Frame protobuf. Used to convert frames
-        passed over the wire as protobufs to Frame objects used in pipelines
-        and frame processors.
-
-        >>> serializer = ProtobufFrameSerializer()
-        >>> serializer.deserialize(
-        ...     serializer.serialize(AudioFrame(data=b'1234567890')))
-        AudioFrame(data=b'1234567890')
-
-        >>> serializer.deserialize(
-        ...     serializer.serialize(TextFrame(text='hello world')))
-        TextFrame(text='hello world')
-
-        >>> serializer.deserialize(serializer.serialize(TranscriptionFrame(
-        ...     text="Hello there!", participantId="123", timestamp="2021-01-01")))
-        TranscriptionFrame(text='Hello there!', participantId='123', timestamp='2021-01-01')
-        """
-
-        proto = frame_protos.Frame.FromString(data)
-        which = proto.WhichOneof("frame")
-        if which not in self.SERIALIZABLE_FIELDS:
-            raise ValueError(
-                "Proto does not contain a valid frame. You may need to add a new case to ProtobufFrameSerializer.deserialize.")
-
-        class_name = self.SERIALIZABLE_FIELDS[which]
-        args = getattr(proto, which)
-        args_dict = {}
-        for field in proto.DESCRIPTOR.fields_by_name[which].message_type.fields:
-            args_dict[field.name] = getattr(args, field.name)
-        return class_name(**args_dict)

src/dailyai/services/ai_services.py

@@ -1,3 +1,4 @@
+import asyncio
 import io
 import logging
 import time
@@ -9,13 +10,15 @@ from dailyai.pipeline.frames import (
     EndFrame,
     EndPipeFrame,
     ImageFrame,
+    LLMMessagesQueueFrame,
+    LLMResponseEndFrame,
+    LLMResponseStartFrame,
+    LLMFunctionStartFrame,
+    LLMFunctionCallFrame,
     Frame,
-    TTSEndFrame,
-    TTSStartFrame,
     TextFrame,
-    TranscriptionFrame,
-    URLImageFrame,
-    VisionImageFrame,
+    TranscriptionQueueFrame,
+    VisionFrame
 )
 
 from abc import abstractmethod
@@ -51,20 +54,15 @@ class TTSService(AIService):
         # yield empty bytes here, so linting can infer what this method does
         yield bytes()
 
-    async def wrap_tts(self, text) -> AsyncGenerator[Frame, None]:
-        yield TTSStartFrame()
-        async for audio_chunk in self.run_tts(text):
-            yield AudioFrame(audio_chunk)
-        yield TTSEndFrame()
-        yield TextFrame(text)
-
     async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
         if isinstance(frame, EndFrame) or isinstance(frame, EndPipeFrame):
             if self.current_sentence:
-                async for cleanup_frame in self.wrap_tts(self.current_sentence):
-                    yield cleanup_frame
+                async for audio_chunk in self.run_tts(self.current_sentence):
+                    yield AudioFrame(audio_chunk)
+                yield TextFrame(self.current_sentence)
 
         if not isinstance(frame, TextFrame):
+            print(f"*** tts yielding non-text: {frame}")
             yield frame
             return
 
@@ -78,17 +76,23 @@ class TTSService(AIService):
                 self.current_sentence = ""
 
         if text:
-            async for frame in self.wrap_tts(text):
-                yield frame
+            async for audio_chunk in self.run_tts(text):
+                yield AudioFrame(audio_chunk)
+
+            # note we pass along the text frame *after* the audio, so the text
+            # frame is completed after the audio is processed.
+            print(f"*** tts yielding text: {text}")
+            yield TextFrame(text)
 
 
 class ImageGenService(AIService):
-    def __init__(self, **kwargs):
+    def __init__(self, image_size, **kwargs):
         super().__init__(**kwargs)
+        self.image_size = image_size
 
     # Renders the image. Returns an Image object.
     @abstractmethod
-    async def run_image_gen(self, prompt: str) -> tuple[str, bytes, tuple[int, int]]:
+    async def run_image_gen(self, sentence: str) -> tuple[str, bytes]:
         pass
 
     async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
@@ -96,27 +100,8 @@ class ImageGenService(AIService):
             yield frame
             return
 
-        (url, image_data, image_size) = await self.run_image_gen(frame.text)
-        yield URLImageFrame(url, image_data, image_size)
-
-
-class VisionService(AIService):
-    """VisionService is a base class for vision services."""
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self._describe_text = None
-
-    @abstractmethod
-    async def run_vision(self, frame: VisionImageFrame) -> str:
-        pass
-
-    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
-        if isinstance(frame, VisionImageFrame):
-            description = await self.run_vision(frame)
-            yield TextFrame(description)
-        else:
-            yield frame
+        (url, image_data) = await self.run_image_gen(frame.text)
+        yield ImageFrame(url, image_data)
 
 
 class STTService(AIService):
@@ -148,7 +133,28 @@ class STTService(AIService):
         ww.close()
         content.seek(0)
         text = await self.run_stt(content)
-        yield TranscriptionFrame(text, "", str(time.time()))
+        yield TranscriptionQueueFrame(text, "", str(time.time()))
+
+
+class VisionService(AIService):
+    def __init__(self):
+        super().__init__()
+
+    # Renders the image. Returns an Image object.
+    # TODO-CB: return type
+    @abstractmethod
+    async def run_vision(self, prompt: str, image: bytes):
+        pass
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, VisionFrame):
+            async for frame in self.run_vision(frame.prompt, frame.image):
+                print(
+                    f"&&& visionservce processframe got frame to yield: {frame}")
+                yield frame
+            yield LLMResponseEndFrame()
+        else:
+            yield frame
 
 
 class FrameLogger(AIService):
@@ -157,8 +163,9 @@ class FrameLogger(AIService):
         self.prefix = prefix
 
     async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
-        if isinstance(frame, (AudioFrame, ImageFrame)):
-            self.logger.info(f"{self.prefix}: {type(frame)}")
+        if isinstance(frame, (AudioFrame)):
+            # self.logger.info(f"{self.prefix}: {type(frame)}")
+            pass
         else:
             print(f"{self.prefix}: {frame}")
 

src/dailyai/services/anthropic_llm_service.py

@@ -1,16 +1,11 @@
+import asyncio
+import os
 from typing import AsyncGenerator
-from dailyai.pipeline.frames import Frame, LLMMessagesFrame, TextFrame
+from anthropic import AsyncAnthropic
+from dailyai.pipeline.frames import Frame, LLMMessagesQueueFrame, TextFrame
 
 from dailyai.services.ai_services import LLMService
 
-try:
-    from anthropic import AsyncAnthropic
-except ModuleNotFoundError as e:
-    print(f"Exception: {e}")
-    print(
-        "In order to use Anthropic, you need to `pip install dailyai[anthropic]`. Also, set `ANTHROPIC_API_KEY` environment variable.")
-    raise Exception(f"Missing module: {e}")
-
 
 class AnthropicLLMService(LLMService):
 
@@ -25,7 +20,7 @@ class AnthropicLLMService(LLMService):
         self.max_tokens = max_tokens
 
     async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
-        if not isinstance(frame, LLMMessagesFrame):
+        if not isinstance(frame, LLMMessagesQueueFrame):
             yield frame
 
         stream = await self.client.messages.create(

src/dailyai/services/azure_ai_services.py

@@ -1,26 +1,25 @@
 import aiohttp
 import asyncio
 import io
+import json
+import time
 from openai import AsyncAzureOpenAI
 
+import os
+import requests
+
 from collections.abc import AsyncGenerator
 
-from dailyai.services.ai_services import TTSService, ImageGenService
+from dailyai.services.ai_services import LLMService, TTSService, ImageGenService
 from PIL import Image
 
 # See .env.example for Azure configuration needed
-try:
-    from azure.cognitiveservices.speech import (
-        SpeechSynthesizer,
-        SpeechConfig,
-        ResultReason,
-        CancellationReason,
-    )
-except ModuleNotFoundError as e:
-    print(f"Exception: {e}")
-    print(
-        "In order to use Azure TTS, you need to `pip install dailyai[azure]`. Also, set `AZURE_SPEECH_API_KEY` and `AZURE_SPEECH_REGION` environment variables.")
-    raise Exception(f"Missing module: {e}")
+from azure.cognitiveservices.speech import (
+    SpeechSynthesizer,
+    SpeechConfig,
+    ResultReason,
+    CancellationReason,
+)
 
 from dailyai.services.openai_api_llm_service import BaseOpenAILLMService
 
@@ -97,24 +96,23 @@ class AzureImageGenServiceREST(ImageGenService):
         endpoint,
         model,
     ):
-        super().__init__()
+        super().__init__(image_size=image_size)
 
         self._api_key = api_key
         self._azure_endpoint = endpoint
         self._api_version = api_version
         self._model = model
         self._aiohttp_session = aiohttp_session
-        self._image_size = image_size
 
-    async def run_image_gen(self, prompt: str) -> tuple[str, bytes, tuple[int, int]]:
+    async def run_image_gen(self, sentence) -> tuple[str, bytes]:
         url = f"{self._azure_endpoint}openai/images/generations:submit?api-version={self._api_version}"
         headers = {
             "api-key": self._api_key,
             "Content-Type": "application/json"}
         body = {
             # Enter your prompt text here
-            "prompt": prompt,
-            "size": self._image_size,
+            "prompt": sentence,
+            "size": self.image_size,
             "n": 1,
         }
         async with self._aiohttp_session.post(
@@ -147,4 +145,4 @@ class AzureImageGenServiceREST(ImageGenService):
             async with self._aiohttp_session.get(image_url) as response:
                 image_stream = io.BytesIO(await response.content.read())
                 image = Image.open(image_stream)
-                return (image_url, image.tobytes(), image.size)
+                return (image_url, image.tobytes())

src/dailyai/services/base_transport_service.py

@@ -1,8 +1,10 @@
 from abc import abstractmethod
 import asyncio
 import itertools
+import logging
 import numpy as np
-
+import pyaudio
+import torch
 import queue
 import threading
 import time
@@ -20,28 +22,49 @@ from dailyai.pipeline.frames import (
     SpriteFrame,
     StartFrame,
     TextFrame,
-    UserImageRequestFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
+    RequestVideoImageFrame,
+    TelestratorImageFrame
 )
 from dailyai.pipeline.pipeline import Pipeline
 from dailyai.services.ai_services import TTSService
-from dailyai.transports.abstract_transport import AbstractTransport
+
+torch.set_num_threads(1)
+
+model, utils = torch.hub.load(
+    repo_or_dir="snakers4/silero-vad", model="silero_vad", force_reload=False
+)
+
+(get_speech_timestamps, save_audio, read_audio, VADIterator, collect_chunks) = utils
+
+# Taken from utils_vad.py
+
+
+def validate(model, inputs: torch.Tensor):
+    with torch.no_grad():
+        outs = model(inputs)
+    return outs
 
 
 # Provided by Alexander Veysov
 
 
 def int2float(sound):
-    try:
-        abs_max = np.abs(sound).max()
-        sound = sound.astype("float32")
-        if abs_max > 0:
-            sound *= 1 / 32768
-        sound = sound.squeeze()  # depends on the use case
-        return sound
-    except ValueError:
-        return sound
+    abs_max = np.abs(sound).max()
+    sound = sound.astype("float32")
+    if abs_max > 0:
+        sound *= 1 / 32768
+    sound = sound.squeeze()  # depends on the use case
+    return sound
+
+
+FORMAT = pyaudio.paInt16
+CHANNELS = 1
+SAMPLE_RATE = 16000
+CHUNK = int(SAMPLE_RATE / 10)
+
+audio = pyaudio.PyAudio()
 
 
 class VADState(Enum):
@@ -51,50 +74,34 @@ class VADState(Enum):
     STOPPING = 4
 
 
-class ThreadedTransport(AbstractTransport):
+class BaseTransportService:
 
     def __init__(
         self,
         **kwargs,
     ) -> None:
-        super().__init__(**kwargs)
+        self._mic_enabled = kwargs.get("mic_enabled") or False
+        self._mic_sample_rate = kwargs.get("mic_sample_rate") or 16000
+        self._camera_enabled = kwargs.get("camera_enabled") or False
+        self._camera_width = kwargs.get("camera_width") or 1024
+        self._camera_height = kwargs.get("camera_height") or 768
+        self._speaker_enabled = kwargs.get("speaker_enabled") or False
+        self._speaker_sample_rate = kwargs.get("speaker_sample_rate") or 16000
+        self._fps = kwargs.get("fps") or 8
         self._vad_start_s = kwargs.get("vad_start_s") or 0.2
         self._vad_stop_s = kwargs.get("vad_stop_s") or 0.8
         self._context = kwargs.get("context") or []
         self._vad_enabled = kwargs.get("vad_enabled") or False
-        self._has_webrtc_vad = kwargs.get("has_webrtc_vad") or False
+        self._receive_video = kwargs.get("receive_video") or False
+        self._receive_video_fps = kwargs.get("receive_video_fps") or 0.0
+        self._participant_frame_times = {}
         if self._vad_enabled and self._speaker_enabled:
             raise Exception(
                 "Sorry, you can't use speaker_enabled and vad_enabled at the same time. Please set one to False."
             )
+
         self._vad_samples = 1536
-
-        if self._vad_enabled:
-            try:
-                global torch, torchaudio
-                import torch
-                # We don't use torchaudio here, but we need to try importing it because
-                # Silero uses it
-                import torchaudio
-                torch.set_num_threads(1)
-
-                (self.model, self.utils) = torch.hub.load(
-                    repo_or_dir="snakers4/silero-vad", model="silero_vad", force_reload=False
-                )
-                self._logger.debug("Loaded Silero VAD")
-
-            except ModuleNotFoundError as e:
-                if self._has_webrtc_vad:
-                    self._logger.debug(
-                        f"Couldn't load torch; using webrtc VAD")
-                    self._vad_samples = int(self._speaker_sample_rate / 100.0)
-                else:
-                    self._logger.error(f"Exception: {e}")
-                    self._logger.error(
-                        "In order to use Silero VAD, you'll need to `pip install dailyai[silero].")
-                    raise Exception(f"Missing module(s): {e}")
-
-        vad_frame_s = self._vad_samples / self._speaker_sample_rate
+        vad_frame_s = self._vad_samples / SAMPLE_RATE
         self._vad_start_frames = round(self._vad_start_s / vad_frame_s)
         self._vad_stop_frames = round(self._vad_stop_s / vad_frame_s)
         self._vad_starting_count = 0
@@ -102,6 +109,14 @@ class ThreadedTransport(AbstractTransport):
         self._vad_state = VADState.QUIET
         self._user_is_speaking = False
 
+        duration_minutes = kwargs.get("duration_minutes") or 10
+        self._expiration = time.time() + duration_minutes * 60
+
+        self.send_queue = asyncio.Queue()
+        self.receive_queue = asyncio.Queue()
+
+        self.completed_queue = asyncio.Queue()
+
         self._threadsafe_send_queue = queue.Queue()
 
         self._images = None
@@ -114,6 +129,8 @@ class ThreadedTransport(AbstractTransport):
         self._stop_threads = threading.Event()
         self._is_interrupted = threading.Event()
 
+        self._logger: logging.Logger = logging.getLogger()
+
     async def run(self, pipeline: Pipeline | None = None, override_pipeline_source_queue=True):
         self._prerun()
 
@@ -180,12 +197,14 @@ class ThreadedTransport(AbstractTransport):
     async def run_interruptible_pipeline(
         self,
         pipeline: Pipeline,
-        pre_processor: FrameProcessor | None = None,
+        allow_interruptions=True,
+        pre_processor=None,
         post_processor: FrameProcessor | None = None,
     ):
         pipeline.set_sink(self.send_queue)
         source_queue = asyncio.Queue()
         pipeline.set_source(source_queue)
+        pipeline.set_sink(self.send_queue)
         pipeline_task = asyncio.create_task(pipeline.run_pipeline())
 
         async def yield_frame(frame: Frame) -> AsyncGenerator[Frame, None]:
@@ -269,32 +288,19 @@ class ThreadedTransport(AbstractTransport):
     def _prerun(self):
         pass
 
-    def _silero_vad_analyze(self):
-        try:
+    def _vad(self):
+        # CB: Starting silero VAD stuff
+        # TODO-CB: Probably need to force virtual speaker creation if we're
+        # going to build this in?
+        # TODO-CB: pyaudio installation
+        while not self._stop_threads.is_set():
             audio_chunk = self.read_audio_frames(self._vad_samples)
             audio_int16 = np.frombuffer(audio_chunk, np.int16)
             audio_float32 = int2float(audio_int16)
-            new_confidence = self.model(
+            new_confidence = model(
                 torch.from_numpy(audio_float32), 16000).item()
-            # yeses = int(new_confidence * 20.0)
-            # nos = 20 - yeses
-            # out = "!" * yeses + "." * nos
-            # print(f"!!! confidence: {out}")
             speaking = new_confidence > 0.5
-            return speaking
-        except BaseException:
-            # This comes from an empty audio array
-            return False
 
-    def _vad(self):
-
-        while not self._stop_threads.is_set():
-            if hasattr(self, 'model'):  # we can use Silero
-                speaking = self._silero_vad_analyze()
-            elif self._has_webrtc_vad:
-                speaking = self._webrtc_vad_analyze()
-            else:
-                raise Exception("VAD is running with no VAD service available")
             if speaking:
                 match self._vad_state:
                     case VADState.QUIET:
@@ -331,7 +337,6 @@ class ThreadedTransport(AbstractTransport):
                 self._vad_state == VADState.STOPPING
                 and self._vad_stopping_count >= self._vad_stop_frames
             ):
-
                 if self._loop:
                     asyncio.run_coroutine_threadsafe(
                         self.receive_queue.put(
@@ -383,11 +388,7 @@ class ThreadedTransport(AbstractTransport):
     def _set_images(self, images: list[bytes], start_frame=0):
         self._images = itertools.cycle(images)
 
-    def request_participant_image(self, participant_id: str):
-        """ Child classes should override this to force an image from a user. """
-        pass
-
-    def send_app_message(self, message: Any, participant_id: str | None):
+    def send_app_message(self, message: Any, participantId: str | None):
         """ Child classes should override this to send a custom message to the room. """
         pass
 
@@ -398,7 +399,7 @@ class ThreadedTransport(AbstractTransport):
                     this_frame = next(self._images)
                     self.write_frame_to_camera(this_frame)
 
-                time.sleep(1.0 / self._camera_framerate)
+                time.sleep(1.0 / self._fps)
         except Exception as e:
             self._logger.error(f"Exception {e} in camera thread.")
             raise e
@@ -438,16 +439,13 @@ class ThreadedTransport(AbstractTransport):
                             asyncio.run_coroutine_threadsafe(
                                 self.completed_queue.put(frame), self._loop
                             )
-                            # Also send the EndFrame to the pipeline so it can stop
-                            asyncio.run_coroutine_threadsafe(
-                                self.receive_queue.put(frame), self._loop
-                            )
                         return
 
                     # if interrupted, we just pull frames off the queue and
                     # discard them
                     if not self._is_interrupted.is_set():
                         if frame:
+
                             if isinstance(frame, AudioFrame):
                                 chunk = frame.data
 
@@ -459,14 +457,28 @@ class ThreadedTransport(AbstractTransport):
                                     self.write_frame_to_mic(
                                         bytes(b[:truncated_length]))
                                     b = b[truncated_length:]
+                            elif isinstance(frame, TelestratorImageFrame):
+                                self._set_image(frame.image)
+                                asyncio.run_coroutine_threadsafe(
+                                    self.receive_queue.put(frame),
+                                    self._loop,
+                                )
                             elif isinstance(frame, ImageFrame):
                                 self._set_image(frame.image)
                             elif isinstance(frame, SpriteFrame):
                                 self._set_images(frame.images)
-                            elif isinstance(frame, UserImageRequestFrame):
-                                self.request_participant_image(frame.user_id)
                             elif isinstance(frame, SendAppMessageFrame):
-                                self.send_app_message(frame.message, frame.participant_id)
+                                self.send_app_message(
+                                    frame.message, frame.participantId)
+                            elif isinstance(frame, RequestVideoImageFrame):
+                                # removing one or all participant IDs from _participant_frame_times
+                                # will cause the transport to send the next available frame from
+                                # that participant
+                                if frame.participantId:
+                                    self._participant_frame_times.pop(
+                                        frame.participantId, None)
+                                else:
+                                    self._participant_frame_times.clear()
                         elif len(b):
                             self.write_frame_to_mic(bytes(b))
                             b = bytearray()

src/dailyai/services/daily_transport_service.py

@@ -11,37 +11,26 @@ from typing import Any
 
 from dailyai.pipeline.frames import (
     ReceivedAppMessageFrame,
-    TranscriptionFrame,
-    UserImageFrame,
+    TranscriptionQueueFrame,
+    VideoImageFrame,
+    TelestratorImageFrame
 )
 
 from threading import Event
 
-try:
-    from daily import (
-        EventHandler,
-        CallClient,
-        Daily,
-        VirtualCameraDevice,
-        VirtualMicrophoneDevice,
-        VirtualSpeakerDevice,
-    )
-except ModuleNotFoundError as e:
-    print(f"Exception: {e}")
-    print(
-        "In order to use the Daily transport, you need to `pip install dailyai[daily]`.")
-    raise Exception(f"Missing module: {e}")
+from daily import (
+    EventHandler,
+    CallClient,
+    Daily,
+    VirtualCameraDevice,
+    VirtualMicrophoneDevice,
+    VirtualSpeakerDevice,
+)
+
+from dailyai.services.base_transport_service import BaseTransportService
 
 
-from dailyai.transports.threaded_transport import ThreadedTransport
-
-NUM_CHANNELS = 1
-
-SPEECH_THRESHOLD = 0.90
-VAD_RESET_PERIOD_MS = 2000
-
-
-class DailyTransport(ThreadedTransport, EventHandler):
+class DailyTransportService(BaseTransportService, EventHandler):
     _daily_initialized = False
     _lock = threading.Lock()
 
@@ -60,11 +49,9 @@ class DailyTransport(ThreadedTransport, EventHandler):
         bot_name: str,
         min_others_count: int = 1,
         start_transcription: bool = False,
-        video_rendering_enabled: bool = False,
         **kwargs,
     ):
-        kwargs['has_webrtc_vad'] = True
-        # This will call ThreadedTransport.__init__ method, not EventHandler
+        # This will call BaseTransportService.__init__ method, not EventHandler
         super().__init__(**kwargs)
 
         self._room_url: str = room_url
@@ -72,7 +59,6 @@ class DailyTransport(ThreadedTransport, EventHandler):
         self._token: str | None = token
         self._min_others_count = min_others_count
         self._start_transcription = start_transcription
-        self._video_rendering_enabled = video_rendering_enabled
 
         self._is_interrupted = Event()
         self._stop_threads = Event()
@@ -80,8 +66,6 @@ class DailyTransport(ThreadedTransport, EventHandler):
         self._other_participant_has_joined = False
         self._my_participant_id = None
 
-        self._video_renderers = {}
-
         self.transcription_settings = {
             "language": "en",
             "tier": "nova",
@@ -98,12 +82,6 @@ class DailyTransport(ThreadedTransport, EventHandler):
 
         self._event_handlers = {}
 
-        self.webrtc_vad = Daily.create_native_vad(
-            reset_period_ms=VAD_RESET_PERIOD_MS,
-            sample_rate=self._speaker_sample_rate,
-            channels=NUM_CHANNELS
-        )
-
     def _patch_method(self, event_name, *args, **kwargs):
         try:
             for handler in self._event_handlers[event_name]:
@@ -124,17 +102,6 @@ class DailyTransport(ThreadedTransport, EventHandler):
             self._logger.error(f"Exception in event handler {event_name}: {e}")
             raise e
 
-    def _webrtc_vad_analyze(self):
-        buffer = self.read_audio_frames(int(self._vad_samples))
-        if len(buffer) > 0:
-            confidence = self.webrtc_vad.analyze_frames(buffer)
-            # yeses = int(confidence * 20.0)
-            # nos = 20 - yeses
-            # out = "!" * yeses + "." * nos
-            # print(f"!!! confidence: {out} {confidence}")
-            talking = confidence > SPEECH_THRESHOLD
-            return talking
-
     def add_event_handler(self, event_name: str, handler):
         if not event_name.startswith("on_"):
             raise Exception(
@@ -162,32 +129,24 @@ class DailyTransport(ThreadedTransport, EventHandler):
         return decorator
 
     def write_frame_to_camera(self, frame: bytes):
-        if self._camera_enabled:
-            self.camera.write_frame(frame)
+        self.camera.write_frame(frame)
 
     def write_frame_to_mic(self, frame: bytes):
-        if self._mic_enabled:
-            self.mic.write_frames(frame)
+        self.mic.write_frames(frame)
 
-    def request_participant_image(self, participant_id: str):
-        if participant_id in self._video_renderers:
-            self._video_renderers[participant_id]["render_next_frame"] = True
-
-    def send_app_message(self, message: Any, participant_id: str | None):
-        self.client.send_app_message(message, participant_id)
+    def send_app_message(self, message: Any, participantId: str | None):
+        self.client.send_app_message(message, participantId)
 
     def read_audio_frames(self, desired_frame_count):
-        bytes = b""
-        if self._speaker_enabled or self._vad_enabled:
-            bytes = self._speaker.read_frames(desired_frame_count)
+        bytes = self._speaker.read_frames(desired_frame_count)
         return bytes
 
     def _prerun(self):
         # Only initialize Daily once
-        if not DailyTransport._daily_initialized:
-            with DailyTransport._lock:
+        if not DailyTransportService._daily_initialized:
+            with DailyTransportService._lock:
                 Daily.init()
-                DailyTransport._daily_initialized = True
+                DailyTransportService._daily_initialized = True
         self.client = CallClient(event_handler=self)
 
         if self._mic_enabled:
@@ -236,9 +195,9 @@ class DailyTransport(ThreadedTransport, EventHandler):
                             "maxQuality": "low",
                             "encodings": {
                                 "low": {
-                                    "maxBitrate": self._camera_bitrate,
+                                    "maxBitrate": 250000,
                                     "scaleResolutionDownBy": 1.333,
-                                    "maxFramerate": self._camera_framerate,
+                                    "maxFramerate": 8,
                                 }
                             },
                         }
@@ -248,14 +207,12 @@ class DailyTransport(ThreadedTransport, EventHandler):
         )
         self._my_participant_id = self.client.participants()["local"]["id"]
 
-        # For performance reasons, never subscribe to video streams (unless a
-        # video renderer is registered).
-        self.client.update_subscription_profiles({
-            "base": {
-                "camera": "unsubscribed",
-                "screenVideo": "unsubscribed"
-            }
-        })
+        if not self._receive_video:
+            self.client.update_subscription_profiles({
+                "base": {
+                    "camera": "unsubscribed",
+                }
+            })
 
         if self._token and self._start_transcription:
             self.client.start_transcription(self.transcription_settings)
@@ -272,7 +229,32 @@ class DailyTransport(ThreadedTransport, EventHandler):
         self.client.leave()
         self.client.release()
 
-    def on_first_other_participant_joined(self, participant):
+    def _handle_video_frame(self, participant_id, video_frame):
+        """If receive_video is true, this function is called once for each frame from each participant. We
+         don't need to send every frame to the pipeline, so there are two ways to decide how to send frames:
+         1. Set a greater-than-zero value for receive_video_fps. The transport will track the last send time
+            for each participant and send a new frame when the requested frame rate has elapsed. This
+            guarantees an image every second, for example.
+         2. Set receive_video_fps less than or equal to zero to disable timed frame sending. Then, put a
+            RequestVideoImageFrame in the pipeline to get a new frame for one or all participants. By
+            sending a RequestVideoImageFrame immediately after successfully processing an image, you can
+            ensure you don't end up queueing up frames faster than you can process them.
+            """
+        send_frame = False
+        if not participant_id in self._participant_frame_times:
+            # then it's a new participant; send the first frame
+            send_frame = True
+        elif self._receive_video_fps > 0 and time.time() > self._participant_frame_times[participant_id] + 1.0/self._receive_video_fps:
+            # Then it's an existing participant who is due to send a new frame
+            send_frame = True
+
+        if send_frame:
+            self._participant_frame_times[participant_id] = time.time()
+            future = asyncio.run_coroutine_threadsafe(
+                self.receive_queue.put(
+                    VideoImageFrame(participant_id, video_frame)), self._loop)
+
+    def on_first_other_participant_joined(self):
         pass
 
     def call_joined(self, join_data, client_error):
@@ -285,59 +267,6 @@ class DailyTransport(ThreadedTransport, EventHandler):
     def start_recording(self):
         self.client.start_recording()
 
-    def render_participant_video(self,
-                                 participant_id,
-                                 framerate=10,
-                                 video_source="camera",
-                                 color_format="RGB") -> None:
-        if not self._video_rendering_enabled:
-            self._logger.warn("Video rendering is not enabled")
-            return
-
-        # Only enable camera subscription on this participant
-        self.client.update_subscriptions(participant_settings={
-            participant_id: {
-                "media": {
-                    video_source: "subscribed"
-                }
-            }
-        })
-
-        self._video_renderers[participant_id] = {
-            "framerate": framerate,
-            "timestamp": 0,
-            "render_next_frame": False,
-        }
-        self.client.set_video_renderer(
-            participant_id,
-            self.on_participant_video_frame,
-            video_source=video_source,
-            color_format=color_format)
-
-    def on_participant_video_frame(self, participant_id, video_frame):
-        if not self._loop:
-            return
-
-        render_frame = False
-
-        curr_time = time.time()
-        framerate = self._video_renderers[participant_id]["framerate"]
-
-        if framerate > 0:
-            prev_time = self._video_renderers[participant_id]["timestamp"]
-            next_time = prev_time + 1 / framerate
-            render_frame = curr_time > next_time
-        elif self._video_renderers[participant_id]["render_next_frame"]:
-            self._video_renderers[participant_id]["render_next_frame"] = False
-            render_frame = True
-
-        if render_frame:
-            frame = UserImageFrame(participant_id, video_frame.buffer,
-                                   (video_frame.width, video_frame.height))
-            asyncio.run_coroutine_threadsafe(self.receive_queue.put(frame), self._loop)
-
-        self._video_renderers[participant_id]["timestamp"] = curr_time
-
     def on_error(self, error):
         self._logger.error(f"on_error: {error}")
 
@@ -347,7 +276,10 @@ class DailyTransport(ThreadedTransport, EventHandler):
     def on_participant_joined(self, participant):
         if not self._other_participant_has_joined and participant["id"] != self._my_participant_id:
             self._other_participant_has_joined = True
-            self.on_first_other_participant_joined(participant)
+            self.on_first_other_participant_joined()
+        if self._receive_video:
+            self.client.set_video_renderer(
+                participant["id"], self._handle_video_frame)
 
     def on_participant_left(self, participant, reason):
         if len(self.client.participants()) < self._min_others_count + 1:
@@ -356,6 +288,7 @@ class DailyTransport(ThreadedTransport, EventHandler):
     def on_app_message(self, message: Any, sender: str):
         if self._loop:
             frame = ReceivedAppMessageFrame(message, sender)
+            print(frame)
             asyncio.run_coroutine_threadsafe(
                 self.receive_queue.put(frame), self._loop
             )
@@ -368,7 +301,7 @@ class DailyTransport(ThreadedTransport, EventHandler):
             elif "session_id" in message:
                 participantId = message["session_id"]
             if self._my_participant_id and participantId != self._my_participant_id:
-                frame = TranscriptionFrame(
+                frame = TranscriptionQueueFrame(
                     message["text"], participantId, message["timestamp"])
                 asyncio.run_coroutine_threadsafe(
                     self.receive_queue.put(frame), self._loop)

src/dailyai/services/deepgram_ai_service.py

@@ -1,4 +1,6 @@
+import os
 import aiohttp
+import requests
 
 from dailyai.services.ai_services import TTSService
 

src/dailyai/services/deepgram_ai_services.py

@@ -1,3 +1,9 @@
+import aiohttp
+import asyncio
+import os
+
+import requests
+
 from collections.abc import AsyncGenerator
 from dailyai.services.ai_services import TTSService
 

src/dailyai/services/elevenlabs_ai_service.py

@@ -1,4 +1,7 @@
 import aiohttp
+import os
+import requests
+import time
 
 from typing import AsyncGenerator
 
@@ -12,18 +15,19 @@ class ElevenLabsTTSService(TTSService):
         *,
         aiohttp_session: aiohttp.ClientSession,
         api_key,
-        voice_id,
+        narrator,
         model="eleven_turbo_v2",
+        aggregate_sentences=True
     ):
-        super().__init__()
+        super().__init__(aggregate_sentences)
 
         self._api_key = api_key
-        self._voice_id = voice_id
+        self._narrator = narrator
         self._aiohttp_session = aiohttp_session
         self._model = model
 
     async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]:
-        url = f"https://api.elevenlabs.io/v1/text-to-speech/{self._voice_id}/stream"
+        url = f"https://api.elevenlabs.io/v1/text-to-speech/{self._narrator['narrator']['voice_id']}/stream"
         payload = {"text": sentence, "model_id": self._model}
         querystring = {
             "output_format": "pcm_16000",
@@ -32,6 +36,7 @@ class ElevenLabsTTSService(TTSService):
             "xi-api-key": self._api_key,
             "Content-Type": "application/json",
         }
+
         async with self._aiohttp_session.post(
             url, json=payload, headers=headers, params=querystring
         ) as r:

src/dailyai/services/fal_ai_services.py

@@ -1,61 +1,43 @@
+import fal
 import aiohttp
 import asyncio
 import io
 import os
 from PIL import Image
-from pydantic import BaseModel
-from typing import Optional, Union, Dict
 
 from dailyai.services.ai_services import ImageGenService
 
-try:
-    import fal
-except ModuleNotFoundError as e:
-    print(f"Exception: {e}")
-    print(
-        "In order to use Fal, you need to `pip install dailyai[fal]`. Also, set `FAL_KEY_ID` and `FAL_KEY_SECRET` environment variables.")
-    raise Exception(f"Missing module: {e}")
+
+from dailyai.services.ai_services import ImageGenService
+
+# Fal expects FAL_KEY_ID and FAL_KEY_SECRET to be set in the env
 
 
 class FalImageGenService(ImageGenService):
-    class InputParams(BaseModel):
-        seed: Optional[int] = None
-        num_inference_steps: int = 4
-        num_images: int = 1
-        image_size: Union[str, Dict[str, int]] = "square_hd"
-        expand_prompt: bool = False
-        enable_safety_checker: bool = True
-        format: str = "png"
-
     def __init__(
         self,
         *,
+        image_size,
         aiohttp_session: aiohttp.ClientSession,
-        params: InputParams,
-        model="fal-ai/fast-sdxl",
         key_id=None,
         key_secret=None
     ):
-        super().__init__()
-        self._model = model
-        self._params = params
+        super().__init__(image_size)
         self._aiohttp_session = aiohttp_session
         if key_id:
             os.environ["FAL_KEY_ID"] = key_id
         if key_secret:
             os.environ["FAL_KEY_SECRET"] = key_secret
 
-    async def run_image_gen(self, prompt: str) -> tuple[str, bytes, tuple[int, int]]:
-        def get_image_url(prompt):
-            handler = fal.apps.submit(  # type: ignore
-                self._model,
-                arguments={
-                    "prompt": prompt,
-                    **self._params.dict(),
-                },
+    async def run_image_gen(self, sentence) -> tuple[str, bytes]:
+        def get_image_url(sentence, size):
+            handler = fal.apps.submit(
+                "110602490-fast-sdxl",
+                # "fal-ai/fast-sdxl",
+                arguments={"prompt": sentence},
             )
             for event in handler.iter_events():
-                if isinstance(event, fal.apps.InProgress):  # type: ignore
+                if isinstance(event, fal.apps.InProgress):
                     pass
 
             result = handler.get()
@@ -66,10 +48,12 @@ class FalImageGenService(ImageGenService):
 
             return image_url
 
-        image_url = await asyncio.to_thread(get_image_url, prompt)
-
+        image_url = await asyncio.to_thread(get_image_url, sentence, self.image_size)
         # Load the image from the url
         async with self._aiohttp_session.get(image_url) as response:
             image_stream = io.BytesIO(await response.content.read())
             image = Image.open(image_stream)
-            return (image_url, image.tobytes(), image.size)
+            image_bytes = image.tobytes()
+            print(f"!!! fal image tobytes is:")
+            print(image)
+            return (image_url, image_bytes)

src/dailyai/services/local_stt_service.py

@@ -4,7 +4,7 @@ import math
 import time
 from typing import AsyncGenerator
 import wave
-from dailyai.pipeline.frames import AudioFrame, Frame, TranscriptionFrame
+from dailyai.pipeline.frames import AudioFrame, Frame, TranscriptionQueueFrame
 from dailyai.services.ai_services import STTService
 
 
@@ -42,7 +42,6 @@ class LocalSTTService(STTService):
     async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
         """Processes a frame of audio data, either buffering or transcribing it."""
         if not isinstance(frame, AudioFrame):
-            yield frame
             return
 
         data = frame.data
@@ -61,7 +60,7 @@ class LocalSTTService(STTService):
             self._content.seek(0)
             text = await self.run_stt(self._content)
             self._new_wave()
-            yield TranscriptionFrame(text, '', str(time.time()))
+            yield TranscriptionQueueFrame(text, '', str(time.time()))
         # If we get this far, this is a frame of silence
         self._current_silence_frames += 1
 

src/dailyai/services/local_transport_service.py

@@ -1,25 +1,18 @@
 import asyncio
+import time
 import numpy as np
 import tkinter as tk
+import pyaudio
 
-from dailyai.transports.threaded_transport import ThreadedTransport
-
-try:
-    import pyaudio
-except ModuleNotFoundError as e:
-    print(f"Exception: {e}")
-    print(
-        "In order to use the local transport, you need to `pip install dailyai[local]`. On MacOS, you also need to `brew install portaudio`.")
-    raise Exception(f"Missing module: {e}")
+from dailyai.services.base_transport_service import BaseTransportService
 
 
-class LocalTransport(ThreadedTransport):
+class LocalTransportService(BaseTransportService):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self._sample_width = kwargs.get("sample_width") or 2
         self._n_channels = kwargs.get("n_channels") or 1
         self._tk_root = kwargs.get("tk_root") or None
-        self._pyaudio = None
 
         if self._camera_enabled and not self._tk_root:
             raise ValueError(
@@ -49,22 +42,18 @@ class LocalTransport(ThreadedTransport):
             )
 
     def write_frame_to_mic(self, frame: bytes):
-        if self._mic_enabled:
-            self._audio_stream.write(frame)
+        self._audio_stream.write(frame)
 
-    def read_audio_frames(self, desired_frame_count):
-        bytes = b""
-        if self._speaker_enabled:
-            bytes = self._speaker_stream.read(
-                desired_frame_count,
-                exception_on_overflow=False,
-            )
+    def read_frames(self, desired_frame_count):
+        bytes = self._speaker_stream.read(
+            desired_frame_count,
+            exception_on_overflow=False,
+        )
         return bytes
 
     def _prerun(self):
         if self._mic_enabled:
-            if not self._pyaudio:
-                self._pyaudio = pyaudio.PyAudio()
+            self._pyaudio = pyaudio.PyAudio()
             self._audio_stream = self._pyaudio.open(
                 format=self._pyaudio.get_format_from_width(self._sample_width),
                 channels=self._n_channels,
@@ -86,8 +75,6 @@ class LocalTransport(ThreadedTransport):
             self._image_label.pack()
 
         if self._speaker_enabled:
-            if not self._pyaudio:
-                self._pyaudio = pyaudio.PyAudio()
             self._speaker_stream = self._pyaudio.open(
                 format=self._pyaudio.get_format_from_width(self._sample_width),
                 channels=self._n_channels,

src/dailyai/services/moondream_ai_service.py

@@ -1,52 +0,0 @@
-from dailyai.pipeline.frames import ImageFrame, VisionImageFrame
-from dailyai.services.ai_services import VisionService
-
-from PIL import Image
-
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-import torch
-
-
-def detect_device():
-    """
-    Detects the appropriate device to run on, and return the device and dtype.
-    """
-    if torch.cuda.is_available():
-        return torch.device("cuda"), torch.float16
-    elif torch.backends.mps.is_available():
-        return torch.device("mps"), torch.float16
-    else:
-        return torch.device("cpu"), torch.float32
-
-
-class MoondreamService(VisionService):
-    def __init__(
-        self,
-        model_id="vikhyatk/moondream2",
-        revision="2024-04-02",
-        device=None
-    ):
-        super().__init__()
-
-        if not device:
-            device, dtype = detect_device()
-        else:
-            device = torch.device("cpu")
-            dtype = torch.float32
-
-        self._tokenizer = AutoTokenizer.from_pretrained(model_id, revision=revision)
-
-        self._model = AutoModelForCausalLM.from_pretrained(
-            model_id, trust_remote_code=True, revision=revision
-        ).to(device=device, dtype=dtype)
-        self._model.eval()
-
-    async def run_vision(self, frame: VisionImageFrame) -> str:
-        image = Image.frombytes("RGB", (frame.size[0], frame.size[1]), frame.image)
-        image_embeds = self._model.encode_image(image)
-        description = self._model.answer_question(
-            image_embeds=image_embeds,
-            question=frame.text,
-            tokenizer=self._tokenizer)
-        return description

src/dailyai/services/open_ai_services.py

@@ -1,19 +1,23 @@
-from typing import Literal
 import aiohttp
 from PIL import Image
 import io
+import time
+import base64
+from openai import AsyncOpenAI, AsyncStream
 
-from dailyai.services.ai_services import ImageGenService
+import json
+from collections.abc import AsyncGenerator
+
+from openai.types.chat import (
+    ChatCompletion,
+    ChatCompletionChunk,
+    ChatCompletionMessageParam,
+)
+
+from daily import VideoFrame
+from dailyai.services.ai_services import LLMService, ImageGenService, VisionService
 from dailyai.services.openai_api_llm_service import BaseOpenAILLMService
-
-
-try:
-    from openai import AsyncOpenAI
-except ModuleNotFoundError as e:
-    print(f"Exception: {e}")
-    print(
-        "In order to use OpenAI, you need to `pip install dailyai[openai]`. Also, set `OPENAI_API_KEY` environment variable.")
-    raise Exception(f"Missing module: {e}")
+from dailyai.pipeline.frames import TextFrame
 
 
 class OpenAILLMService(BaseOpenAILLMService):
@@ -27,25 +31,24 @@ class OpenAIImageGenService(ImageGenService):
     def __init__(
         self,
         *,
-        image_size: Literal["256x256", "512x512", "1024x1024", "1792x1024", "1024x1792"],
+        image_size: str,
         aiohttp_session: aiohttp.ClientSession,
         api_key,
         model="dall-e-3",
     ):
-        super().__init__()
+        super().__init__(image_size=image_size)
         self._model = model
-        self._image_size = image_size
         self._client = AsyncOpenAI(api_key=api_key)
         self._aiohttp_session = aiohttp_session
 
-    async def run_image_gen(self, prompt: str) -> tuple[str, bytes, tuple[int, int]]:
-        self.logger.info("Generating OpenAI image", prompt)
+    async def run_image_gen(self, sentence) -> tuple[str, bytes]:
+        self.logger.info("Generating OpenAI image", sentence)
 
         image = await self._client.images.generate(
-            prompt=prompt,
+            prompt=sentence,
             model=self._model,
             n=1,
-            size=self._image_size
+            size=self.image_size
         )
         image_url = image.data[0].url
         if not image_url:
@@ -55,4 +58,68 @@ class OpenAIImageGenService(ImageGenService):
         async with self._aiohttp_session.get(image_url) as response:
             image_stream = io.BytesIO(await response.content.read())
             image = Image.open(image_stream)
-            return (image_url, image.tobytes(), image.size)
+            return (image_url, image.tobytes())
+
+
+class OpenAIVisionService(VisionService):
+    def __init__(
+        self,
+        *,
+        model="gpt-4-vision-preview",
+        api_key,
+    ):
+        self._model = model
+        self._client = AsyncOpenAI(api_key=api_key)
+
+    async def run_vision(self, prompt: str, image: bytes):
+        if isinstance(image, VideoFrame):
+            # Then it's from a daily video frame
+            print("### processing daily video frame for recognition")
+            IMAGE_WIDTH = image.width
+            IMAGE_HEIGHT = image.height
+            COLOR_FORMAT = image.color_format
+            a_image = Image.frombytes(
+                'RGBA', (IMAGE_WIDTH, IMAGE_HEIGHT), image.buffer)
+            new_image = a_image.convert('RGB')
+        else:
+            # handle it as a byte stream from image gen
+            new_image = Image.frombytes('RGB', (1024, 1024), image)
+            # Uncomment these lines to write the frame to a jpg in the same directory.
+            # current_path = os.getcwd()
+            # image_path = os.path.join(current_path, "image.jpg")
+            # image.save(image_path, format="JPEG")
+
+        jpeg_buffer = io.BytesIO()
+
+        new_image.save(jpeg_buffer, format='JPEG')
+
+        jpeg_bytes = jpeg_buffer.getvalue()
+        base64_image = base64.b64encode(jpeg_bytes).decode('utf-8')
+
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": prompt},
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/jpeg;base64,{base64_image}"
+                        },
+                    },
+                ],
+            }
+        ]
+        chunks: AsyncStream[ChatCompletionChunk] = (
+            await self._client.chat.completions.create(
+                model=self._model,
+                stream=True,
+                messages=messages,
+            )
+        )
+        async for chunk in chunks:
+            print(f"%%% chunk: {chunk}")
+            if len(chunk.choices) == 0:
+                continue
+            if chunk.choices[0].delta.content:
+                yield TextFrame(chunk.choices[0].delta.content)

src/dailyai/services/openai_api_llm_service.py

@@ -1,32 +1,25 @@
 import json
 import time
 from typing import AsyncGenerator, List
+from openai import AsyncOpenAI, AsyncStream
 from dailyai.pipeline.frames import (
     Frame,
     LLMFunctionCallFrame,
     LLMFunctionStartFrame,
-    LLMMessagesFrame,
+    LLMMessagesQueueFrame,
     LLMResponseEndFrame,
     LLMResponseStartFrame,
+    OpenAILLMContextFrame,
     TextFrame,
 )
 from dailyai.services.ai_services import LLMService
-from dailyai.pipeline.openai_frames import OpenAILLMContextFrame
 from dailyai.services.openai_llm_context import OpenAILLMContext
 
-try:
-    from openai import AsyncOpenAI, AsyncStream
-
-    from openai.types.chat import (
-        ChatCompletion,
-        ChatCompletionChunk,
-        ChatCompletionMessageParam,
-    )
-except ModuleNotFoundError as e:
-    print(f"Exception: {e}")
-    print(
-        "In order to use OpenAI, you need to `pip install dailyai[openai]`. Also, set `OPENAI_API_KEY` environment variable.")
-    raise Exception(f"Missing module: {e}")
+from openai.types.chat import (
+    ChatCompletion,
+    ChatCompletionChunk,
+    ChatCompletionMessageParam,
+)
 
 
 class BaseOpenAILLMService(LLMService):
@@ -82,7 +75,7 @@ class BaseOpenAILLMService(LLMService):
     async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
         if isinstance(frame, OpenAILLMContextFrame):
             context: OpenAILLMContext = frame.context
-        elif isinstance(frame, LLMMessagesFrame):
+        elif isinstance(frame, LLMMessagesQueueFrame):
             context = OpenAILLMContext.from_messages(frame.messages)
         else:
             yield frame

src/dailyai/services/openai_llm_context.py

@@ -1,18 +1,11 @@
 from typing import List
+from openai._types import NOT_GIVEN, NotGiven
 
-try:
-    from openai._types import NOT_GIVEN, NotGiven
-
-    from openai.types.chat import (
-        ChatCompletionToolParam,
-        ChatCompletionToolChoiceOptionParam,
-        ChatCompletionMessageParam,
-    )
-except ModuleNotFoundError as e:
-    print(f"Exception: {e}")
-    print(
-        "In order to use OpenAI, you need to `pip install dailyai[openai]`. Also, set `OPENAI_API_KEY` environment variable.")
-    raise Exception(f"Missing module: {e}")
+from openai.types.chat import (
+    ChatCompletionToolParam,
+    ChatCompletionToolChoiceOptionParam,
+    ChatCompletionMessageParam,
+)
 
 
 class OpenAILLMContext:

src/dailyai/services/playht_ai_service.py

@@ -1,18 +1,11 @@
 import io
 import struct
+from pyht import Client
+from pyht.client import TTSOptions
+from pyht.protos.api_pb2 import Format
 
 from dailyai.services.ai_services import TTSService
 
-try:
-    from pyht import Client
-    from pyht.client import TTSOptions
-    from pyht.protos.api_pb2 import Format
-except ModuleNotFoundError as e:
-    print(f"Exception: {e}")
-    print(
-        "In order to use PlayHT, you need to `pip install dailyai[playht]`. Also, set `PLAY_HT_USER_ID` and `PLAY_HT_API_KEY` environment variables.")
-    raise Exception(f"Missing module: {e}")
-
 
 class PlayHTAIService(TTSService):
 

src/dailyai/services/to_be_updated/mock_ai_service.py

@@ -19,7 +19,7 @@ class MockAIService(AIService):
         image_stream = io.BytesIO(response.content)
         image = Image.open(image_stream)
         time.sleep(1)
-        return (image_url, image.tobytes(), image.size)
+        return (image_url, image)
 
     def run_llm(self, messages, latest_user_message=None, stream=True):
         for i in range(5):

src/dailyai/services/whisper_ai_services.py

@@ -3,18 +3,10 @@ import asyncio
 from enum import Enum
 import logging
 from typing import BinaryIO
+from faster_whisper import WhisperModel
 from dailyai.services.local_stt_service import LocalSTTService
 
 
-try:
-    from faster_whisper import WhisperModel
-except ModuleNotFoundError as e:
-    print(f"Exception: {e}")
-    print(
-        "In order to use Whisper, you need to `pip install dailyai[whisper]`.")
-    raise Exception(f"Missing module: {e}")
-
-
 class Model(Enum):
     """Class of basic Whisper model selection options"""
     TINY = "tiny"

src/dailyai/tests/integration/integration_azure_llm.py

@@ -1,6 +1,8 @@
 import asyncio
 import os
-from dailyai.pipeline.openai_frames import OpenAILLMContextFrame
+from dailyai.pipeline.frames import (
+    OpenAILLMContextFrame,
+)
 from dailyai.services.azure_ai_services import AzureLLMService
 from dailyai.services.openai_llm_context import OpenAILLMContext
 

src/dailyai/tests/integration/integration_ollama_llm.py

@@ -1,5 +1,7 @@
 import asyncio
-from dailyai.pipeline.openai_frames import OpenAILLMContextFrame
+from dailyai.pipeline.frames import (
+    OpenAILLMContextFrame,
+)
 from dailyai.services.openai_llm_context import OpenAILLMContext
 
 from openai.types.chat import (

src/dailyai/tests/integration/integration_openai_llm.py

@@ -1,6 +1,8 @@
 import asyncio
 import os
-from dailyai.pipeline.openai_frames import OpenAILLMContextFrame
+from dailyai.pipeline.frames import (
+    OpenAILLMContextFrame,
+)
 from dailyai.services.openai_llm_context import OpenAILLMContext
 
 from openai.types.chat import (

src/dailyai/tests/test_aggregators.py

@@ -54,13 +54,13 @@ class TestDailyFrameAggregators(unittest.IsolatedAsyncioTestCase):
             TextFrame("Hello, "),
             TextFrame("world."),
             AudioFrame(b"hello"),
-            ImageFrame(b"image", (0, 0)),
+            ImageFrame("image", b"image"),
             AudioFrame(b"world"),
             LLMResponseEndFrame(),
         ]
 
         expected_output_frames = [
-            ImageFrame(b"image", (0, 0)),
+            ImageFrame("image", b"image"),
             LLMResponseStartFrame(),
             TextFrame("Hello, "),
             TextFrame("world."),

src/dailyai/tests/test_ai_services.py

@@ -1,6 +1,6 @@
 import unittest
 
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Generator
 
 from dailyai.services.ai_services import AIService
 from dailyai.pipeline.frames import EndFrame, Frame, TextFrame

src/dailyai/tests/test_daily_transport_service.py

@@ -1,21 +1,27 @@
+import asyncio
+import threading
 import unittest
 
+from unittest.mock import MagicMock, patch
+
+from dailyai.pipeline.frames import AudioFrame, ImageFrame
+
 
 class TestDailyTransport(unittest.IsolatedAsyncioTestCase):
 
     async def test_event_handler(self):
-        from dailyai.transports.daily_transport import DailyTransport
+        from dailyai.services.daily_transport_service import DailyTransportService
 
-        transport = DailyTransport("mock.daily.co/mock", "token", "bot")
+        transport = DailyTransportService("mock.daily.co/mock", "token", "bot")
 
         was_called = False
 
         @transport.event_handler("on_first_other_participant_joined")
-        def test_event_handler(transport, participant):
+        def test_event_handler(transport):
             nonlocal was_called
             was_called = True
 
-        transport.on_first_other_participant_joined({"id": "user-id"})
+        transport.on_first_other_participant_joined()
 
         self.assertTrue(was_called)
 
@@ -29,7 +35,7 @@ class TestDailyTransport(unittest.IsolatedAsyncioTestCase):
         event = asyncio.Event()
 
         @transport.event_handler("on_first_other_participant_joined")
-        async def test_event_handler(transport, participant):
+        async def test_event_handler(transport):
             nonlocal event
             print("sleeping")
             await asyncio.sleep(0.1)
@@ -65,10 +71,10 @@ class TestDailyTransport(unittest.IsolatedAsyncioTestCase):
         daily_mock.create_camera_device.return_value = camera
 
         async def send_audio_frame():
-            await transport.send_queue.put(AudioFrame(bytes([0] * 3300)))
+            await transport.send_queue.put(AudioQueueFrame(bytes([0] * 3300)))
 
         async def send_video_frame():
-            await transport.send_queue.put(ImageFrame(b"test", (0, 0)))
+            await transport.send_queue.put(ImageQueueFrame(None, b"test"))
 
         await asyncio.gather(transport.run(), send_audio_frame(), send_video_frame())
 

src/dailyai/tests/test_pipeline.py

@@ -0,0 +1,59 @@
+import asyncio
+import unittest
+from dailyai.pipeline.aggregators import SentenceAggregator, StatelessTextTransformer
+from dailyai.pipeline.frames import EndFrame, TextFrame
+
+from dailyai.pipeline.pipeline import Pipeline
+
+
+class TestDailyPipeline(unittest.IsolatedAsyncioTestCase):
+
+    async def test_pipeline_simple(self):
+        aggregator = SentenceAggregator()
+
+        outgoing_queue = asyncio.Queue()
+        incoming_queue = asyncio.Queue()
+        pipeline = Pipeline([aggregator], incoming_queue, outgoing_queue)
+
+        await incoming_queue.put(TextFrame("Hello, "))
+        await incoming_queue.put(TextFrame("world."))
+        await incoming_queue.put(EndFrame())
+
+        await pipeline.run_pipeline()
+
+        self.assertEqual(await outgoing_queue.get(), TextFrame("Hello, world."))
+        self.assertIsInstance(await outgoing_queue.get(), EndFrame)
+
+    async def test_pipeline_multiple_stages(self):
+        sentence_aggregator = SentenceAggregator()
+        to_upper = StatelessTextTransformer(lambda x: x.upper())
+        add_space = StatelessTextTransformer(lambda x: x + " ")
+
+        outgoing_queue = asyncio.Queue()
+        incoming_queue = asyncio.Queue()
+        pipeline = Pipeline(
+            [add_space, sentence_aggregator, to_upper],
+            incoming_queue,
+            outgoing_queue
+        )
+
+        sentence = "Hello, world. It's me, a pipeline."
+        for c in sentence:
+            await incoming_queue.put(TextFrame(c))
+        await incoming_queue.put(EndFrame())
+
+        await pipeline.run_pipeline()
+
+        self.assertEqual(
+            await outgoing_queue.get(), TextFrame("H E L L O ,   W O R L D .")
+        )
+        self.assertEqual(
+            await outgoing_queue.get(),
+            TextFrame("   I T ' S   M E ,   A   P I P E L I N E ."),
+        )
+        # leftover little bit because of the spacing
+        self.assertEqual(
+            await outgoing_queue.get(),
+            TextFrame(" "),
+        )
+        self.assertIsInstance(await outgoing_queue.get(), EndFrame)

src/dailyai/transports/abstract_transport.py

@@ -1,42 +0,0 @@
-from abc import abstractmethod
-import asyncio
-import logging
-import time
-
-from dailyai.pipeline.frame_processor import FrameProcessor
-from dailyai.pipeline.pipeline import Pipeline
-
-
-class AbstractTransport:
-    def __init__(self, **kwargs):
-        self.send_queue = asyncio.Queue()
-        self.receive_queue = asyncio.Queue()
-        self.completed_queue = asyncio.Queue()
-
-        duration_minutes = kwargs.get("duration_minutes") or 10
-        self._expiration = time.time() + duration_minutes * 60
-
-        self._mic_enabled = kwargs.get("mic_enabled") or False
-        self._mic_sample_rate = kwargs.get("mic_sample_rate") or 16000
-        self._camera_enabled = kwargs.get("camera_enabled") or False
-        self._camera_width = kwargs.get("camera_width") or 1024
-        self._camera_height = kwargs.get("camera_height") or 768
-        self._camera_bitrate = kwargs.get("camera_bitrate") or 250000
-        self._camera_framerate = kwargs.get("camera_framerate") or 10
-        self._speaker_enabled = kwargs.get("speaker_enabled") or False
-        self._speaker_sample_rate = kwargs.get("speaker_sample_rate") or 16000
-
-        self._logger: logging.Logger = logging.getLogger("dailyai.transport")
-
-    @abstractmethod
-    async def run(self, pipeline: Pipeline, override_pipeline_source_queue=True):
-        pass
-
-    @abstractmethod
-    async def run_interruptible_pipeline(
-        self,
-        pipeline: Pipeline,
-        pre_processor: FrameProcessor | None = None,
-        post_processor: FrameProcessor | None = None,
-    ):
-        pass

src/dailyai/transports/websocket_transport.py

@@ -1,125 +0,0 @@
-import asyncio
-import time
-from typing import AsyncGenerator, List
-
-from dailyai.pipeline.frame_processor import FrameProcessor
-from dailyai.pipeline.frames import AudioFrame, ControlFrame, EndFrame, Frame, TTSEndFrame, TTSStartFrame, TextFrame
-from dailyai.pipeline.pipeline import Pipeline
-from dailyai.serializers.protobuf_serializer import ProtobufFrameSerializer
-from dailyai.transports.abstract_transport import AbstractTransport
-from dailyai.transports.threaded_transport import ThreadedTransport
-
-try:
-    import websockets
-except ModuleNotFoundError as e:
-    print(f"Exception: {e}")
-    print(
-        "In order to use the websocket transport, you need to `pip install dailyai[websocket]`.")
-    raise Exception(f"Missing module: {e}")
-
-
-class WebSocketFrameProcessor(FrameProcessor):
-    """This FrameProcessor filters and mutates frames before they're sent over the websocket.
-    This is necessary to aggregate audio frames into sizes that are cleanly playable by the client"""
-
-    def __init__(
-            self,
-            audio_frame_size: int | None = None,
-            sendable_frames: List[Frame] | None = None):
-        super().__init__()
-        if not audio_frame_size:
-            raise ValueError("audio_frame_size must be provided")
-
-        self._audio_frame_size = audio_frame_size
-        self._sendable_frames = sendable_frames or [TextFrame, AudioFrame]
-        self._audio_buffer = bytes()
-        self._in_tts_audio = False
-
-    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
-        if isinstance(frame, TTSStartFrame):
-            self._in_tts_audio = True
-        elif isinstance(frame, AudioFrame):
-            if self._in_tts_audio:
-                self._audio_buffer += frame.data
-                while len(self._audio_buffer) >= self._audio_frame_size:
-                    yield AudioFrame(self._audio_buffer[:self._audio_frame_size])
-                    self._audio_buffer = self._audio_buffer[self._audio_frame_size:]
-        elif isinstance(frame, TTSEndFrame):
-            self._in_tts_audio = False
-            if self._audio_buffer:
-                yield AudioFrame(self._audio_buffer)
-                self._audio_buffer = bytes()
-        elif type(frame) in self._sendable_frames or isinstance(frame, ControlFrame):
-            yield frame
-
-
-class WebsocketTransport(AbstractTransport):
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self._sample_width = kwargs.get("sample_width", 2)
-        self._n_channels = kwargs.get("n_channels", 1)
-        self._port = kwargs.get("port", 8765)
-        self._host = kwargs.get("host", "localhost")
-        self._audio_frame_size = kwargs.get("audio_frame_size", 16000)
-        self._sendable_frames = kwargs.get(
-            "sendable_frames", [
-                TextFrame, AudioFrame, TTSEndFrame, TTSStartFrame])
-        self._serializer = kwargs.get("serializer", ProtobufFrameSerializer())
-
-        self._server: websockets.WebSocketServer | None = None
-        self._websocket: websockets.WebSocketServerProtocol | None = None
-
-        self._connection_handlers = []
-
-    async def run(self, pipeline: Pipeline, override_pipeline_source_queue=True):
-        self._stop_server_event = asyncio.Event()
-        pipeline.set_sink(self.send_queue)
-        if override_pipeline_source_queue:
-            pipeline.set_source(self.receive_queue)
-
-        pipeline.add_processor(WebSocketFrameProcessor(
-            audio_frame_size=self._audio_frame_size,
-            sendable_frames=self._sendable_frames))
-
-        async def timeout():
-            sleep_time = self._expiration - time.time()
-            await asyncio.sleep(sleep_time)
-            self._stop_server_event.set()
-
-        async def send_task():
-            while not self._stop_server_event.is_set():
-                frame = await self.send_queue.get()
-                if isinstance(frame, EndFrame):
-                    self._stop_server_event.set()
-                    break
-                if self._websocket and frame:
-                    proto = self._serializer.serialize(frame)
-                    await self._websocket.send(proto)
-
-        async def start_server():
-            async with websockets.serve(self._websocket_handler, self._host, self._port) as server:
-                self._logger.debug("Websocket server started.")
-                await self._stop_server_event.wait()
-                self._logger.debug("Websocket server stopped.")
-            await self.receive_queue.put(EndFrame())
-
-        timeout_task = asyncio.create_task(timeout())
-        await asyncio.gather(start_server(), send_task(), pipeline.run_pipeline())
-        timeout_task.cancel()
-
-    def on_connection(self, handler):
-        self._connection_handlers.append(handler)
-
-    async def _websocket_handler(self, websocket: websockets.WebSocketServerProtocol, path):
-        if self._websocket:
-            await self._websocket.close()
-            self._logger.warning(
-                "Got another websocket connection; closing first.")
-
-        for handler in self._connection_handlers:
-            await handler()
-
-        self._websocket = websocket
-        async for message in websocket:
-            frame = self._serializer.deserialize(message)
-            await self.receive_queue.put(frame)

src/examples/foundational/01-say-one-thing.py

@@ -5,13 +5,10 @@ import os
 from dailyai.pipeline.frames import EndFrame, TextFrame
 from dailyai.pipeline.pipeline import Pipeline
 
-from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.daily_transport_service import DailyTransportService
 from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
 
-from runner import configure
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
+from examples.support.runner import configure
 
 logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
 logger = logging.getLogger("dailyai")
@@ -20,7 +17,7 @@ logger.setLevel(logging.DEBUG)
 
 async def main(room_url):
     async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
+        transport = DailyTransportService(
             room_url,
             None,
             "Say One Thing",

src/examples/foundational/01a-local-transport.py

@@ -4,10 +4,7 @@ import logging
 import os
 
 from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
-from dailyai.transports.local_transport import LocalTransport
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
+from dailyai.services.local_transport_service import LocalTransportService
 
 logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
 logger = logging.getLogger("dailyai")
@@ -17,7 +14,7 @@ logger.setLevel(logging.DEBUG)
 async def main():
     async with aiohttp.ClientSession() as session:
         meeting_duration_minutes = 1
-        transport = LocalTransport(
+        transport = LocalTransportService(
             duration_minutes=meeting_duration_minutes, mic_enabled=True
         )
         tts = ElevenLabsTTSService(
@@ -28,7 +25,10 @@ async def main():
 
         async def say_something():
             await asyncio.sleep(1)
-            await transport.say("Hello there.", tts)
+            await tts.say(
+                "Hello there.",
+                transport.send_queue,
+            )
             await transport.stop_when_done()
 
         await asyncio.gather(transport.run(), say_something())

src/examples/foundational/02-llm-say-one-thing.py

@@ -4,16 +4,13 @@ import logging
 
 import aiohttp
 
-from dailyai.pipeline.frames import EndFrame, LLMMessagesFrame
+from dailyai.pipeline.frames import EndFrame, LLMMessagesQueueFrame
 from dailyai.pipeline.pipeline import Pipeline
-from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.daily_transport_service import DailyTransportService
 from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
 from dailyai.services.open_ai_services import OpenAILLMService
 
-from runner import configure
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
+from examples.support.runner import configure
 
 logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
 logger = logging.getLogger("dailyai")
@@ -22,7 +19,7 @@ logger.setLevel(logging.DEBUG)
 
 async def main(room_url):
     async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
+        transport = DailyTransportService(
             room_url,
             None,
             "Say One Thing From an LLM",
@@ -36,7 +33,7 @@ async def main(room_url):
         )
 
         llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
             model="gpt-4-turbo-preview")
 
         messages = [
@@ -48,8 +45,8 @@ async def main(room_url):
         pipeline = Pipeline([llm, tts])
 
         @transport.event_handler("on_first_other_participant_joined")
-        async def on_first_other_participant_joined(transport, participant):
-            await pipeline.queue_frames([LLMMessagesFrame(messages), EndFrame()])
+        async def on_first_other_participant_joined(transport):
+            await pipeline.queue_frames([LLMMessagesQueueFrame(messages), EndFrame()])
 
         await transport.run(pipeline)
 

src/examples/foundational/03-still-frame.py

@@ -3,15 +3,12 @@ import aiohttp
 import logging
 import os
 
-from dailyai.pipeline.frames import TextFrame
+from dailyai.pipeline.frames import EndFrame, TextFrame
 from dailyai.pipeline.pipeline import Pipeline
-from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.daily_transport_service import DailyTransportService
 from dailyai.services.fal_ai_services import FalImageGenService
 
-from runner import configure
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
+from examples.support.runner import configure
 
 logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
 logger = logging.getLogger("dailyai")
@@ -20,7 +17,7 @@ logger.setLevel(logging.DEBUG)
 
 async def main(room_url):
     async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
+        transport = DailyTransportService(
             room_url,
             None,
             "Show a still frame image",
@@ -31,9 +28,7 @@ async def main(room_url):
         )
 
         imagegen = FalImageGenService(
-            params=FalImageGenService.InputParams(
-                image_size="square_hd"
-            ),
+            image_size="square_hd",
             aiohttp_session=session,
             key_id=os.getenv("FAL_KEY_ID"),
             key_secret=os.getenv("FAL_KEY_SECRET"),
@@ -42,7 +37,7 @@ async def main(room_url):
         pipeline = Pipeline([imagegen])
 
         @transport.event_handler("on_first_other_participant_joined")
-        async def on_first_other_participant_joined(transport, participant):
+        async def on_first_other_participant_joined(transport):
             # Note that we do not put an EndFrame() item in the pipeline for this demo.
             # This means that the bot will stay in the channel until it times out.
             # An EndFrame() in the pipeline would cause the transport to shut

src/examples/foundational/03a-image-local.py

@@ -6,28 +6,25 @@ import os
 import tkinter as tk
 
 from dailyai.pipeline.frames import TextFrame
-from dailyai.pipeline.pipeline import Pipeline
 from dailyai.services.fal_ai_services import FalImageGenService
-from dailyai.transports.local_transport import LocalTransport
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
+from dailyai.services.local_transport_service import LocalTransportService
 
 logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
 logger = logging.getLogger("dailyai")
 logger.setLevel(logging.DEBUG)
 
+local_joined = False
+participant_joined = False
+
 
 async def main():
     async with aiohttp.ClientSession() as session:
         meeting_duration_minutes = 2
-
         tk_root = tk.Tk()
-        tk_root.title("dailyai")
-
-        transport = LocalTransport(
+        tk_root.title("Calendar")
+        transport = LocalTransportService(
             tk_root=tk_root,
-            mic_enabled=False,
+            mic_enabled=True,
             camera_enabled=True,
             camera_width=1024,
             camera_height=1024,
@@ -35,16 +32,15 @@ async def main():
         )
 
         imagegen = FalImageGenService(
-            params=FalImageGenService.InputParams(
-                image_size="square_hd"
-            ),
+            image_size="1024x1024",
             aiohttp_session=session,
             key_id=os.getenv("FAL_KEY_ID"),
             key_secret=os.getenv("FAL_KEY_SECRET"),
         )
-
-        pipeline = Pipeline([imagegen])
-        await pipeline.queue_frames([TextFrame("a cat in the style of picasso")])
+        image_task = asyncio.create_task(
+            imagegen.run_to_queue(
+                transport.send_queue, [
+                    TextFrame("a cat in the style of picasso")]))
 
         async def run_tk():
             while not transport._stop_threads.is_set():
@@ -52,7 +48,7 @@ async def main():
                 tk_root.update_idletasks()
                 await asyncio.sleep(0.1)
 
-        await asyncio.gather(transport.run(pipeline, override_pipeline_source_queue=False), run_tk())
+        await asyncio.gather(transport.run(), image_task, run_tk())
 
 
 if __name__ == "__main__":

src/examples/foundational/04-utterance-and-speech.py

@@ -6,16 +6,12 @@ import aiohttp
 from dailyai.pipeline.merge_pipeline import SequentialMergePipeline
 from dailyai.pipeline.pipeline import Pipeline
 
-from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.daily_transport_service import DailyTransportService
 from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
 from dailyai.services.deepgram_ai_services import DeepgramTTSService
-from dailyai.pipeline.frames import EndPipeFrame, LLMMessagesFrame, TextFrame
+from dailyai.pipeline.frames import EndFrame, EndPipeFrame, LLMMessagesQueueFrame, TextFrame
 from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
-
-from runner import configure
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
+from examples.support.runner import configure
 
 logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
 logger = logging.getLogger("dailyai")
@@ -24,7 +20,7 @@ logger.setLevel(logging.DEBUG)
 
 async def main(room_url: str):
     async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
+        transport = DailyTransportService(
             room_url,
             None,
             "Static And Dynamic Speech",
@@ -60,12 +56,12 @@ async def main(room_url: str):
         # will run in parallel with generating and speaking the audio for static text, so there's no delay to
         # speak the LLM response.
         llm_pipeline = Pipeline([llm, elevenlabs_tts])
-        await llm_pipeline.queue_frames([LLMMessagesFrame(messages), EndPipeFrame()])
+        await llm_pipeline.queue_frames([LLMMessagesQueueFrame(messages), EndPipeFrame()])
 
         simple_tts_pipeline = Pipeline([azure_tts])
         await simple_tts_pipeline.queue_frames(
             [
-                TextFrame("My friend the LLM is going to tell a joke about llamas."),
+                TextFrame("My friend the LLM is going to tell a joke about llamas"),
                 EndPipeFrame(),
             ]
         )

src/examples/foundational/05-sync-speech-and-image.py

@@ -1,4 +1,5 @@
 import asyncio
+from re import S
 import aiohttp
 import os
 import logging
@@ -17,21 +18,18 @@ from dailyai.pipeline.frames import (
     TextFrame,
     EndFrame,
     ImageFrame,
-    LLMMessagesFrame,
+    LLMMessagesQueueFrame,
     LLMResponseStartFrame,
 )
 from dailyai.pipeline.frame_processor import FrameProcessor
 
 from dailyai.pipeline.pipeline import Pipeline
-from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.daily_transport_service import DailyTransportService
 from dailyai.services.open_ai_services import OpenAILLMService
 from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
 from dailyai.services.fal_ai_services import FalImageGenService
 
-from runner import configure
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
+from examples.support.runner import configure
 
 logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
 logger = logging.getLogger("dailyai")
@@ -63,7 +61,7 @@ class MonthPrepender(FrameProcessor):
 
 async def main(room_url):
     async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
+        transport = DailyTransportService(
             room_url,
             None,
             "Month Narration Bot",
@@ -81,13 +79,11 @@ async def main(room_url):
         )
 
         llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
             model="gpt-4-turbo-preview")
 
         imagegen = FalImageGenService(
-            params=FalImageGenService.InputParams(
-                image_size="square_hd"
-            ),
+            image_size="square_hd",
             aiohttp_session=session,
             key_id=os.getenv("FAL_KEY_ID"),
             key_secret=os.getenv("FAL_KEY_SECRET"),
@@ -135,7 +131,7 @@ async def main(room_url):
                 }
             ]
             frames.append(MonthFrame(month))
-            frames.append(LLMMessagesFrame(messages))
+            frames.append(LLMMessagesQueueFrame(messages))
 
         frames.append(EndFrame())
         await pipeline.queue_frames(frames)

src/examples/foundational/05a-local-sync-speech-and-text.py

@@ -1,31 +1,28 @@
 import aiohttp
+import argparse
 import asyncio
 import logging
 import tkinter as tk
 import os
-from dailyai.pipeline.aggregators import LLMFullResponseAggregator
 
-from dailyai.pipeline.frames import AudioFrame, URLImageFrame, LLMMessagesFrame, TextFrame
+from dailyai.pipeline.frames import AudioFrame, ImageFrame
 from dailyai.services.open_ai_services import OpenAILLMService
 from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
 from dailyai.services.fal_ai_services import FalImageGenService
-from dailyai.transports.local_transport import LocalTransport
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
+from dailyai.services.local_transport_service import LocalTransportService
 
 logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
 logger = logging.getLogger("dailyai")
 logger.setLevel(logging.DEBUG)
 
 
-async def main():
+async def main(room_url):
     async with aiohttp.ClientSession() as session:
         meeting_duration_minutes = 5
         tk_root = tk.Tk()
-        tk_root.title("dailyai")
+        tk_root.title("Calendar")
 
-        transport = LocalTransport(
+        transport = LocalTransportService(
             mic_enabled=True,
             camera_enabled=True,
             camera_width=1024,
@@ -41,13 +38,11 @@ async def main():
         )
 
         llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
             model="gpt-4-turbo-preview")
 
-        imagegen = FalImageGenService(
-            params=FalImageGenService.InputParams(
-                image_size="1024x1024"
-            ),
+        dalle = FalImageGenService(
+            image_size="1024x1024",
             aiohttp_session=session,
             key_id=os.getenv("FAL_KEY_ID"),
             key_secret=os.getenv("FAL_KEY_SECRET"),
@@ -63,32 +58,22 @@ async def main():
 
             return all_audio
 
-        async def get_month_description(aggregator, frame):
-            async for frame in aggregator.process_frame(frame):
-                if isinstance(frame, TextFrame):
-                    return frame.text
-
         async def get_month_data(month):
-            messages = [{"role": "system", "content": f"Describe a nature photograph suitable for use in a calendar, for the month of {month}. Include only the image description with no preamble. Limit the description to one sentence, please.", }]
-
-            messages_frame = LLMMessagesFrame(messages)
-
-            llm_full_response_aggregator = LLMFullResponseAggregator()
-
-            image_description = None
-            async for frame in llm.process_frame(messages_frame):
-                result = await get_month_description(llm_full_response_aggregator, frame)
-                if result:
-                    image_description = result
-                    break
+            messages = [
+                {
+                    "role": "system",
+                    "content": f"Describe a nature photograph suitable for use in a calendar, for the month of {month}. Include only the image description with no preamble. Limit the description to one sentence, please.",
+                }
+            ]
 
+            image_description = await llm.run_llm(messages)
             if not image_description:
                 return
 
             to_speak = f"{month}: {image_description}"
             audio_task = asyncio.create_task(get_all_audio(to_speak))
             image_task = asyncio.create_task(
-                imagegen.run_image_gen(image_description))
+                dalle.run_image_gen(image_description))
             (audio, image_data) = await asyncio.gather(audio_task, image_task)
 
             return {
@@ -96,18 +81,22 @@ async def main():
                 "text": image_description,
                 "image_url": image_data[0],
                 "image": image_data[1],
-                "image_size": image_data[2],
                 "audio": audio,
             }
 
-        # We only specify 5 months as we create tasks all at once and we might
-        # get rate limited otherwise.
         months: list[str] = [
             "January",
             "February",
             "March",
             "April",
             "May",
+            "June",
+            "July",
+            "August",
+            "September",
+            "October",
+            "November",
+            "December",
         ]
 
         async def show_images():
@@ -120,7 +109,7 @@ async def main():
                 if data:
                     await transport.send_queue.put(
                         [
-                            URLImageFrame(data["image_url"], data["image"], data["image_size"]),
+                            ImageFrame(data["image_url"], data["image"]),
                             AudioFrame(data["audio"]),
                         ]
                     )
@@ -144,4 +133,14 @@ async def main():
 
 
 if __name__ == "__main__":
-    asyncio.run(main())
+    parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
+    parser.add_argument(
+        "-u",
+        "--url",
+        type=str,
+        required=True,
+        help="URL of the Daily room to join")
+
+    args, unknown = parser.parse_known_args()
+
+    asyncio.run(main(args.url))

src/examples/foundational/06-listen-and-respond.py

@@ -2,10 +2,10 @@ import asyncio
 import aiohttp
 import logging
 import os
-from dailyai.pipeline.frames import LLMMessagesFrame
+from dailyai.pipeline.frames import LLMMessagesQueueFrame
 from dailyai.pipeline.pipeline import Pipeline
 
-from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.daily_transport_service import DailyTransportService
 from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
 from dailyai.services.open_ai_services import OpenAILLMService
 from dailyai.services.ai_services import FrameLogger
@@ -13,10 +13,7 @@ from dailyai.pipeline.aggregators import (
     LLMAssistantContextAggregator,
     LLMUserContextAggregator,
 )
-from runner import configure
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
+from examples.support.runner import configure
 
 logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
 logger = logging.getLogger("dailyai")
@@ -25,7 +22,7 @@ logger.setLevel(logging.DEBUG)
 
 async def main(room_url: str, token):
     async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
+        transport = DailyTransportService(
             room_url,
             token,
             "Respond bot",
@@ -44,7 +41,7 @@ async def main(room_url: str, token):
         )
 
         llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
             model="gpt-4-turbo-preview")
         fl = FrameLogger("Inner")
         fl2 = FrameLogger("Outer")
@@ -72,11 +69,11 @@ async def main(room_url: str, token):
         )
 
         @transport.event_handler("on_first_other_participant_joined")
-        async def on_first_other_participant_joined(transport, participant):
+        async def on_first_other_participant_joined(transport):
             # Kick off the conversation.
             messages.append(
                 {"role": "system", "content": "Please introduce yourself to the user."})
-            await pipeline.queue_frames([LLMMessagesFrame(messages)])
+            await pipeline.queue_frames([LLMMessagesQueueFrame(messages)])
 
         transport.transcription_settings["extra"]["endpointing"] = True
         transport.transcription_settings["extra"]["punctuate"] = True

src/examples/foundational/06a-image-sync.py

@@ -0,0 +1,122 @@
+import argparse
+import asyncio
+import os
+import logging
+from typing import AsyncGenerator
+import aiohttp
+import requests
+import time
+import urllib.parse
+from PIL import Image
+
+from dailyai.pipeline.frames import ImageFrame, Frame
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.ai_services import AIService
+from dailyai.pipeline.aggregators import (
+    LLMAssistantContextAggregator,
+    LLMUserContextAggregator,
+)
+from dailyai.services.open_ai_services import OpenAILLMService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.fal_ai_services import FalImageGenService
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+class ImageSyncAggregator(AIService):
+    def __init__(self, speaking_path: str, waiting_path: str):
+        self._speaking_image = Image.open(speaking_path)
+        self._speaking_image_bytes = self._speaking_image.tobytes()
+
+        self._waiting_image = Image.open(waiting_path)
+        self._waiting_image_bytes = self._waiting_image.tobytes()
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        yield ImageFrame(None, self._speaking_image_bytes)
+        yield frame
+        yield ImageFrame(None, self._waiting_image_bytes)
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransportService(
+            room_url,
+            token,
+            "Respond bot",
+            5,
+        )
+        transport._camera_enabled = True
+        transport._camera_width = 1024
+        transport._camera_height = 1024
+        transport._mic_enabled = True
+        transport._mic_sample_rate = 16000
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        img = FalImageGenService(
+            image_size="1024x1024",
+            aiohttp_session=session,
+            key_id=os.getenv("FAL_KEY_ID"),
+            key_secret=os.getenv("FAL_KEY_SECRET"),
+        )
+
+        async def get_images():
+            get_speaking_task = asyncio.create_task(
+                img.run_image_gen("An image of a cat speaking")
+            )
+            get_waiting_task = asyncio.create_task(
+                img.run_image_gen("An image of a cat waiting")
+            )
+
+            (speaking_data, waiting_data) = await asyncio.gather(
+                get_speaking_task, get_waiting_task
+            )
+
+            return speaking_data, waiting_data
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            await tts.say("Hi, I'm listening!", transport.send_queue)
+
+        async def handle_transcriptions():
+            messages = [
+                {
+                    "role": "system",
+                    "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
+                },
+            ]
+
+            tma_in = LLMUserContextAggregator(
+                messages, transport._my_participant_id)
+            tma_out = LLMAssistantContextAggregator(
+                messages, transport._my_participant_id
+            )
+            image_sync_aggregator = ImageSyncAggregator(
+                os.path.join(
+                    os.path.dirname(__file__), "assets", "speaking.png"), os.path.join(
+                    os.path.dirname(__file__), "assets", "waiting.png"), )
+            await tts.run_to_queue(
+                transport.send_queue,
+                image_sync_aggregator.run(
+                    tma_out.run(llm.run(tma_in.run(transport.get_receive_frames())))
+                ),
+            )
+
+        transport.transcription_settings["extra"]["punctuate"] = True
+        await asyncio.gather(transport.run(), handle_transcriptions())
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

src/examples/foundational/07-interruptible.py

@@ -3,20 +3,18 @@ import aiohttp
 import logging
 import os
 from dailyai.pipeline.aggregators import (
-    LLMAssistantResponseAggregator,
-    LLMUserResponseAggregator,
+    LLMAssistantContextAggregator,
+    LLMResponseAggregator,
+    LLMUserContextAggregator,
+    UserResponseAggregator,
 )
 
 from dailyai.pipeline.pipeline import Pipeline
 from dailyai.services.ai_services import FrameLogger
-from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.daily_transport_service import DailyTransportService
 from dailyai.services.open_ai_services import OpenAILLMService
 from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
-
-from runner import configure
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
+from examples.support.runner import configure
 
 logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
 logger = logging.getLogger("dailyai")
@@ -25,7 +23,7 @@ logger.setLevel(logging.DEBUG)
 
 async def main(room_url: str, token):
     async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
+        transport = DailyTransportService(
             room_url,
             token,
             "Respond bot",
@@ -44,13 +42,13 @@ async def main(room_url: str, token):
         )
 
         llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
             model="gpt-4-turbo-preview")
 
         pipeline = Pipeline([FrameLogger(), llm, FrameLogger(), tts])
 
         @transport.event_handler("on_first_other_participant_joined")
-        async def on_first_other_participant_joined(transport, participant):
+        async def on_first_other_participant_joined(transport):
             await transport.say("Hi, I'm listening!", tts)
 
         async def run_conversation():
@@ -63,8 +61,8 @@ async def main(room_url: str, token):
 
             await transport.run_interruptible_pipeline(
                 pipeline,
-                post_processor=LLMAssistantResponseAggregator(messages),
-                pre_processor=LLMUserResponseAggregator(messages),
+                post_processor=LLMResponseAggregator(messages),
+                pre_processor=UserResponseAggregator(messages),
             )
 
         transport.transcription_settings["extra"]["punctuate"] = False

src/examples/foundational/08-bots-arguing.py

@@ -6,16 +6,12 @@ import os
 from dailyai.pipeline.aggregators import SentenceAggregator
 from dailyai.pipeline.pipeline import Pipeline
 
-from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.daily_transport_service import DailyTransportService
 from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
 from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
 from dailyai.services.fal_ai_services import FalImageGenService
-from dailyai.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame
-
-from runner import configure
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
+from dailyai.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesQueueFrame, TextFrame
+from examples.support.runner import configure
 
 logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
 logger = logging.getLogger("dailyai")
@@ -24,7 +20,7 @@ logger.setLevel(logging.DEBUG)
 
 async def main(room_url: str):
     async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
+        transport = DailyTransportService(
             room_url,
             None,
             "Respond bot",
@@ -51,9 +47,7 @@ async def main(room_url: str):
             voice_id="jBpfuIE2acCO8z3wKNLl",
         )
         dalle = FalImageGenService(
-            params=FalImageGenService.InputParams(
-                image_size="1024x1024"
-            ),
+            image_size="1024x1024",
             aiohttp_session=session,
             key_id=os.getenv("FAL_KEY_ID"),
             key_secret=os.getenv("FAL_KEY_SECRET"),
@@ -82,7 +76,7 @@ async def main(room_url: str):
                 [llm, sentence_aggregator, tts1], source_queue, sink_queue
             )
 
-            await source_queue.put(LLMMessagesFrame(messages))
+            await source_queue.put(LLMMessagesQueueFrame(messages))
             await source_queue.put(EndFrame())
             await pipeline.run_pipeline()
 
@@ -124,7 +118,7 @@ async def main(room_url: str):
                 )
                 await transport.send_queue.put(
                     [
-                        ImageFrame(image_data1[1], image_data1[2]),
+                        ImageFrame(None, image_data1[1]),
                         AudioFrame(audio1),
                     ]
                 )
@@ -136,7 +130,7 @@ async def main(room_url: str):
                 )
                 await transport.send_queue.put(
                     [
-                        ImageFrame(image_data2[1], image_data2[2]),
+                        ImageFrame(None, image_data2[1]),
                         AudioFrame(audio2),
                     ]
                 )

src/examples/foundational/10-wake-word.py

@@ -5,9 +5,8 @@ import os
 import random
 from typing import AsyncGenerator
 from PIL import Image
-from dailyai.pipeline.pipeline import Pipeline
 
-from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.daily_transport_service import DailyTransportService
 from dailyai.services.open_ai_services import OpenAILLMService
 from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
 from dailyai.pipeline.aggregators import (
@@ -19,14 +18,10 @@ from dailyai.pipeline.frames import (
     TextFrame,
     ImageFrame,
     SpriteFrame,
-    TranscriptionFrame,
+    TranscriptionQueueFrame,
 )
 from dailyai.services.ai_services import AIService
-
-from runner import configure
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
+from examples.support.runner import configure
 
 logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
 logger = logging.getLogger("dailyai")
@@ -55,7 +50,7 @@ for file in image_files:
         sprites[file] = img.tobytes()
 
 # When the bot isn't talking, show a static image of the cat listening
-quiet_frame = ImageFrame(sprites["sc-listen-1.png"], (720, 1280))
+quiet_frame = ImageFrame("", sprites["sc-listen-1.png"])
 # When the bot is talking, build an animation from two sprites
 talking_list = [sprites["sc-default.png"], sprites["sc-talk.png"]]
 talking = [random.choice(talking_list) for x in range(30)]
@@ -77,7 +72,7 @@ class TranscriptFilter(AIService):
         self.bot_participant_id = bot_participant_id
 
     async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
-        if isinstance(frame, TranscriptionFrame):
+        if isinstance(frame, TranscriptionQueueFrame):
             if frame.participantId != self.bot_participant_id:
                 yield frame
 
@@ -117,7 +112,7 @@ class ImageSyncAggregator(AIService):
 
 async def main(room_url: str, token):
     async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
+        transport = DailyTransportService(
             room_url,
             token,
             "Santa Cat",
@@ -134,10 +129,9 @@ async def main(room_url: str, token):
         transport._camera_enabled = True
         transport._camera_width = 720
         transport._camera_height = 1280
-        transport.transcription_settings["extra"]["punctuate"] = True
 
         llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
             model="gpt-4-turbo-preview")
 
         tts = ElevenLabsTTSService(
@@ -147,34 +141,44 @@ async def main(room_url: str, token):
         )
         isa = ImageSyncAggregator()
 
-        messages = [
-            {
-                "role": "system",
-                "content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while.  Your responses should only be a few sentences long.",
-            },
-        ]
-
-        tma_in = LLMUserContextAggregator(
-            messages, transport._my_participant_id)
-        tma_out = LLMAssistantContextAggregator(
-            messages, transport._my_participant_id
-        )
-        tf = TranscriptFilter(transport._my_participant_id)
-        ncf = NameCheckFilter(["Santa Cat", "Santa"])
-
-        pipeline = Pipeline([isa, tf, ncf, tma_in, llm, tma_out, tts])
-
         @transport.event_handler("on_first_other_participant_joined")
-        async def on_first_other_participant_joined(transport, participant):
-            await transport.say(
+        async def on_first_other_participant_joined(transport):
+            await tts.say(
                 "Hi! If you want to talk to me, just say 'hey Santa Cat'.",
-                tts,
+                transport.send_queue,
+            )
+
+        async def handle_transcriptions():
+            messages = [
+                {
+                    "role": "system",
+                    "content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while.  Your responses should only be a few sentences long.",
+                },
+            ]
+
+            tma_in = LLMUserContextAggregator(
+                messages, transport._my_participant_id)
+            tma_out = LLMAssistantContextAggregator(
+                messages, transport._my_participant_id
+            )
+            tf = TranscriptFilter(transport._my_participant_id)
+            ncf = NameCheckFilter(["Santa Cat", "Santa"])
+            await tts.run_to_queue(
+                transport.send_queue,
+                isa.run(
+                    tma_out.run(
+                        llm.run(
+                            tma_in.run(ncf.run(tf.run(transport.get_receive_frames())))
+                        )
+                    )
+                ),
             )
 
         async def starting_image():
             await transport.send_queue.put(quiet_frame)
 
-        await asyncio.gather(transport.run(pipeline), starting_image())
+        transport.transcription_settings["extra"]["punctuate"] = True
+        await asyncio.gather(transport.run(), handle_transcriptions(), starting_image())
 
 
 if __name__ == "__main__":

src/examples/foundational/11-sound-effects.py

@@ -3,12 +3,12 @@ import asyncio
 import logging
 import os
 import wave
-from dailyai.pipeline.pipeline import Pipeline
 
-from dailyai.transports.daily_transport import DailyTransport
+from dailyai.services.daily_transport_service import DailyTransportService
 from dailyai.services.open_ai_services import OpenAILLMService
 from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
 from dailyai.pipeline.aggregators import (
+    LLMContextAggregator,
     LLMUserContextAggregator,
     LLMAssistantContextAggregator,
 )
@@ -17,14 +17,11 @@ from dailyai.pipeline.frames import (
     Frame,
     AudioFrame,
     LLMResponseEndFrame,
-    LLMMessagesFrame,
+    LLMMessagesQueueFrame,
 )
 from typing import AsyncGenerator
 
-from runner import configure
-
-from dotenv import load_dotenv
-load_dotenv(override=True)
+from examples.support.runner import configure
 
 logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
 logger = logging.getLogger("dailyai")
@@ -63,7 +60,7 @@ class InboundSoundEffectWrapper(AIService):
         pass
 
     async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
-        if isinstance(frame, LLMMessagesFrame):
+        if isinstance(frame, LLMMessagesQueueFrame):
             yield AudioFrame(sounds["ding2.wav"])
             # In case anything else up the stack needs it
             yield frame
@@ -73,7 +70,7 @@ class InboundSoundEffectWrapper(AIService):
 
 async def main(room_url: str, token):
     async with aiohttp.ClientSession() as session:
-        transport = DailyTransport(
+        transport = DailyTransportService(
             room_url,
             token,
             "Respond bot",
@@ -82,10 +79,9 @@ async def main(room_url: str, token):
             mic_sample_rate=16000,
             camera_enabled=False,
         )
-        transport.transcription_settings["extra"]["punctuate"] = True
 
         llm = OpenAILLMService(
-            api_key=os.getenv("OPENAI_API_KEY"),
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
             model="gpt-4-turbo-preview")
 
         tts = ElevenLabsTTSService(
@@ -94,31 +90,47 @@ async def main(room_url: str, token):
             voice_id="ErXwobaYiN019PkySvjV",
         )
 
-        messages = [
-            {
-                "role": "system",
-                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
-            },
-        ]
-
-        tma_in = LLMUserContextAggregator(
-            messages, transport._my_participant_id)
-        tma_out = LLMAssistantContextAggregator(
-            messages, transport._my_participant_id
-        )
-        out_sound = OutboundSoundEffectWrapper()
-        in_sound = InboundSoundEffectWrapper()
-        fl = FrameLogger("LLM Out")
-        fl2 = FrameLogger("Transcription In")
-
-        pipeline = Pipeline([tma_in, in_sound, fl2, llm, tma_out, fl, tts, out_sound])
-
         @transport.event_handler("on_first_other_participant_joined")
-        async def on_first_other_participant_joined(transport, participant):
-            await transport.say("Hi, I'm listening!", tts)
+        async def on_first_other_participant_joined(transport):
+            await tts.say("Hi, I'm listening!", transport.send_queue)
             await transport.send_queue.put(AudioFrame(sounds["ding1.wav"]))
 
-        await asyncio.gather(transport.run(pipeline))
+        async def handle_transcriptions():
+            messages = [
+                {
+                    "role": "system",
+                    "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
+                },
+            ]
+
+            tma_in = LLMUserContextAggregator(
+                messages, transport._my_participant_id)
+            tma_out = LLMAssistantContextAggregator(
+                messages, transport._my_participant_id
+            )
+            out_sound = OutboundSoundEffectWrapper()
+            in_sound = InboundSoundEffectWrapper()
+            fl = FrameLogger("LLM Out")
+            fl2 = FrameLogger("Transcription In")
+            await out_sound.run_to_queue(
+                transport.send_queue,
+                tts.run(
+                    fl.run(
+                        tma_out.run(
+                            llm.run(
+                                fl2.run(
+                                    in_sound.run(
+                                        tma_in.run(transport.get_receive_frames())
+                                    )
+                                )
+                            )
+                        )
+                    )
+                ),
+            )
+
+        transport.transcription_settings["extra"]["punctuate"] = True
+        await asyncio.gather(transport.run(), handle_transcriptions())
 
 
 if __name__ == "__main__":

src/examples/foundational/12-describe-video.py

@@ -0,0 +1,97 @@
+import asyncio
+import aiohttp
+import logging
+import os
+from typing import AsyncGenerator
+
+from dailyai.pipeline.frames import Frame, LLMMessagesQueueFrame, RequestVideoImageFrame, LLMResponseEndFrame
+from dailyai.pipeline.pipeline import Pipeline
+from dailyai.pipeline.frame_processor import FrameProcessor
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.open_ai_services import OpenAILLMService, OpenAIVisionService
+from dailyai.services.deepgram_ai_services import DeepgramTTSService
+from dailyai.services.ai_services import FrameLogger
+from dailyai.pipeline.aggregators import (
+    LLMAssistantContextAggregator,
+    LLMUserContextAggregator,
+)
+from dailyai.pipeline.frames import VideoImageFrame, VisionFrame
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+class VideoImageFrameProcessor(FrameProcessor):
+    def __init__(self):
+        pass
+
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, VideoImageFrame):
+            yield VisionFrame("Describe the image in one sentence.", frame.image)
+        else:
+            yield frame
+
+
+class ImageRefresher(FrameProcessor):
+    async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
+        if isinstance(frame, LLMResponseEndFrame):
+            yield RequestVideoImageFrame(participantId=None)
+            yield frame
+        else:
+            yield frame
+
+
+async def main(room_url: str, token):
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransportService(
+            room_url,
+            token,
+            "Respond bot",
+            duration_minutes=5,
+            start_transcription=True,
+            mic_enabled=True,
+            mic_sample_rate=16000,
+            camera_enabled=False,
+            vad_enabled=True,
+            receive_video=True,
+            receive_video_fps=0
+        )
+
+        tts = ElevenLabsTTSService(
+            aiohttp_session=session,
+            api_key=os.getenv("ELEVENLABS_API_KEY"),
+            voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+        )
+
+        llm = OpenAILLMService(
+            api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
+            model="gpt-4-turbo-preview")
+
+        vs = OpenAIVisionService(api_key=os.getenv("OPENAI_CHATGPT_API_KEY"))
+        vifp = VideoImageFrameProcessor()
+        ir = ImageRefresher()
+        pipeline = Pipeline(
+            processors=[
+                vifp,
+                vs,
+                llm,
+                tts,
+                ir,
+            ],
+        )
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            await pipeline.queue_frames([RequestVideoImageFrame(participantId=None)])
+
+        transport.transcription_settings["extra"]["endpointing"] = True
+        transport.transcription_settings["extra"]["punctuate"] = True
+        await transport.run(pipeline)
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url, token))

src/examples/foundational/13-whisper-transcription.py

@@ -0,0 +1,43 @@
+import asyncio
+import logging
+
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.whisper_ai_services import WhisperSTTService
+from examples.support.runner import configure
+
+logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
+logger = logging.getLogger("dailyai")
+logger.setLevel(logging.DEBUG)
+
+
+async def main(room_url: str):
+    transport = DailyTransportService(
+        room_url,
+        None,
+        "Transcription bot",
+        start_transcription=True,
+        mic_enabled=False,
+        camera_enabled=False,
+        speaker_enabled=True,
+    )
+
+    stt = WhisperSTTService()
+    transcription_output_queue = asyncio.Queue()
+
+    async def handle_transcription():
+        print("`````````TRANSCRIPTION`````````")
+        while True:
+            item = await transcription_output_queue.get()
+            print(item.text)
+
+    async def handle_speaker():
+        await stt.run_to_queue(
+            transcription_output_queue, transport.get_receive_frames()
+        )
+
+    await asyncio.gather(transport.run(), handle_speaker(), handle_transcription())
+
+
+if __name__ == "__main__":
+    (url, token) = configure()
+    asyncio.run(main(url))