Wake word and animation sprites (#15)

* WIP: golden kitty

* added web server

* added health check

* added flask to module build

* trying requirements.txt

* added dotenv

* flask_cors

* gunicorn

* requirements cleanup

* Dockerfile

* WOOF

* basic wake word

* removed otel

* basic animation kind of works

* i think animation defeated me

* added santa cat assets

* cleanup

* cleanup

* server example and cleanup

* more cleanup

* fix up some class variable names

* minor cleanup, remove mistakenly-added print and logger stuff

* cleanup

* cleanup

---------

Co-authored-by: Moishe Lettvin <moishel@gmail.com>

requirements.txt

@@ -1,4 +1,4 @@
 autopep8==2.0.4
 build==1.0.3
 packaging==23.2
-pyproject_hooks==1.0.0
+pyproject_hooks==1.0.0

src/dailyai/queue_frame.py

@@ -30,6 +30,10 @@ class ImageQueueFrame(QueueFrame):
     image: bytes
 
 
+@dataclass()
+class SpriteQueueFrame(QueueFrame):
+    images: list[bytes] | None
+
 @dataclass()
 class TextQueueFrame(QueueFrame):
     text: str

src/dailyai/services/daily_transport_service.py

@@ -1,6 +1,7 @@
 import asyncio
 import inspect
 import logging
+import sys
 import threading
 import time
 import types
@@ -13,6 +14,7 @@ from dailyai.queue_frame import (
     AudioQueueFrame,
     EndStreamQueueFrame,
     ImageQueueFrame,
+    SpriteQueueFrame,
     QueueFrame,
     StartStreamQueueFrame,
     TranscriptionQueueFrame,
@@ -167,7 +169,9 @@ class DailyTransportService(EventHandler):
             Daily.select_speaker_device("speaker")
 
         self._image: bytes | None = None
-        self._camera_thread = Thread(target=self._run_camera, daemon=True)
+        self._images: list[bytes] | None = None
+
+        self._camera_thread = Thread(target=self.run_camera, daemon=True)
         self._camera_thread.start()
 
         self._logger.info("Starting frame consumer thread")
@@ -341,12 +345,22 @@ class DailyTransportService(EventHandler):
 
     def set_image(self, image: bytes):
         self._image: bytes | None = image
-
-    def _run_camera(self):
+        self._images: list[bytes] | None = None
+    
+    def set_images(self, images: list[bytes], start_frame=0):
+        self._images: list[bytes] | None = images
+        self._image = None
+        self._current_frame = start_frame
+    
+    def run_camera(self):
         try:
             while not self._stop_threads.is_set():
                 if self._image:
                     self.camera.write_frame(self._image)
+                if self._images:
+                    this_frame = self._images[self._current_frame]
+                    self.camera.write_frame(this_frame)
+                    self._current_frame = (self._current_frame + 1) % len(self._images)
 
                 time.sleep(1.0 / 8)  # 8 fps
         except Exception as e:
@@ -389,6 +403,8 @@ class DailyTransportService(EventHandler):
                                     b = b[l:]
                             elif isinstance(frame, ImageQueueFrame):
                                 self.set_image(frame.image)
+                            elif isinstance(frame, SpriteQueueFrame):
+                                self.set_images(frame.images)
                         elif len(b):
                             self.mic.write_frames(bytes(b))
                             b = bytearray()

src/samples/foundational/01a-greet-user.py

@@ -1,61 +0,0 @@
-import asyncio
-import time
-from typing import AsyncGenerator
-
-import aiohttp
-
-from dailyai.queue_frame import AudioQueueFrame
-from dailyai.services.daily_transport_service import DailyTransportService
-from dailyai.services.deepgram_ai_services import DeepgramTTSService
-
-
-async def main(room_url):
-    async with aiohttp.ClientSession() as session:
-        # create a transport service object using environment variables for
-        # the transport service's API key, room url, and any other configuration.
-        # services can all define and document the environment variables they use.
-        # services all also take an optional config object that is used instead of
-        # environment variables.
-        #
-        # the abstract transport service APIs presumably can map pretty closely
-        # to the daily-python basic API
-        meeting_duration_minutes = 1
-        transport = DailyTransportService(
-            room_url,
-            None,
-            "Greeter",
-            meeting_duration_minutes,
-        )
-        transport.mic_enabled = True
-
-        # similarly, create a tts service
-        tts = DeepgramTTSService(session)
-
-        # Get the generator for the audio. This will start running in the background,
-        # and when we ask the generator for its items, we'll get what it's generated.
-
-        # Register an event handler so we can play the audio when the participant joins.
-        print("settting up handler")
-
-        @transport.event_handler("on_participant_joined")
-        async def on_participant_joined(transport, participant):
-            print(f"participant joined: {participant['info']['userName']}")
-            if participant["info"]["isLocal"]:
-                return
-            audio_generator: AsyncGenerator[bytes, None] = tts.run_tts(
-                f"Hello there, {participant['info']['userName']}!")
-
-            async for audio in audio_generator:
-                await transport.send_queue.put(AudioQueueFrame(audio))
-
-        print("setting up call state handler")
-
-        @transport.event_handler("on_call_state_updated")
-        async def on_call_joined(transport, state):
-            print(f"call state callback: {state}")
-
-        await transport.run()
-
-
-if __name__ == "__main__":
-    asyncio.run(main("https://chad-hq.daily.co/howdy"))

src/samples/foundational/10-wake-word.py

@@ -0,0 +1,196 @@
+import aiohttp
+import argparse
+import asyncio
+import os
+import random
+import requests
+import time
+import urllib.parse
+
+from dotenv import load_dotenv
+from PIL import Image
+
+load_dotenv()
+
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
+from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
+from dailyai.services.fal_ai_services import FalImageGenService
+from dailyai.services.open_ai_services import OpenAIImageGenService
+from dailyai.queue_aggregators import LLMContextAggregator
+from dailyai.queue_frame import LLMMessagesQueueFrame, QueueFrame, TextQueueFrame, ImageQueueFrame, SpriteQueueFrame
+from dailyai.services.ai_services import AIService
+
+from typing import AsyncGenerator, List
+
+sprites = {}
+image_files = [
+    'sc-default.png',
+    'sc-talk.png',
+    'sc-listen-1.png',
+    'sc-think-1.png',
+    'sc-think-2.png',
+    'sc-think-3.png',
+    'sc-think-4.png'
+]
+
+script_dir = os.path.dirname(__file__)
+
+for file in image_files:
+    # Build the full path to the image file
+    full_path = os.path.join(script_dir, "images", file)
+    # Get the filename without the extension to use as the dictionary key
+    filename = os.path.splitext(os.path.basename(full_path))[0]
+    # Open the image and convert it to bytes
+    with Image.open(full_path) as img:
+        sprites[file] = img.tobytes()
+
+# When the bot isn't talking, show a static image of the cat listening
+quiet_frame = ImageQueueFrame("", sprites["sc-listen-1.png"])
+# When the bot is talking, build an animation from two sprites
+talking_list = [sprites['sc-default.png'], sprites['sc-talk.png']]
+talking = [random.choice(talking_list) for x in range(30)]
+talking_frame = SpriteQueueFrame(images=talking)
+
+# TODO: Support "thinking" as soon as we get a valid transcript, while LLM is processing
+thinking_list = [sprites['sc-think-1.png'], sprites['sc-think-2.png'], sprites['sc-think-3.png'], sprites['sc-think-4.png']]
+thinking_frame = SpriteQueueFrame(images=thinking_list)
+
+class TranscriptFilter(AIService):
+    def __init__(self, bot_participant_id=None):
+        self.bot_participant_id = bot_participant_id
+
+    async def process_frame(self, frame:QueueFrame) -> AsyncGenerator[QueueFrame, None]:
+        if frame.participantId != self.bot_participant_id:
+            yield frame
+
+class NameCheckFilter(AIService):
+    def __init__(self, names=None):
+        self.names = names
+        self.sentence = ""
+
+    async def process_frame(self, frame:QueueFrame) -> AsyncGenerator[QueueFrame, None]:
+        content: str = ""
+
+        # TODO: split up transcription by participant
+        if isinstance(frame, TextQueueFrame):
+            content = frame.text
+
+        self.sentence += content
+        if self.sentence.endswith((".", "?", "!")):
+            if any(name in self.sentence for name in self.names):
+                out = self.sentence
+                self.sentence = ""
+                yield TextQueueFrame(out)
+            else:
+                out = self.sentence
+                self.sentence = ""
+
+class ImageSyncAggregator(AIService):
+    def __init__(self):
+        pass
+
+    async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
+        yield talking_frame
+        yield frame
+        yield quiet_frame
+
+async def main(room_url:str, token):
+    async with aiohttp.ClientSession() as session:
+        global transport
+        global llm
+        global tts
+
+        transport = DailyTransportService(
+            room_url,
+            token,
+            "Santa Cat",
+            180,
+        )
+        transport.mic_enabled = True
+        transport.mic_sample_rate = 16000
+        transport.camera_enabled = True
+        transport.camera_width = 720
+        transport.camera_height = 1280
+
+        llm = AzureLLMService()
+        tts = ElevenLabsTTSService(session)
+        isa = ImageSyncAggregator()
+
+        @transport.event_handler("on_first_other_participant_joined")
+        async def on_first_other_participant_joined(transport):
+            await tts.say("Hi! If you want to talk to me, just say 'hey Santa Cat'.", transport.send_queue)
+
+        async def handle_transcriptions():
+            messages = [
+                {"role": "system", "content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while.  Your responses should only be a few sentences long."},
+            ]
+
+            tma_in = LLMContextAggregator(
+                messages, "user", transport.my_participant_id
+            )
+            tma_out = LLMContextAggregator(
+                messages, "assistant", transport.my_participant_id
+            )
+            tf = TranscriptFilter(transport.my_participant_id)
+            ncf = NameCheckFilter(["Santa Cat", "Santa"])
+            await tts.run_to_queue(
+                transport.send_queue,
+                isa.run(
+                    tma_out.run(
+                        llm.run(
+                            tma_in.run(
+                                ncf.run(
+                                    tf.run(
+                                        transport.get_receive_frames()
+                                    )
+                                )
+                            )
+                        )
+                    )
+                )
+            )
+
+        async def starting_image():
+            await transport.send_queue.put(quiet_frame)
+            
+        transport.transcription_settings["extra"]["punctuate"] = True
+        await asyncio.gather(transport.run(), handle_transcriptions(), starting_image())
+
+
+
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
+    parser.add_argument(
+        "-u", "--url", type=str, required=True, help="URL of the Daily room to join"
+    )
+    parser.add_argument(
+        "-k",
+        "--apikey",
+        type=str,
+        required=True,
+        help="Daily API Key (needed to create token)",
+    )
+
+    args, unknown = parser.parse_known_args()
+
+    # Create a meeting token for the given room with an expiration 24 hours in the future.
+    room_name: str = urllib.parse.urlparse(args.url).path[1:]
+    expiration: float = time.time() + 60 * 60 * 24
+
+    res: requests.Response = requests.post(
+        f"https://api.daily.co/v1/meeting-tokens",
+        headers={"Authorization": f"Bearer {args.apikey}"},
+        json={
+            "properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
+        },
+    )
+
+    if res.status_code != 200:
+        raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
+
+    token: str = res.json()["token"]
+
+    asyncio.run(main(args.url, token))

src/samples/server/Dockerfile

@@ -0,0 +1,39 @@
+# setup
+FROM python:3.11.5
+
+WORKDIR /app
+COPY requirements.txt /app
+COPY *.py /app
+COPY pyproject.toml /app
+
+COPY src/ /app/src/
+
+WORKDIR /app
+RUN ls --recursive /app/
+RUN pip3 install --upgrade -r requirements.txt
+RUN python -m build .
+RUN pip3 install .
+
+# If running on Ubuntu, Azure TTS requires some extra config
+# https://learn.microsoft.com/en-us/azure/ai-services/speech-service/quickstarts/setup-platform?pivots=programming-language-python&tabs=linux%2Cubuntu%2Cdotnetcli%2Cdotnet%2Cjre%2Cmaven%2Cnodejs%2Cmac%2Cpypi
+
+RUN wget -O - https://www.openssl.org/source/openssl-1.1.1w.tar.gz | tar zxf -
+WORKDIR openssl-1.1.1w
+RUN ./config --prefix=/usr/local
+RUN make -j $(nproc)
+RUN make install_sw install_ssldirs
+RUN ldconfig -v
+ENV SSL_CERT_DIR=/etc/ssl/certs
+
+#ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+RUN apt clean
+RUN apt-get update
+RUN apt-get -y install build-essential libssl-dev ca-certificates libasound2 wget
+
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+EXPOSE 8000
+# run
+CMD ["gunicorn", "--workers=2", "--log-level", "debug", "--capture-output", "daily-bot-manager:app", "--bind=0.0.0.0:8000"]

src/samples/server/README.md

@@ -0,0 +1,13 @@
+# Server Example
+
+This is an example server based on [Santa Cat](https://santacat.ai). You can run the server with this command:
+
+```
+flask --app daily-bot-manager.py --debug run
+```
+
+Once the server is started, you can load `http://127.0.0.1:5000/spin-up-kitty` in a browser, and the server will do the following:
+
+- Create a new, randomly-named Daily room with `DAILY_API_KEY` from your .env file or environment
+- Start the `10-wake-word.py` example and connect it to that room
+- 301 redirect your browser to the room

src/samples/server/auth.py

@@ -0,0 +1,26 @@
+import time
+import urllib
+
+from dotenv import load_dotenv
+import requests
+from flask import jsonify
+import os
+
+load_dotenv()
+
+def get_meeting_token(room_name, daily_api_key, token_expiry):
+    api_path = os.getenv('DAILY_API_PATH') or 'https://api.daily.co/v1'
+
+    if not token_expiry:
+        token_expiry = time.time() + 600
+    res = requests.post(f'{api_path}/meeting-tokens',
+                        headers={'Authorization': f'Bearer {daily_api_key}'},
+                        json={'properties': {'room_name': room_name, 'is_owner': True, 'exp': token_expiry}})
+    if res.status_code != 200:
+        return jsonify({'error': 'Unable to create meeting token', 'detail': res.text}), 500
+    meeting_token = res.json()['token']
+    return meeting_token
+
+
+def get_room_name(room_url):
+    return urllib.parse.urlparse(room_url).path[1:]

src/samples/server/daily-bot-manager.py

@@ -0,0 +1,99 @@
+import os
+import requests
+import subprocess
+import time
+
+from flask import Flask, jsonify, request, redirect
+from flask_cors import CORS
+from samples.server.auth import get_meeting_token
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+app = Flask(__name__)
+CORS(app)
+
+print(f"I loaded an environment, and my FAL_KEY_ID is {os.getenv('FAL_KEY_ID')}")
+
+def start_bot(bot_path, args=None):
+    daily_api_key = os.getenv("DAILY_API_KEY")
+    api_path = os.getenv("DAILY_API_PATH") or "https://api.daily.co/v1"
+
+    timeout = int(os.getenv("ROOM_TIMEOUT") or os.getenv("BOT_MAX_DURATION") or 300)
+    exp = time.time() + timeout
+    res = requests.post(
+        f"{api_path}/rooms",
+        headers={"Authorization": f"Bearer {daily_api_key}"},
+        json={
+            "properties": {
+                "exp": exp,
+                "enable_chat": True,
+                "enable_emoji_reactions": True,
+                "eject_at_room_exp": True,
+                "enable_prejoin_ui": False,
+                "enable_recording": "cloud"
+            }
+        },
+    )
+    if res.status_code != 200:
+        return (
+            jsonify(
+                {
+                    "error": "Unable to create room",
+                    "status_code": res.status_code,
+                    "text": res.text,
+                }
+            ),
+            500,
+        )
+    room_url = res.json()["url"]
+    room_name = res.json()["name"]
+
+    meeting_token = get_meeting_token(room_name, daily_api_key, exp)
+
+    if args:
+        extra_args = " ".join([f'-{x[0]} "{x[1]}"' for x in args])
+    else:
+        extra_args = ""
+
+    proc = subprocess.Popen(
+        [
+            f"python {bot_path} -u {room_url} -t {meeting_token} -k {daily_api_key} {extra_args}"
+        ],
+        shell=True,
+        bufsize=1,
+    )
+
+    # Don't return until the bot has joined the room, but wait for at most 2 seconds.
+    attempts = 0
+    while attempts < 20:
+        time.sleep(0.1)
+        attempts += 1
+        res = requests.get(
+            f"{api_path}/rooms/{room_name}/get-session-data",
+            headers={"Authorization": f"Bearer {daily_api_key}"},
+        )
+        if res.status_code == 200:
+            break
+    print(f"Took {attempts} attempts to join room {room_name}")
+    
+    # Additional client config
+    config = {}
+    if os.getenv("CLIENT_VAD_TIMEOUT_SEC"):
+        config['vad_timeout_sec'] = float(os.getenv("CLIENT_VAD_TIMEOUT_SEC"))
+    else:
+        config['vad_timeout_sec'] = 1.5
+
+    #return jsonify({"room_url": room_url, "token": meeting_token, "config": config}), 200
+    return redirect(room_url, code=301)
+
+
+@app.route("/spin-up-kitty", methods=["GET", "POST"])
+def spin_up_kitty():
+    return start_bot("./src/samples/foundational/10-wake-word.py")
+
+
+@app.route("/healthz")
+def health_check():
+    return "ok", 200