diff --git a/requirements.txt b/requirements.txt index 04c6a29a7..6f4b399f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ autopep8==2.0.4 build==1.0.3 packaging==23.2 -pyproject_hooks==1.0.0 \ No newline at end of file +pyproject_hooks==1.0.0 diff --git a/src/dailyai/queue_frame.py b/src/dailyai/queue_frame.py index da30677ec..c00f0827d 100644 --- a/src/dailyai/queue_frame.py +++ b/src/dailyai/queue_frame.py @@ -30,6 +30,10 @@ class ImageQueueFrame(QueueFrame): image: bytes +@dataclass() +class SpriteQueueFrame(QueueFrame): + images: list[bytes] | None + @dataclass() class TextQueueFrame(QueueFrame): text: str diff --git a/src/dailyai/services/daily_transport_service.py b/src/dailyai/services/daily_transport_service.py index c4d1fcbe6..e7ab5e8e9 100644 --- a/src/dailyai/services/daily_transport_service.py +++ b/src/dailyai/services/daily_transport_service.py @@ -1,6 +1,7 @@ import asyncio import inspect import logging +import sys import threading import time import types @@ -13,6 +14,7 @@ from dailyai.queue_frame import ( AudioQueueFrame, EndStreamQueueFrame, ImageQueueFrame, + SpriteQueueFrame, QueueFrame, StartStreamQueueFrame, TranscriptionQueueFrame, @@ -167,7 +169,9 @@ class DailyTransportService(EventHandler): Daily.select_speaker_device("speaker") self._image: bytes | None = None - self._camera_thread = Thread(target=self._run_camera, daemon=True) + self._images: list[bytes] | None = None + + self._camera_thread = Thread(target=self.run_camera, daemon=True) self._camera_thread.start() self._logger.info("Starting frame consumer thread") @@ -341,12 +345,22 @@ class DailyTransportService(EventHandler): def set_image(self, image: bytes): self._image: bytes | None = image - - def _run_camera(self): + self._images: list[bytes] | None = None + + def set_images(self, images: list[bytes], start_frame=0): + self._images: list[bytes] | None = images + self._image = None + self._current_frame = start_frame + + def run_camera(self): try: while not self._stop_threads.is_set(): if self._image: self.camera.write_frame(self._image) + if self._images: + this_frame = self._images[self._current_frame] + self.camera.write_frame(this_frame) + self._current_frame = (self._current_frame + 1) % len(self._images) time.sleep(1.0 / 8) # 8 fps except Exception as e: @@ -389,6 +403,8 @@ class DailyTransportService(EventHandler): b = b[l:] elif isinstance(frame, ImageQueueFrame): self.set_image(frame.image) + elif isinstance(frame, SpriteQueueFrame): + self.set_images(frame.images) elif len(b): self.mic.write_frames(bytes(b)) b = bytearray() diff --git a/src/samples/foundational/01a-greet-user.py b/src/samples/foundational/01a-greet-user.py deleted file mode 100644 index 7aae1621c..000000000 --- a/src/samples/foundational/01a-greet-user.py +++ /dev/null @@ -1,61 +0,0 @@ -import asyncio -import time -from typing import AsyncGenerator - -import aiohttp - -from dailyai.queue_frame import AudioQueueFrame -from dailyai.services.daily_transport_service import DailyTransportService -from dailyai.services.deepgram_ai_services import DeepgramTTSService - - -async def main(room_url): - async with aiohttp.ClientSession() as session: - # create a transport service object using environment variables for - # the transport service's API key, room url, and any other configuration. - # services can all define and document the environment variables they use. - # services all also take an optional config object that is used instead of - # environment variables. - # - # the abstract transport service APIs presumably can map pretty closely - # to the daily-python basic API - meeting_duration_minutes = 1 - transport = DailyTransportService( - room_url, - None, - "Greeter", - meeting_duration_minutes, - ) - transport.mic_enabled = True - - # similarly, create a tts service - tts = DeepgramTTSService(session) - - # Get the generator for the audio. This will start running in the background, - # and when we ask the generator for its items, we'll get what it's generated. - - # Register an event handler so we can play the audio when the participant joins. - print("settting up handler") - - @transport.event_handler("on_participant_joined") - async def on_participant_joined(transport, participant): - print(f"participant joined: {participant['info']['userName']}") - if participant["info"]["isLocal"]: - return - audio_generator: AsyncGenerator[bytes, None] = tts.run_tts( - f"Hello there, {participant['info']['userName']}!") - - async for audio in audio_generator: - await transport.send_queue.put(AudioQueueFrame(audio)) - - print("setting up call state handler") - - @transport.event_handler("on_call_state_updated") - async def on_call_joined(transport, state): - print(f"call state callback: {state}") - - await transport.run() - - -if __name__ == "__main__": - asyncio.run(main("https://chad-hq.daily.co/howdy")) diff --git a/src/samples/foundational/10-wake-word.py b/src/samples/foundational/10-wake-word.py new file mode 100644 index 000000000..e56d527ea --- /dev/null +++ b/src/samples/foundational/10-wake-word.py @@ -0,0 +1,196 @@ +import aiohttp +import argparse +import asyncio +import os +import random +import requests +import time +import urllib.parse + +from dotenv import load_dotenv +from PIL import Image + +load_dotenv() + +from dailyai.services.daily_transport_service import DailyTransportService +from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService +from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService +from dailyai.services.fal_ai_services import FalImageGenService +from dailyai.services.open_ai_services import OpenAIImageGenService +from dailyai.queue_aggregators import LLMContextAggregator +from dailyai.queue_frame import LLMMessagesQueueFrame, QueueFrame, TextQueueFrame, ImageQueueFrame, SpriteQueueFrame +from dailyai.services.ai_services import AIService + +from typing import AsyncGenerator, List + +sprites = {} +image_files = [ + 'sc-default.png', + 'sc-talk.png', + 'sc-listen-1.png', + 'sc-think-1.png', + 'sc-think-2.png', + 'sc-think-3.png', + 'sc-think-4.png' +] + +script_dir = os.path.dirname(__file__) + +for file in image_files: + # Build the full path to the image file + full_path = os.path.join(script_dir, "images", file) + # Get the filename without the extension to use as the dictionary key + filename = os.path.splitext(os.path.basename(full_path))[0] + # Open the image and convert it to bytes + with Image.open(full_path) as img: + sprites[file] = img.tobytes() + +# When the bot isn't talking, show a static image of the cat listening +quiet_frame = ImageQueueFrame("", sprites["sc-listen-1.png"]) +# When the bot is talking, build an animation from two sprites +talking_list = [sprites['sc-default.png'], sprites['sc-talk.png']] +talking = [random.choice(talking_list) for x in range(30)] +talking_frame = SpriteQueueFrame(images=talking) + +# TODO: Support "thinking" as soon as we get a valid transcript, while LLM is processing +thinking_list = [sprites['sc-think-1.png'], sprites['sc-think-2.png'], sprites['sc-think-3.png'], sprites['sc-think-4.png']] +thinking_frame = SpriteQueueFrame(images=thinking_list) + +class TranscriptFilter(AIService): + def __init__(self, bot_participant_id=None): + self.bot_participant_id = bot_participant_id + + async def process_frame(self, frame:QueueFrame) -> AsyncGenerator[QueueFrame, None]: + if frame.participantId != self.bot_participant_id: + yield frame + +class NameCheckFilter(AIService): + def __init__(self, names=None): + self.names = names + self.sentence = "" + + async def process_frame(self, frame:QueueFrame) -> AsyncGenerator[QueueFrame, None]: + content: str = "" + + # TODO: split up transcription by participant + if isinstance(frame, TextQueueFrame): + content = frame.text + + self.sentence += content + if self.sentence.endswith((".", "?", "!")): + if any(name in self.sentence for name in self.names): + out = self.sentence + self.sentence = "" + yield TextQueueFrame(out) + else: + out = self.sentence + self.sentence = "" + +class ImageSyncAggregator(AIService): + def __init__(self): + pass + + async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: + yield talking_frame + yield frame + yield quiet_frame + +async def main(room_url:str, token): + async with aiohttp.ClientSession() as session: + global transport + global llm + global tts + + transport = DailyTransportService( + room_url, + token, + "Santa Cat", + 180, + ) + transport.mic_enabled = True + transport.mic_sample_rate = 16000 + transport.camera_enabled = True + transport.camera_width = 720 + transport.camera_height = 1280 + + llm = AzureLLMService() + tts = ElevenLabsTTSService(session) + isa = ImageSyncAggregator() + + @transport.event_handler("on_first_other_participant_joined") + async def on_first_other_participant_joined(transport): + await tts.say("Hi! If you want to talk to me, just say 'hey Santa Cat'.", transport.send_queue) + + async def handle_transcriptions(): + messages = [ + {"role": "system", "content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while. Your responses should only be a few sentences long."}, + ] + + tma_in = LLMContextAggregator( + messages, "user", transport.my_participant_id + ) + tma_out = LLMContextAggregator( + messages, "assistant", transport.my_participant_id + ) + tf = TranscriptFilter(transport.my_participant_id) + ncf = NameCheckFilter(["Santa Cat", "Santa"]) + await tts.run_to_queue( + transport.send_queue, + isa.run( + tma_out.run( + llm.run( + tma_in.run( + ncf.run( + tf.run( + transport.get_receive_frames() + ) + ) + ) + ) + ) + ) + ) + + async def starting_image(): + await transport.send_queue.put(quiet_frame) + + transport.transcription_settings["extra"]["punctuate"] = True + await asyncio.gather(transport.run(), handle_transcriptions(), starting_image()) + + + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") + parser.add_argument( + "-u", "--url", type=str, required=True, help="URL of the Daily room to join" + ) + parser.add_argument( + "-k", + "--apikey", + type=str, + required=True, + help="Daily API Key (needed to create token)", + ) + + args, unknown = parser.parse_known_args() + + # Create a meeting token for the given room with an expiration 24 hours in the future. + room_name: str = urllib.parse.urlparse(args.url).path[1:] + expiration: float = time.time() + 60 * 60 * 24 + + res: requests.Response = requests.post( + f"https://api.daily.co/v1/meeting-tokens", + headers={"Authorization": f"Bearer {args.apikey}"}, + json={ + "properties": {"room_name": room_name, "is_owner": True, "exp": expiration} + }, + ) + + if res.status_code != 200: + raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}") + + token: str = res.json()["token"] + + asyncio.run(main(args.url, token)) diff --git a/src/samples/foundational/images/sc-default.png b/src/samples/foundational/images/sc-default.png new file mode 100644 index 000000000..f36fd9f3c Binary files /dev/null and b/src/samples/foundational/images/sc-default.png differ diff --git a/src/samples/foundational/images/sc-listen-1.png b/src/samples/foundational/images/sc-listen-1.png new file mode 100644 index 000000000..cf852b27f Binary files /dev/null and b/src/samples/foundational/images/sc-listen-1.png differ diff --git a/src/samples/foundational/images/sc-listen-2.png b/src/samples/foundational/images/sc-listen-2.png new file mode 100644 index 000000000..f07f0dbe1 Binary files /dev/null and b/src/samples/foundational/images/sc-listen-2.png differ diff --git a/src/samples/foundational/images/sc-talk.png b/src/samples/foundational/images/sc-talk.png new file mode 100644 index 000000000..393283329 Binary files /dev/null and b/src/samples/foundational/images/sc-talk.png differ diff --git a/src/samples/foundational/images/sc-think-1.png b/src/samples/foundational/images/sc-think-1.png new file mode 100644 index 000000000..317bfbb60 Binary files /dev/null and b/src/samples/foundational/images/sc-think-1.png differ diff --git a/src/samples/foundational/images/sc-think-2.png b/src/samples/foundational/images/sc-think-2.png new file mode 100644 index 000000000..72207dd4f Binary files /dev/null and b/src/samples/foundational/images/sc-think-2.png differ diff --git a/src/samples/foundational/images/sc-think-3.png b/src/samples/foundational/images/sc-think-3.png new file mode 100644 index 000000000..50f63e7c2 Binary files /dev/null and b/src/samples/foundational/images/sc-think-3.png differ diff --git a/src/samples/foundational/images/sc-think-4.png b/src/samples/foundational/images/sc-think-4.png new file mode 100644 index 000000000..5b3c32ccd Binary files /dev/null and b/src/samples/foundational/images/sc-think-4.png differ diff --git a/src/samples/server/Dockerfile b/src/samples/server/Dockerfile new file mode 100644 index 000000000..7acfa2de8 --- /dev/null +++ b/src/samples/server/Dockerfile @@ -0,0 +1,39 @@ +# setup +FROM python:3.11.5 + +WORKDIR /app +COPY requirements.txt /app +COPY *.py /app +COPY pyproject.toml /app + +COPY src/ /app/src/ + +WORKDIR /app +RUN ls --recursive /app/ +RUN pip3 install --upgrade -r requirements.txt +RUN python -m build . +RUN pip3 install . + +# If running on Ubuntu, Azure TTS requires some extra config +# https://learn.microsoft.com/en-us/azure/ai-services/speech-service/quickstarts/setup-platform?pivots=programming-language-python&tabs=linux%2Cubuntu%2Cdotnetcli%2Cdotnet%2Cjre%2Cmaven%2Cnodejs%2Cmac%2Cpypi + +RUN wget -O - https://www.openssl.org/source/openssl-1.1.1w.tar.gz | tar zxf - +WORKDIR openssl-1.1.1w +RUN ./config --prefix=/usr/local +RUN make -j $(nproc) +RUN make install_sw install_ssldirs +RUN ldconfig -v +ENV SSL_CERT_DIR=/etc/ssl/certs + +#ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH +RUN apt clean +RUN apt-get update +RUN apt-get -y install build-essential libssl-dev ca-certificates libasound2 wget + +ENV PYTHONUNBUFFERED=1 + +WORKDIR /app + +EXPOSE 8000 +# run +CMD ["gunicorn", "--workers=2", "--log-level", "debug", "--capture-output", "daily-bot-manager:app", "--bind=0.0.0.0:8000"] diff --git a/src/samples/server/README.md b/src/samples/server/README.md new file mode 100644 index 000000000..7aa498649 --- /dev/null +++ b/src/samples/server/README.md @@ -0,0 +1,13 @@ +# Server Example + +This is an example server based on [Santa Cat](https://santacat.ai). You can run the server with this command: + +``` +flask --app daily-bot-manager.py --debug run +``` + +Once the server is started, you can load `http://127.0.0.1:5000/spin-up-kitty` in a browser, and the server will do the following: + +- Create a new, randomly-named Daily room with `DAILY_API_KEY` from your .env file or environment +- Start the `10-wake-word.py` example and connect it to that room +- 301 redirect your browser to the room diff --git a/src/samples/server/auth.py b/src/samples/server/auth.py new file mode 100644 index 000000000..5fab362cd --- /dev/null +++ b/src/samples/server/auth.py @@ -0,0 +1,26 @@ +import time +import urllib + +from dotenv import load_dotenv +import requests +from flask import jsonify +import os + +load_dotenv() + +def get_meeting_token(room_name, daily_api_key, token_expiry): + api_path = os.getenv('DAILY_API_PATH') or 'https://api.daily.co/v1' + + if not token_expiry: + token_expiry = time.time() + 600 + res = requests.post(f'{api_path}/meeting-tokens', + headers={'Authorization': f'Bearer {daily_api_key}'}, + json={'properties': {'room_name': room_name, 'is_owner': True, 'exp': token_expiry}}) + if res.status_code != 200: + return jsonify({'error': 'Unable to create meeting token', 'detail': res.text}), 500 + meeting_token = res.json()['token'] + return meeting_token + + +def get_room_name(room_url): + return urllib.parse.urlparse(room_url).path[1:] \ No newline at end of file diff --git a/src/samples/server/daily-bot-manager.py b/src/samples/server/daily-bot-manager.py new file mode 100644 index 000000000..877861866 --- /dev/null +++ b/src/samples/server/daily-bot-manager.py @@ -0,0 +1,99 @@ +import os +import requests +import subprocess +import time + +from flask import Flask, jsonify, request, redirect +from flask_cors import CORS +from samples.server.auth import get_meeting_token + +from dotenv import load_dotenv + +load_dotenv() + +app = Flask(__name__) +CORS(app) + +print(f"I loaded an environment, and my FAL_KEY_ID is {os.getenv('FAL_KEY_ID')}") + +def start_bot(bot_path, args=None): + daily_api_key = os.getenv("DAILY_API_KEY") + api_path = os.getenv("DAILY_API_PATH") or "https://api.daily.co/v1" + + timeout = int(os.getenv("ROOM_TIMEOUT") or os.getenv("BOT_MAX_DURATION") or 300) + exp = time.time() + timeout + res = requests.post( + f"{api_path}/rooms", + headers={"Authorization": f"Bearer {daily_api_key}"}, + json={ + "properties": { + "exp": exp, + "enable_chat": True, + "enable_emoji_reactions": True, + "eject_at_room_exp": True, + "enable_prejoin_ui": False, + "enable_recording": "cloud" + } + }, + ) + if res.status_code != 200: + return ( + jsonify( + { + "error": "Unable to create room", + "status_code": res.status_code, + "text": res.text, + } + ), + 500, + ) + room_url = res.json()["url"] + room_name = res.json()["name"] + + meeting_token = get_meeting_token(room_name, daily_api_key, exp) + + if args: + extra_args = " ".join([f'-{x[0]} "{x[1]}"' for x in args]) + else: + extra_args = "" + + proc = subprocess.Popen( + [ + f"python {bot_path} -u {room_url} -t {meeting_token} -k {daily_api_key} {extra_args}" + ], + shell=True, + bufsize=1, + ) + + # Don't return until the bot has joined the room, but wait for at most 2 seconds. + attempts = 0 + while attempts < 20: + time.sleep(0.1) + attempts += 1 + res = requests.get( + f"{api_path}/rooms/{room_name}/get-session-data", + headers={"Authorization": f"Bearer {daily_api_key}"}, + ) + if res.status_code == 200: + break + print(f"Took {attempts} attempts to join room {room_name}") + + # Additional client config + config = {} + if os.getenv("CLIENT_VAD_TIMEOUT_SEC"): + config['vad_timeout_sec'] = float(os.getenv("CLIENT_VAD_TIMEOUT_SEC")) + else: + config['vad_timeout_sec'] = 1.5 + + #return jsonify({"room_url": room_url, "token": meeting_token, "config": config}), 200 + return redirect(room_url, code=301) + + +@app.route("/spin-up-kitty", methods=["GET", "POST"]) +def spin_up_kitty(): + return start_bot("./src/samples/foundational/10-wake-word.py") + + +@app.route("/healthz") +def health_check(): + return "ok", 200 \ No newline at end of file