Getting started

services/ai_services.py

@@ -0,0 +1,56 @@
+import logging
+
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Generator
+from PIL import Image
+
+
+class AIService:
+    def __init__(self):
+        self.logger = logging.getLogger("bot-instance")
+
+    def close(self):
+        pass
+
+
+class LLMService(AIService):
+    # Generate a set of responses to a prompt. Yields a list of responses.
+    @abstractmethod
+    def run_llm_async(
+        self, messages
+    ) -> Generator[str, None, None]:
+        pass
+
+    # Generate a responses to a prompt. Returns the response
+    @abstractmethod
+    def run_llm(
+        self, messages
+    ) -> str or None:
+        pass
+
+
+class TTSService(AIService):
+    # Some TTS services require a specific sample rate. We default to 16k
+    def get_mic_sample_rate(self):
+        return 16000
+
+    # Converts the sentence to audio. Yields a list of audio frames that can
+    # be sent to the microphone device
+    @abstractmethod
+    def run_tts(self, sentence) -> Generator[bytes, None, None]:
+        pass
+
+
+class ImageGenService(AIService):
+    # Renders the image. Returns an Image object.
+    @abstractmethod
+    def run_image_gen(self, sentence) -> Image.Image:
+        pass
+
+
+@dataclass
+class AIServiceConfig:
+    tts: TTSService
+    image: ImageGenService
+    llm: LLMService

services/azure_ai_services.py

@@ -0,0 +1,116 @@
+import json
+import io
+import openai
+import os
+import requests
+
+from typing import Generator
+
+from daily_ai.services.ai_services import LLMService, TTSService, ImageGenService
+from PIL import Image
+
+# See .env.example for Azure configuration needed
+from azure.cognitiveservices.speech import SpeechSynthesizer, SpeechConfig, ResultReason, CancellationReason
+
+class AzureTTSService(TTSService):
+    def __init__(self):
+        super().__init__()
+
+        self.speech_key = os.getenv("AZURE_SPEECH_SERVICE_KEY")
+        self.speech_region = os.getenv("AZURE_SPEECH_SERVICE_REGION")
+
+        self.speech_config = SpeechConfig(subscription=self.speech_key, region=self.speech_region)
+        self.speech_synthesizer = SpeechSynthesizer(speech_config=self.speech_config, audio_config=None)
+
+    def run_tts(self, sentence) -> Generator[bytes, None, None]:
+        self.logger.info("⌨️ running azure tts async")
+        ssml = "<speak version='1.0' xml:lang='en-US' xmlns='http://www.w3.org/2001/10/synthesis' " \
+           "xmlns:mstts='http://www.w3.org/2001/mstts'>" \
+           "<voice name='en-US-SaraNeural'>" \
+           "<mstts:silence type='Sentenceboundary' value='20ms' />" \
+           "<mstts:express-as style='lyrical' styledegree='2' role='SeniorFemale'>" \
+           "<prosody rate='1.05'>" \
+           f"{sentence}" \
+           "</prosody></mstts:express-as></voice></speak> "
+        result = self.speech_synthesizer.speak_ssml(ssml)
+        self.logger.info("⌨️ got azure tts result")
+        if result.reason == ResultReason.SynthesizingAudioCompleted:
+            self.logger.info("⌨️ returning result")
+            # azure always sends a 44-byte header. Strip it off.
+            yield result.audio_data[44:]
+        elif result.reason == ResultReason.Canceled:
+            cancellation_details = result.cancellation_details
+            self.logger.info("Speech synthesis canceled: {}".format(cancellation_details.reason))
+            if cancellation_details.reason == CancellationReason.Error:
+                self.logger.info("Error details: {}".format(cancellation_details.error_details))
+
+class AzureLLMService(LLMService):
+    def get_response(self, messages, stream):
+        return openai.ChatCompletion.create(
+            api_type="azure",
+            api_version="2023-06-01-preview",
+            api_key=os.getenv("AZURE_CHATGPT_KEY"),
+            api_base=os.getenv("AZURE_CHATGPT_ENDPOINT"),
+            deployment_id=os.getenv("AZURE_CHATGPT_DEPLOYMENT_ID"),
+            stream=stream,
+            messages=messages,
+        )
+
+
+    def run_llm_async(self, messages) -> Generator[str, None, None]:
+        local_messages = messages.copy()
+        messages_for_log = json.dumps(local_messages)
+        self.logger.info(f"==== generating chat via azure: {messages_for_log}")
+
+        response = self.get_response(local_messages, stream=True)
+
+        for chunk in response:
+            if len(chunk["choices"]) == 0:
+                continue
+
+            if "content" in chunk["choices"][0]["delta"]:
+                if (
+                    chunk["choices"][0]["delta"]["content"] != {}
+                ):  # streaming a content chunk
+                    yield chunk["choices"][0]["delta"]["content"]
+
+
+    def run_llm(self, messages) -> str or None:
+        local_messages = messages.copy()
+        messages_for_log = json.dumps(local_messages)
+        self.logger.info(f"==== generating chat via azure: {messages_for_log}")
+
+        response = self.get_response(local_messages, stream=False)
+        if (
+            response
+            and len(response["choices"]) > 0
+            and "message" in response["choices"][0]
+            and "content" in response["choices"][0]["message"]
+        ):
+            return response["choices"][0]["message"]["content"]
+        else:
+            return None
+
+
+class AzureImageGenService(ImageGenService):
+    def run_image_gen(self, sentence) -> Image.Image:
+        self.logger.info("generating azure image", sentence)
+
+        image = openai.Image.create(
+            api_type = 'azure',
+            api_version = '2023-06-01-preview',
+            api_key = os.getenv('AZURE_DALLE_KEY'),
+            api_base = os.getenv('AZURE_DALLE_ENDPOINT'),
+            deployment_id = os.getenv("AZURE_DALLE_DEPLOYMENT_ID"),
+            prompt=f'{sentence} in the style of {self.image_style}',
+            n=1,
+            size=f"1024x1024",
+        )
+
+        url = image["data"][0]["url"]
+        response = requests.get(url)
+
+        dalle_stream = io.BytesIO(response.content)
+        dalle_im = Image.open(dalle_stream)
+
+        return (url, dalle_im)

services/cloudflare_ai_service.py

@@ -0,0 +1,65 @@
+import requests
+import os
+from services.ai_service import AIService
+
+# Note that Cloudflare's AI workers are still in beta.
+# https://developers.cloudflare.com/workers-ai/
+class CloudflareAIService(AIService):
+    def __init__(self):
+        super().__init__()
+        self.cloudflare_account_id = os.getenv("CLOUDFLARE_ACCOUNT_ID")
+        self.cloudflare_api_token = os.getenv("CLOUDFLARE_API_TOKEN")
+
+        self.api_base_url = f'https://api.cloudflare.com/client/v4/accounts/{self.cloudflare_account_id}/ai/run/'
+        self.headers = {"Authorization": f'Bearer {self.cloudflare_api_token}'}
+
+    # base endpoint, used by the others
+    def run(self, model, input):
+        response = requests.post(f"{self.api_base_url}{model}", headers=self.headers, json=input)
+        return response.json()
+
+    # https://developers.cloudflare.com/workers-ai/models/llm/
+    def run_llm(self, messages, latest_user_message=None, stream = True):
+        input = {
+            "messages": [
+                { "role": "system", "content": "You are a friendly assistant" },
+                { "role": "user", "content": sentence }
+            ]
+        }
+
+        return self.run("@cf/meta/llama-2-7b-chat-int8", input)
+
+    # https://developers.cloudflare.com/workers-ai/models/translation/
+    def run_text_translation(self, sentence, source_language, target_language):
+        return self.run('@cf/meta/m2m100-1.2b', {
+            "text": sentence,
+            "source_lang": source_language,
+            "target_lang": target_language
+        })
+
+    # https://developers.cloudflare.com/workers-ai/models/sentiment-analysis/
+    def run_text_sentiment(self, sentence):
+        return self.run("@cf/huggingface/distilbert-sst-2-int8", {"text": sentence})
+
+    # https://developers.cloudflare.com/workers-ai/models/image-classification/
+    def run_image_classification(self, image_url):
+        response = requests.get(image_url)
+
+        if response.status_code != 200:
+            return {"error": "There was a problem downloading the image."}
+
+        if response.status_code == 200:
+            data = response.content
+            inputs = {"image": list(data)}
+
+        return self.run("@cf/microsoft/resnet-50", inputs)
+
+    # https://developers.cloudflare.com/workers-ai/models/embedding/
+    def run_embeddings(self, texts, size="medium"):
+        models = {
+            "small": "@cf/baai/bge-small-en-v1.5", # 384 output dimensions
+            "medium": "@cf/baai/bge-base-en-v1.5", # 768 output dimensions
+            "large": "@cf/baai/bge-large-en-v1.5" #1024 output dimensions
+        }
+
+        return self.run(models[size], {"text": texts})

services/deepgram_ai_service.py

@@ -0,0 +1,28 @@
+import os
+import requests
+
+from services.ai_service import AIService
+from PIL import Image
+
+
+class DeepgramAIService(AIService):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.api_key = os.getenv("DEEPGRAM_API_KEY")
+
+    def get_mic_sample_rate(self):
+        return 24000
+
+    def run_tts(self, sentence):
+        self.logger.info(f"running deepgram tts for {sentence}")
+        base_url = "https://api.beta.deepgram.com/v1/speak"
+        voice = os.getenv("DEEPGRAM_VOICE") or "alpha-apollo-en-v1"  # move this to an environment variable
+        request_url = f"{base_url}?model={voice}&encoding=linear16&container=none"
+        headers = {"authorization": f"token {self.api_key}"}
+
+        r = requests.post(request_url, headers=headers, data=sentence)
+        self.logger.info(
+            f"audio fetch status code: {r.status_code}, content length: {len(r.content)}"
+        )
+        yield r.content

services/elevenlabs_ai_service.py

@@ -0,0 +1,38 @@
+import os
+import requests
+import time
+
+from typing import Generator
+
+from daily_ai.services.ai_services import TTSService
+
+
+class ElevenLabsTTSService(TTSService):
+    def __init__(self):
+        super().__init__()
+
+        self.api_key = os.getenv("ELEVENLABS_API_KEY")
+        self.voice_id = os.getenv("ELEVENLABS_VOICE_ID")
+
+    def run_tts(self, sentence) -> Generator[bytes, None, None]:
+        url = f"https://api.elevenlabs.io/v1/text-to-speech/{self.voice_id}/stream"
+        payload = {"text": sentence, "model_id": "eleven_turbo_v2"}
+        querystring = {"output_format": "pcm_16000", "optimize_streaming_latency": 2}
+        headers = {
+            "xi-api-key": self.api_key,
+            "Content-Type": "application/json",
+        }
+
+        r = requests.request(
+            "POST", url, json=payload, headers=headers, params=querystring, stream=True
+        )
+
+        if r.status_code != 200:
+            self.logger.error(
+                f"audio fetch status code: {r.status_code}, error: {r.text}"
+            )
+            return
+
+        for chunk in r.iter_content(chunk_size=3200):
+            if chunk:
+                yield chunk

services/google_ai_service.py

@@ -0,0 +1,26 @@
+from services.ai_service import AIService
+import openai
+import os
+
+# To use Google Cloud's AI products, you'll need to install Google Cloud CLI and enable the TTS and in your project: https://cloud.google.com/sdk/docs/install
+from google.cloud import texttospeech
+
+class GoogleAIService(AIService):
+    def __init__(self):
+        super().__init__()
+
+        self.client = texttospeech.TextToSpeechClient()
+        self.voice = texttospeech.VoiceSelectionParams(
+            language_code="en-GB", name="en-GB-Neural2-F"
+        )
+
+        self.audio_config = texttospeech.AudioConfig(
+            audio_encoding = texttospeech.AudioEncoding.LINEAR16,
+            sample_rate_hertz = 16000
+        )
+
+    def run_tts(self, sentence):
+        print("running google tts")
+        synthesis_input = texttospeech.SynthesisInput(text = sentence.strip())
+        result = self.client.synthesize_speech(input=synthesis_input, voice=self.voice, audio_config=self.audio_config)
+        return result

services/huggingface_ai_service.py

@@ -0,0 +1,26 @@
+from services.ai_service import AIService
+from transformers import pipeline
+
+# These functions are just intended for testing, not production use. If you'd like to use HuggingFace, you should use your own models, or do some research into the specific models that will work best for your use case.
+class HuggingFaceAIService(AIService):
+    def __init__(self):
+        super().__init__()
+
+    def run_text_sentiment(self, sentence):
+        classifier = pipeline("sentiment-analysis")
+        return classifier(sentence)
+
+    # available models at https://huggingface.co/Helsinki-NLP (**not all models use 2-character language codes**)
+    def run_text_translation(self, sentence, source_language, target_language):
+        translator = pipeline(f"translation", model=f"Helsinki-NLP/opus-mt-{source_language}-{target_language}")
+        print(translator(sentence))
+
+        return translator(sentence)[0]["translation_text"]
+
+    def run_text_summarization(self, sentence):
+        summarizer = pipeline("summarization")
+        return summarizer(sentence)
+
+    def run_image_classification(self, image_path):
+        classifier = pipeline("image-classification")
+        return classifier(image_path)

services/mock_ai_service.py

@@ -0,0 +1,27 @@
+import io
+import requests
+import time
+from PIL import Image
+from services.ai_service import AIService
+
+class MockAIService(AIService):
+    def __init__(self):
+        super().__init__()
+
+    def run_tts(self, sentence):
+        print("running tts", sentence)
+        time.sleep(2)
+
+    def run_image_gen(self, sentence):
+        image_url = "https://d3d00swyhr67nd.cloudfront.net/w800h800/collection/ASH/ASHM/ASH_ASHM_WA1940_2_22-001.jpg"
+        response = requests.get(image_url)
+        image_stream = io.BytesIO(response.content)
+        image = Image.open(image_stream)
+        time.sleep(1)
+        return (image_url, image)
+
+    def run_llm(self, messages, latest_user_message=None, stream = True):
+        for i in range(5):
+            time.sleep(1)
+            yield({"choices": [{"delta": {"content": f"hello {i}!"}}]})
+

services/open_ai_service.py

@@ -0,0 +1,57 @@
+from services.ai_service import AIService
+import requests
+from PIL import Image
+import io
+import openai
+import os
+import time
+import json
+
+class OpenAIService(AIService):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def run_llm(self, messages, latest_user_message=None, stream = True):
+        local_messages = messages.copy()
+        if latest_user_message:
+            local_messages.append({"role": "user", "content": latest_user_message})
+        messages_for_log = json.dumps(local_messages, indent=2)
+        self.logger.info(f"==== generating chat via openai: {messages_for_log}")
+
+        model = os.getenv("OPEN_AI_MODEL")
+        if not model:
+            model = "gpt-4"
+        response = openai.ChatCompletion.create(
+            api_type = 'openai',
+            api_version = '2020-11-07',
+            api_base = "https://api.openai.com/v1",
+            api_key = os.getenv("OPEN_AI_KEY"),
+            model=model,
+            stream=stream,
+            messages=local_messages
+        )
+
+        return response
+
+    def run_image_gen(self, sentence):
+        self.logger.info("🖌️ generating openai image async for ", sentence)
+        start = time.time()
+
+        image = openai.Image.create(
+            api_type = 'openai',
+            api_version = '2020-11-07',
+            api_base = "https://api.openai.com/v1",
+            api_key = os.getenv("OPEN_AI_KEY"),
+            prompt=f'{sentence} in the style of {self.image_style}',
+            n=1,
+            size=f"1024x1024",
+        )
+        image_url = image["data"][0]["url"]
+        self.logger.info("🖌️ generated image from url", image["data"][0]["url"])
+        response = requests.get(image_url)
+        self.logger.info("🖌️ got image from url", response)
+        dalle_stream = io.BytesIO(response.content)
+        dalle_im = Image.open(dalle_stream)
+        self.logger.info("🖌️ total time", time.time() - start)
+
+        return (image_url, dalle_im)

services/playht_ai_service.py

@@ -0,0 +1,56 @@
+import io
+import os
+import struct
+from pyht import Client
+from dotenv import load_dotenv
+from pyht.client import TTSOptions
+from pyht.protos.api_pb2 import Format
+
+from services.ai_service import AIService
+
+class PlayHTAIService(AIService):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.speech_key = os.getenv("PLAY_HT_KEY") or ''
+        self.user_id = os.getenv("PLAY_HT_USER_ID") or ''
+
+        self.client = Client(
+            user_id=self.user_id,
+            api_key=self.speech_key,
+        )
+        self.options = TTSOptions(
+            voice="s3://voice-cloning-zero-shot/820da3d2-3a3b-42e7-844d-e68db835a206/sarah/manifest.json",
+            sample_rate=16000,
+            quality="higher",
+            format=Format.FORMAT_WAV
+        )
+
+    def close(self):
+        super().close()
+        self.client.close()
+
+    def run_tts(self, sentence):
+        b = bytearray()
+        in_header = True
+        for chunk in self.client.tts(sentence, self.options):
+            # skip the RIFF header.
+            if in_header:
+                b.extend(chunk)
+                if len(b) <= 36:
+                    continue
+                else:
+                    fh = io.BytesIO(b)
+                    fh.seek(36)
+                    (data, size) = struct.unpack('<4sI', fh.read(8))
+                    self.logger.info(f"first attempt: data: {data}, size: {hex(size)}, position: {fh.tell()}")
+                    while data != b'data':
+                        fh.read(size)
+                        (data, size) = struct.unpack('<4sI', fh.read(8))
+                        self.logger.info(f"subsequent data: {data}, size: {hex(size)}, position: {fh.tell()}, data != data: {data != b'data'}")
+                    self.logger.info("position: ", fh.tell())
+                    in_header = False
+            else:
+                if len(chunk):
+                    yield chunk
+