wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use inference text in demo, clean up image generation

Browse Source
This commit is contained in:
Moishe Lettvin
2024-01-04 17:26:13 -05:00
parent c68703749b
commit fcd9a248d9
5 changed files with 390 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@@ -22,4 +22,6 @@ share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
MANIFEST
.DS_Store

13
src/dailyai/services/ai_services.py
View File

@@ -1,6 +1,8 @@
import logging
from abc import abstractmethod
from collections.abc import AsyncGenerator
from dataclasses import dataclass
from typing import Generator
from PIL import Image
@@ -13,18 +15,17 @@ class AIService:
def close(self):
pass
class LLMService(AIService):
# Generate a set of responses to a prompt. Yields a list of responses.
@abstractmethod
def run_llm_async(
async def run_llm_async(
self, messages
) -> Generator[str, None, None]:
) -> AsyncGenerator[str, None, None]:
pass
# Generate a responses to a prompt. Returns the response
@abstractmethod
def run_llm(
async def run_llm(
self, messages
) -> str or None:
pass
@@ -38,14 +39,14 @@ class TTSService(AIService):
# Converts the sentence to audio. Yields a list of audio frames that can
# be sent to the microphone device
@abstractmethod
def run_tts(self, sentence) -> Generator[bytes, None, None]:
async def run_tts(self, sentence) -> AsyncGenerator[bytes, None, None]:
pass
class ImageGenService(AIService):
# Renders the image. Returns an Image object.
@abstractmethod
def run_image_gen(self, sentence) -> tuple[str, Image.Image]:
async def run_image_gen(self, sentence) -> tuple[str, Image.Image]:
pass

61
src/dailyai/services/azure_ai_services.py
View File

@@ -1,11 +1,13 @@
import json
import aiohttp
import asyncio
import io
import json
from openai import AzureOpenAI
import os
import requests
from typing import Generator
from collections.abc import AsyncGenerator
from dailyai.services.ai_services import LLMService, TTSService, ImageGenService
from PIL import Image
@@ -23,7 +25,7 @@ class AzureTTSService(TTSService):
self.speech_config = SpeechConfig(subscription=speech_key, region=speech_region)
self.speech_synthesizer = SpeechSynthesizer(speech_config=self.speech_config, audio_config=None)
def run_tts(self, sentence) -> Generator[bytes, None, None]:
async def run_tts(self, sentence) -> AsyncGenerator[bytes, None, None]:
self.logger.info("Running azure tts")
ssml = "<speak version='1.0' xml:lang='en-US' xmlns='http://www.w3.org/2001/10/synthesis' " \
"xmlns:mstts='http://www.w3.org/2001/mstts'>" \
@@ -33,7 +35,7 @@ class AzureTTSService(TTSService):
"<prosody rate='1.05'>" \
f"{sentence}" \
"</prosody></mstts:express-as></voice></speak> "
result = self.speech_synthesizer.speak_ssml(ssml)
result = await asyncio.to_thread(self.speech_synthesizer.speak_ssml, (ssml))
self.logger.info("Got azure tts result")
if result.reason == ResultReason.SynthesizingAudioCompleted:
self.logger.info("Returning result")
@@ -65,7 +67,7 @@ class AzureLLMService(LLMService):
model=self.model,
)
def run_llm_async(self, messages) -> Generator[str, None, None]:
async def run_llm_async(self, messages) -> AsyncGenerator[str, None, None]:
messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via azure: {messages_for_log}")
@@ -78,7 +80,7 @@ class AzureLLMService(LLMService):
if chunk.choices[0].delta.content:
yield chunk.choices[0].delta.content
def run_llm(self, messages) -> str | None:
async def run_llm(self, messages) -> str | None:
messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via azure: {messages_for_log}")
@@ -88,6 +90,49 @@ class AzureLLMService(LLMService):
else:
return None
class AzureImageGenServiceREST(ImageGenService):
def __init__(self, api_key=None, azure_endpoint=None, api_version=None, model=None):
super().__init__()
self.api_key = api_key or os.getenv("AZURE_DALLE_KEY")
self.azure_endpoint = azure_endpoint or os.getenv("AZURE_DALLE_ENDPOINT")
self.api_version = api_version or "2023-06-01-preview"
self.model = model or os.getenv("AZURE_DALLE_DEPLOYMENT_ID")
async def run_image_gen(self, sentence, size) -> tuple[str, Image.Image]:
# TODO hoist the session to app-level
async with aiohttp.ClientSession() as session:
url = f"{self.azure_endpoint}openai/images/generations:submit?api-version={self.api_version}"
headers= { "api-key": self.api_key, "Content-Type": "application/json" }
body = {
# Enter your prompt text here
"prompt": sentence,
"size": size,
"n": 1,
}
async with session.post(url, headers=headers, json=body) as submission:
operation_location = submission.headers['operation-location']
status = ""
attempts_left = 120
while status != "succeeded":
attempts_left -= 1
if attempts_left == 0:
raise Exception("Image generation timed out")
await asyncio.sleep(1)
response = await session.get(operation_location, headers=headers)
json_response = await response.json()
status = json_response["status"]
image_url = json_response["result"]["data"][0]["url"]
# Load the image from the url
async with session.get(image_url) as response:
image_stream = io.BytesIO(await response.content.read())
image = Image.open(image_stream)
return (image_url, image.tobytes())
class AzureImageGenService(ImageGenService):
@@ -96,7 +141,7 @@ class AzureImageGenService(ImageGenService):
api_key = api_key or os.getenv("AZURE_DALLE_KEY")
azure_endpoint = azure_endpoint or os.getenv("AZURE_DALLE_ENDPOINT")
api_version = api_version or "2023-12-01-preview"
api_version = api_version or "2023-06-01-preview"
self.model = model or os.getenv("AZURE_DALLE_DEPLOYMENT_ID")
self.client = AzureOpenAI(
@@ -105,7 +150,7 @@ class AzureImageGenService(ImageGenService):
api_version=api_version,
)
def run_image_gen(self, sentence) -> tuple[str, Image.Image]:
async def run_image_gen(self, sentence) -> tuple[str, Image.Image]:
self.logger.info("Generating azure image", sentence)
image = self.client.images.generate(

267
src/dailyai/services/daily_transport_service.py Normal file
View File

@@ -0,0 +1,267 @@
import inspect
import logging
import time
import types
from functools import partial
from queue import Queue, Empty
from dailyai.output_queue import OutputQueueFrame, FrameType
from threading import Thread, Event, Timer
from daily import (
EventHandler,
CallClient,
Daily,
VirtualCameraDevice,
VirtualMicrophoneDevice,
VirtualSpeakerDevice,
)
class DailyTransportService(EventHandler):
def __init__(
self,
room_url: str,
token: str,
bot_name: str,
duration: float = 10,
):
super().__init__()
self.bot_name: str = bot_name
self.room_url: str = room_url
self.token: str = token
self.duration: float = duration
self.expiration = time.time() + duration * 60
self.output_queue = Queue()
self.is_interrupted = Event()
self.stop_threads = Event()
self.story_started = False
self.logger: logging.Logger = logging.getLogger("dailyai")
self.event_handlers = {}
def monkeypatch(self, event_name, *args):
for handler in self.event_handlers[event_name]:
handler(*args)
def add_event_handler(self, event_name: str, handler):
if not event_name.startswith("on_"):
raise Exception(f"Event handler {event_name} must start with 'on_'")
methods = inspect.getmembers(self, predicate=inspect.ismethod)
if event_name not in [method[0] for method in methods]:
raise Exception(f"Event handler {event_name} not found")
if not event_name in self.event_handlers:
self.event_handlers[event_name] = [getattr(self, event_name), types.MethodType(handler, self)]
setattr(self, event_name, partial(self.monkeypatch, event_name))
else:
self.event_handlers[event_name].append(types.MethodType(handler, self))
def configure_daily(self):
Daily.init()
self.client = CallClient(event_handler=self)
if self.mic_enabled:
self.mic: VirtualMicrophoneDevice = Daily.create_microphone_device(
"mic", sample_rate=self.mic_sample_rate, channels=1
)
if self.camera_enabled:
self.camera: VirtualCameraDevice = Daily.create_camera_device(
"camera", width=self.camera_width, height=self.camera_height, color_format="RGB"
)
self.speaker: VirtualSpeakerDevice = Daily.create_speaker_device(
"speaker", sample_rate=16000, channels=1
)
Daily.select_speaker_device("speaker")
self.client.set_user_name(self.bot_name)
self.client.join(self.room_url, self.token, completion=self.call_joined)
self.client.update_inputs(
{
"camera": {
"isEnabled": True,
"settings": {
"deviceId": "camera",
},
},
"microphone": {
"isEnabled": True,
"settings": {
"deviceId": "mic",
"customConstraints": {
"autoGainControl": {"exact": False},
"echoCancellation": {"exact": False},
"noiseSuppression": {"exact": False},
},
},
},
}
)
self.client.update_publishing(
{
"camera": {
"sendSettings": {
"maxQuality": "low",
"encodings": {
"low": {
"maxBitrate": 250000,
"scaleResolutionDownBy": 1.333,
"maxFramerate": 8,
}
},
}
}
}
)
self.my_participant_id = self.client.participants()["local"]["id"]
def run(self) -> None:
self.configure_daily()
self.running_thread = Thread(target=self.run_daily, daemon=True)
self.running_thread.start()
def run_daily(self):
# TODO: this loop could, I think, be replaced with a timer and an event
self.participant_left = False
try:
participant_count: int = len(self.client.participants())
self.logger.info(f"{participant_count} participants in room")
while time.time() < self.expiration and not self.participant_left:
# all handling of incoming transcriptions happens in on_transcription_message
time.sleep(1)
except Exception as e:
self.logger.error(f"Exception {e}")
finally:
self.client.leave()
def stop(self):
self.stop_threads.set()
self.camera_thread.join()
self.output_queue.put(OutputQueueFrame(FrameType.END_STREAM, None))
self.frame_consumer_thread.join()
self.client.leave()
def call_joined(self, join_data, client_error):
self.logger.info(f"Call_joined: {join_data}, {client_error}")
self.image: bytes | None = None
self.camera_thread = Thread(target=self.run_camera, daemon=True)
self.camera_thread.start()
self.logger.info("Starting frame consumer thread")
self.frame_consumer_thread = Thread(target=self.frame_consumer, daemon=True)
self.frame_consumer_thread.start()
if self.token:
self.client.start_transcription(
{
"language": "en",
"tier": "nova",
"model": "2-conversationalai",
"profanity_filter": True,
"redact": False,
"extra": {
"endpointing": True,
"punctuate": False,
},
}
)
def on_participant_joined(self, participant):
pass
def on_participant_left(self, participant, reason):
pass
def on_app_message(self, message, sender):
pass
def on_transcription_message(self, message):
with self.tracer.start_as_current_span(
"on_transcription_message", context=self.ctx
):
if message["session_id"] != self.my_participant_id:
self.handle_transcription_fragment(message["text"])
def on_transcription_stopped(self, stopped_by, stopped_by_error):
self.logger.info(f"Transcription stopped {stopped_by}, {stopped_by_error}")
def on_transcription_error(self, message):
self.logger.error(f"Transcription error {message}")
def on_transcription_started(self, status):
self.logger.info(f"Transcription started {status}")
def set_image(self, image: bytes):
self.image: bytes | None = image
def run_camera(self):
try:
while not self.stop_threads.is_set():
if self.image:
self.camera.write_frame(self.image)
time.sleep(1.0 / 8.0) # 8 fps
except Exception as e:
self.logger.error(f"Exception {e} in camera thread.")
def frame_consumer(self):
self.logger.info("🎬 Starting frame consumer thread")
b = bytearray()
smallest_write_size = 3200
all_audio_frames = bytearray()
while True:
try:
frame: OutputQueueFrame = self.output_queue.get()
if frame.frame_type == FrameType.END_STREAM:
self.logger.info("Stopping frame consumer thread")
return
# if interrupted, we just pull frames off the queue and discard them
if not self.is_interrupted.is_set():
if frame:
if frame.frame_type == FrameType.AUDIO_FRAME:
chunk = frame.frame_data
all_audio_frames.extend(chunk)
b.extend(chunk)
l = len(b) - (len(b) % smallest_write_size)
if l:
self.mic.write_frames(bytes(b[:l]))
b = b[l:]
elif frame.frame_type == FrameType.IMAGE_FRAME:
self.set_image(frame.frame_data)
elif len(b):
self.mic.write_frames(bytes(b))
b = bytearray()
else:
if self.interrupt_time:
self.logger.info(
f"Lag to stop stream after interruption {time.perf_counter() - self.interrupt_time}"
)
self.interrupt_time = None
if frame.frame_type == FrameType.START_STREAM:
self.is_interrupted.clear()
self.output_queue.task_done()
except Empty:
try:
if len(b):
self.mic.write_frames(bytes(b))
except Exception as e:
self.logger.error(f"Exception in frame_consumer: {e}, {len(b)}")
b = bytearray()

60
src/samples/05-sync-speech-and-text.py Normal file
View File

@@ -0,0 +1,60 @@
import asyncio
from dailyai.output_queue import OutputQueueFrame, FrameType
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService, AzureImageGenServiceREST
from dailyai.services.daily_transport_service import DailyTransportService
async def main(room_url, token):
class Sample05Transport(DailyTransportService):
def on_participant_joined(self, participant):
super().on_participant_joined(participant)
meeting_duration_minutes = 4
transport = Sample05Transport(
room_url,
token,
"Simple Bot",
meeting_duration_minutes,
)
transport.mic_enabled = True
transport.camera_enabled = True
transport.mic_sample_rate = 16000
transport.camera_width = 1024
transport.camera_height = 1024
llm = AzureLLMService()
tts = AzureTTSService()
dalle = AzureImageGenServiceREST()
inference_text_process = llm.run_llm(
[
{
"role": "system",
"content": f"Describe a nature photograph suitable for use in a calendar, for the month of January. Include only the image description with no preamble."
}
]
)
try:
transport.run()
inference_text = await inference_text_process
tts_iterator = tts.run_tts(inference_text)
(image, audio) = await asyncio.gather(
*[dalle.run_image_gen(inference_text, "1024x1024"), anext(tts_iterator)]
)
transport.output_queue.put(OutputQueueFrame(FrameType.IMAGE_FRAME, image[1]))
transport.output_queue.put(OutputQueueFrame(FrameType.AUDIO_FRAME, audio))
async for audio in tts_iterator:
transport.output_queue.put(
OutputQueueFrame(FrameType.AUDIO_FRAME, audio)
)
await asyncio.sleep(meeting_duration_minutes * 60)
finally:
transport.stop()
print("Done")
if __name__=="__main__":
asyncio.run(main("https://moishe.daily.co/Lettvins", None))