From 36f4001877ae750b352ba37b2f6d242fe4d78838 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Wed, 3 Jan 2024 11:55:48 -0800 Subject: [PATCH] three more theoretical samples --- .../theoretical/01-say-one-thing.py | 7 +- .../theoretical/02-llm-say-one-thing.py | 13 +-- .../03-generate-one-video-frame.py | 27 +++++ .../theoretical/04-say-two-things.py | 37 +++++++ .../theoretical/05-llm-speech-and-images.py | 101 ++++++++++++++++++ src/khk-working/theoretical/notes.txt | 5 +- 6 files changed, 173 insertions(+), 17 deletions(-) create mode 100644 src/khk-working/theoretical/03-generate-one-video-frame.py create mode 100644 src/khk-working/theoretical/04-say-two-things.py create mode 100644 src/khk-working/theoretical/05-llm-speech-and-images.py diff --git a/src/khk-working/theoretical/01-say-one-thing.py b/src/khk-working/theoretical/01-say-one-thing.py index 431e91db3..8941a2880 100644 --- a/src/khk-working/theoretical/01-say-one-thing.py +++ b/src/khk-working/theoretical/01-say-one-thing.py @@ -1,14 +1,15 @@ - from dailyai.services.transport.DailyTransport import DailyTransportService from dailyai.services.tts.AzureTTSService import AzureTTSService transport = None +mic = None tts = None def main(): global transport + global mic global tts # create a transport service object using environment variables for @@ -28,7 +29,7 @@ def main(): # chunks of audio to play sequentially. the "mic" object is a handle # we can use to inspect and control the queue if we need to. in this # case we will pipe into this queue from the tts service - mic = transport.audio_queue() + mic = transport.create_audio_queue() tts.set_output(mic) transport.on("error", lambda e: print(e)) @@ -39,7 +40,7 @@ def main(): def say_one_thing(): # say one thing, then leave tts.run_tts("hello world") - transport.on("audio-queue-empty", shutdown) + mic.on("audio-queue-empty", shutdown) def shutdown(): diff --git a/src/khk-working/theoretical/02-llm-say-one-thing.py b/src/khk-working/theoretical/02-llm-say-one-thing.py index 048b0f2cd..a2b66d4b6 100644 --- a/src/khk-working/theoretical/02-llm-say-one-thing.py +++ b/src/khk-working/theoretical/02-llm-say-one-thing.py @@ -1,4 +1,3 @@ - from dailyai.services.transport.DailyTransport import DailyTransportService from dailyai.services.llm.AzureLLMService import AzureLLMService from dailyai.services.tts.AzureTTSService import AzureTTSService @@ -16,18 +15,8 @@ def main(): transport = DailyTransportService() llm = AzureLLMService() tts = AzureTTSService() - mic = transport.audio_queue() + mic = transport.create_audio_queue() tts.set_output(mic) - - # similarly, we can tell the llm to pipe infeference output to our tts - # service. the design idea here is that any time we call llm.run_llm() - # we are creating a cancelable inference call, and somehow behind the - # scenes the full pipeline from the llm to the tts service to the - # transport's audio queue is managed in such a way as to be - # introspectible and cancelable. also, instead of piping the - # output to the tts service directly, we could pipe it to an - # adapter object that does chunking or processing before sending - # to the tts service. llm.set_output(tts) transport.on("error", lambda e: print(e)) diff --git a/src/khk-working/theoretical/03-generate-one-video-frame.py b/src/khk-working/theoretical/03-generate-one-video-frame.py new file mode 100644 index 000000000..e3ac558f3 --- /dev/null +++ b/src/khk-working/theoretical/03-generate-one-video-frame.py @@ -0,0 +1,27 @@ +from dailyai.services.transport.DailyTransport import DailyTransportService +from dailyai.services.genimage.AzureDalleService import AzureDalleService + +dalle = None + + +def main(): + global dalle + + transport = DailyTransportService() + dalle = AzureDalleService() + + # create_video_queue() could presumably take configuration parameters that + # correspond to Daily video settings (resolution, framerate, target + # bitrate, etc.) + cam = transport.create_video_queue() + dalle.set_output(cam) + + transport.on("error", lambda e: print(e)) + transport.on("joined-meeting", say_one_thing) + transport.start() + + +def say_one_thing(): + # make one image, send it to the video queue, then just hang out. + # for simplicity we have not implemented graceful shutdown :-) + dalle.generate_image("an astronaut riding a skateboard") diff --git a/src/khk-working/theoretical/04-say-two-things.py b/src/khk-working/theoretical/04-say-two-things.py new file mode 100644 index 000000000..715412eb6 --- /dev/null +++ b/src/khk-working/theoretical/04-say-two-things.py @@ -0,0 +1,37 @@ +from dailyai.services.transport.DailyTransport import DailyTransportService +from dailyai.services.llm.AzureLLMService import AzureLLMService +from dailyai.services.tts.AzureTTSService import AzureTTSService + +transport = None +llm = None +tts = None + + +def main(): + global transport + global llm + global tts + + transport = DailyTransportService() + llm = AzureLLMService() + tts = AzureTTSService() + mic = transport.create_audio_queue() + tts.set_output(mic) + llm.set_output(tts) + + transport.on("error", lambda e: print(e)) + transport.on("joined-meeting", say_two_things) + transport.start() + + +def say_two_things(): + # queue two pieces of speech: one specified as a text literal, + # and one generated by an llm + tts.run_tts("My friend the LLM is now going to tell a joke about llamas.") + llm.run_llm("tell me a joke about llamas") + transport.on("audio-queue-empty", shutdown) + + +def shutdown(): + transport.stop() + tts.close() diff --git a/src/khk-working/theoretical/05-llm-speech-and-images.py b/src/khk-working/theoretical/05-llm-speech-and-images.py new file mode 100644 index 000000000..93ab7c421 --- /dev/null +++ b/src/khk-working/theoretical/05-llm-speech-and-images.py @@ -0,0 +1,101 @@ +from dailyai.services.transport.DailyTransport import DailyTransportService +from dailyai.services.llm.AzureLLMService import AzureLLMService +from dailyai.services.tts.AzureTTSService import AzureTTSService +from dailyai.services.genimage.AzureDalleService import AzureDalleService +from dailyai.services.utils.AudioImageSynchronizedPair import AudioImageSynchronizedPair + +transport = None +llm = None +tts = None +dalle = None +mic = None +cam = None + + +def main(): + global transport + global llm + global tts + global dalle + + transport = DailyTransportService() + llm = AzureLLMService() + tts = AzureTTSService() + dalle = AzureDalleService() + + # set up mic and cam. but don't wire up automatic output to the mic + # and cam from our AI services because we need to manage synchronization + # of image/speech pairings + mic = transport.create_audio_queue() + cam = transport.create_video_queue() + + transport.on("error", lambda e: print(e)) + transport.on("joined-meeting", narrate_calendar_images) + transport.start() + + +def narrate_calendar_images(): + # let's loop over the months of the year. for each month name, we will have + # our llm generate a description of a nice photograph for that month's page + # in a calendar. + # + # then we'll take the text description and: + # 1. turn it into speech that we send into the session as audio + # 2. turn it into an image that we send into the session as video + # we want the audio and video to be synchronized, so we'll use a helper + # class to manage that. + # + # the first `run_llm()` call defines a lambda to process its output. + # + # the design idea here is that output can be piped into a function that + # takes inference completion text as its argument. *or* output can be + # piped into an object that has more options (maybe a callback for streaming + # results, or a callback for inference completion, or both). + # + # note that we might queue up the month outputs out of order, but that's + # okay for this demo + # + for month in ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]: + synchronizer = AudioImageSynchronizedPair( + audio_output=mic, video_output=cam) + llm.run_llm( + f"""" + Describe a nature photograph suitable for use in a calendar, + for the month of {month}. Include only the image description + with no preamble. + """, + output=lambda inference_text: ( + dalle.generate_image(inference_text, output=synchronizer), + tts.run_tts(inference_text, output=synchronizer) + ), + ) + + +# the AudioImageSynchronizedPair class seems useful enough that I've listed +# it above as a standard utility we can import. but here's a theoretical +# implementation + +class TheoreticalAudioImageSynchronizedPair: + def __init__(self, audio_output, video_output): + self.audio_output = audio_output + self.video_output = video_output + self.image = None + self.audio = None + + def image_generation_complete(self, image): + self.image = image + self._maybe_send() + + def tts_complete(self, audio): + self.audio = audio + self._maybe_send() + + def _maybe_send(self): + if self.image is not None and self.audio is not None: + self.video_output.queue_frame(self.image) + self.audio_output.queue_audio(self.audio) + + +def shutdown(): + transport.stop() + tts.close() diff --git a/src/khk-working/theoretical/notes.txt b/src/khk-working/theoretical/notes.txt index a795fab10..f097198e3 100644 --- a/src/khk-working/theoretical/notes.txt +++ b/src/khk-working/theoretical/notes.txt @@ -1,9 +1,9 @@ -01 just say one thing and exit -02 llm say one thing and exit --03 send "still frame" video +-03 send "still frame" of video -04 manual intro utterance and then llm say one thing and exit --05 queue 10 spoken image prompts and synchronize the audio with the generated image frames +-05 generate images for the months of the year, synchronized with their spoken descriptions -06 chat: llm speak and respond (ignoring transcription input while speaking) -07 chat: llm speak and respond (interruptible) -08 two llms arguing about a topic (in the same process) @@ -12,3 +12,4 @@ -11 06 plus sound effects queued from sound file -12 06 plus background music played through a second "mic" device +