three more theoretical samples

src/khk-working/theoretical/01-say-one-thing.py

@@ -1,14 +1,15 @@
-
 from dailyai.services.transport.DailyTransport import DailyTransportService
 from dailyai.services.tts.AzureTTSService import AzureTTSService
 
 
 transport = None
+mic = None
 tts = None
 
 
 def main():
     global transport
+    global mic
     global tts
 
     # create a transport service object using environment variables for
@@ -28,7 +29,7 @@ def main():
     # chunks of audio to play sequentially. the "mic" object is a handle
     # we can use to inspect and control the queue if we need to. in this
     # case we will pipe into this queue from the tts service
-    mic = transport.audio_queue()
+    mic = transport.create_audio_queue()
     tts.set_output(mic)
 
     transport.on("error", lambda e: print(e))
@@ -39,7 +40,7 @@ def main():
 def say_one_thing():
     # say one thing, then leave
     tts.run_tts("hello world")
-    transport.on("audio-queue-empty", shutdown)
+    mic.on("audio-queue-empty", shutdown)
 
 
 def shutdown():

src/khk-working/theoretical/02-llm-say-one-thing.py

@@ -1,4 +1,3 @@
-
 from dailyai.services.transport.DailyTransport import DailyTransportService
 from dailyai.services.llm.AzureLLMService import AzureLLMService
 from dailyai.services.tts.AzureTTSService import AzureTTSService
@@ -16,18 +15,8 @@ def main():
     transport = DailyTransportService()
     llm = AzureLLMService()
     tts = AzureTTSService()
-    mic = transport.audio_queue()
+    mic = transport.create_audio_queue()
     tts.set_output(mic)
-
-    # similarly, we can tell the llm to pipe infeference output to our tts
-    # service. the design idea here is that any time we call llm.run_llm()
-    # we are creating a cancelable inference call, and somehow behind the
-    # scenes the full pipeline from the llm to the tts service to the
-    # transport's audio queue is managed in such a way as to be
-    # introspectible and cancelable. also, instead of piping the
-    # output to the tts service directly, we could pipe it to an
-    # adapter object that does chunking or processing before sending
-    # to the tts service.
     llm.set_output(tts)
 
     transport.on("error", lambda e: print(e))

src/khk-working/theoretical/03-generate-one-video-frame.py

@@ -0,0 +1,27 @@
+from dailyai.services.transport.DailyTransport import DailyTransportService
+from dailyai.services.genimage.AzureDalleService import AzureDalleService
+
+dalle = None
+
+
+def main():
+    global dalle
+
+    transport = DailyTransportService()
+    dalle = AzureDalleService()
+
+    # create_video_queue() could presumably take configuration parameters that
+    # correspond to Daily video settings (resolution, framerate, target
+    # bitrate, etc.)
+    cam = transport.create_video_queue()
+    dalle.set_output(cam)
+
+    transport.on("error", lambda e: print(e))
+    transport.on("joined-meeting", say_one_thing)
+    transport.start()
+
+
+def say_one_thing():
+    # make one image, send it to the video queue, then just hang out.
+    # for simplicity we have not implemented graceful shutdown :-)
+    dalle.generate_image("an astronaut riding a skateboard")

src/khk-working/theoretical/04-say-two-things.py

@@ -0,0 +1,37 @@
+from dailyai.services.transport.DailyTransport import DailyTransportService
+from dailyai.services.llm.AzureLLMService import AzureLLMService
+from dailyai.services.tts.AzureTTSService import AzureTTSService
+
+transport = None
+llm = None
+tts = None
+
+
+def main():
+    global transport
+    global llm
+    global tts
+
+    transport = DailyTransportService()
+    llm = AzureLLMService()
+    tts = AzureTTSService()
+    mic = transport.create_audio_queue()
+    tts.set_output(mic)
+    llm.set_output(tts)
+
+    transport.on("error", lambda e: print(e))
+    transport.on("joined-meeting", say_two_things)
+    transport.start()
+
+
+def say_two_things():
+    # queue two pieces of speech: one specified as a text literal,
+    # and one generated by an llm
+    tts.run_tts("My friend the LLM is now going to tell a joke about llamas.")
+    llm.run_llm("tell me a joke about llamas")
+    transport.on("audio-queue-empty", shutdown)
+
+
+def shutdown():
+    transport.stop()
+    tts.close()

src/khk-working/theoretical/05-llm-speech-and-images.py

@@ -0,0 +1,101 @@
+from dailyai.services.transport.DailyTransport import DailyTransportService
+from dailyai.services.llm.AzureLLMService import AzureLLMService
+from dailyai.services.tts.AzureTTSService import AzureTTSService
+from dailyai.services.genimage.AzureDalleService import AzureDalleService
+from dailyai.services.utils.AudioImageSynchronizedPair import AudioImageSynchronizedPair
+
+transport = None
+llm = None
+tts = None
+dalle = None
+mic = None
+cam = None
+
+
+def main():
+    global transport
+    global llm
+    global tts
+    global dalle
+
+    transport = DailyTransportService()
+    llm = AzureLLMService()
+    tts = AzureTTSService()
+    dalle = AzureDalleService()
+
+    # set up mic and cam. but don't wire up automatic output to the mic
+    # and cam from our AI services because we need to manage synchronization
+    # of image/speech pairings
+    mic = transport.create_audio_queue()
+    cam = transport.create_video_queue()
+
+    transport.on("error", lambda e: print(e))
+    transport.on("joined-meeting", narrate_calendar_images)
+    transport.start()
+
+
+def narrate_calendar_images():
+    # let's loop over the months of the year. for each month name, we will have
+    # our llm generate a description of a nice photograph for that month's page
+    # in a calendar.
+    #
+    # then we'll take the text description and:
+    #  1. turn it into speech that we send into the session as audio
+    #  2. turn it into an image that we send into the session as video
+    # we want the audio and video to be synchronized, so we'll use a helper
+    # class to manage that.
+    #
+    # the first `run_llm()` call defines a lambda to process its output.
+    #
+    # the design idea here is that output can be piped into a function that
+    # takes inference completion text as its argument. *or* output can be
+    # piped into an object that has more options (maybe a callback for streaming
+    # results, or a callback for inference completion, or both).
+    #
+    # note that we might queue up the month outputs out of order, but that's
+    # okay for this demo
+    #
+    for month in ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]:
+        synchronizer = AudioImageSynchronizedPair(
+            audio_output=mic, video_output=cam)
+        llm.run_llm(
+            f""""
+            Describe a nature photograph suitable for use in a calendar,
+            for the month of {month}. Include only the image description
+            with no preamble.
+            """,
+            output=lambda inference_text: (
+                dalle.generate_image(inference_text, output=synchronizer),
+                tts.run_tts(inference_text, output=synchronizer)
+            ),
+        )
+
+
+# the AudioImageSynchronizedPair class seems useful enough that I've listed
+# it above as a standard utility we can import. but here's a theoretical
+# implementation
+
+class TheoreticalAudioImageSynchronizedPair:
+    def __init__(self, audio_output, video_output):
+        self.audio_output = audio_output
+        self.video_output = video_output
+        self.image = None
+        self.audio = None
+
+    def image_generation_complete(self, image):
+        self.image = image
+        self._maybe_send()
+
+    def tts_complete(self, audio):
+        self.audio = audio
+        self._maybe_send()
+
+    def _maybe_send(self):
+        if self.image is not None and self.audio is not None:
+            self.video_output.queue_frame(self.image)
+            self.audio_output.queue_audio(self.audio)
+
+
+def shutdown():
+    transport.stop()
+    tts.close()

src/khk-working/theoretical/notes.txt

@@ -1,9 +1,9 @@
 
 -01 just say one thing and exit
 -02 llm say one thing and exit
--03 send "still frame" video
+-03 send "still frame" of video 
 -04 manual intro utterance and then llm say one thing and exit
--05 queue 10 spoken image prompts and synchronize the audio with the generated image frames
+-05 generate images for the months of the year, synchronized with their spoken descriptions
 -06 chat: llm speak and respond (ignoring transcription input while speaking)
 -07 chat: llm speak and respond (interruptible)
 -08 two llms arguing about a topic (in the same process)
@@ -12,3 +12,4 @@
 -11 06 plus sound effects queued from sound file
 -12 06 plus background music played through a second "mic" device 
 
+