From c77db79447a9030009da771b9222e25c764b25e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 17 May 2024 14:52:51 -0700 Subject: [PATCH] examples: pipelines readability and add LLM assistants after transport --- .../05a-local-sync-speech-and-image.py | 10 +++++++--- examples/foundational/06-listen-and-respond.py | 12 ++++++++++-- examples/foundational/06a-image-sync.py | 11 +++++++++-- examples/foundational/07-interruptible.py | 8 +++++++- examples/foundational/10-wake-word.py | 12 ++++++++++-- examples/foundational/11-sound-effects.py | 14 ++++++++++++-- examples/foundational/12-describe-video.py | 11 +++++++++-- examples/moondream-chatbot/bot.py | 16 +++++++++++----- examples/simple-chatbot/bot.py | 10 ++++++++-- examples/storytelling-chatbot/src/bot.py | 4 ++-- examples/translation-chatbot/bot.py | 11 ++++++++++- 11 files changed, 95 insertions(+), 24 deletions(-) diff --git a/examples/foundational/05a-local-sync-speech-and-image.py b/examples/foundational/05a-local-sync-speech-and-image.py index a5629745b..bfbd453e2 100644 --- a/examples/foundational/05a-local-sync-speech-and-image.py +++ b/examples/foundational/05a-local-sync-speech-and-image.py @@ -98,9 +98,13 @@ async def main(): image_grabber = ImageGrabber() - pipeline = Pipeline([llm, aggregator, description, - ParallelPipeline([tts, audio_grabber], - [imagegen, image_grabber])]) + pipeline = Pipeline([ + llm, + aggregator, + description, + ParallelPipeline([tts, audio_grabber], + [imagegen, image_grabber]) + ]) task = PipelineTask(pipeline) await task.queue_frame(LLMMessagesFrame(messages)) diff --git a/examples/foundational/06-listen-and-respond.py b/examples/foundational/06-listen-and-respond.py index e561c3c64..b7e0d13bc 100644 --- a/examples/foundational/06-listen-and-respond.py +++ b/examples/foundational/06-listen-and-respond.py @@ -70,8 +70,16 @@ async def main(room_url: str, token): tma_in = LLMUserResponseAggregator(messages) tma_out = LLMAssistantResponseAggregator(messages) - pipeline = Pipeline([fl_in, transport.input(), tma_in, llm, - fl_out, tts, tma_out, transport.output()]) + pipeline = Pipeline([ + fl_in, + transport.input(), + tma_in, + llm, + fl_out, + tts, + transport.output(), + tma_out + ]) task = PipelineTask(pipeline) diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index 77278f21d..30f2eea95 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -95,8 +95,15 @@ async def main(room_url: str, token): os.path.join(os.path.dirname(__file__), "assets", "waiting.png"), ) - pipeline = Pipeline([transport.input(), image_sync_aggregator, - tma_in, llm, tma_out, tts, transport.output()]) + pipeline = Pipeline([ + transport.input(), + image_sync_aggregator, + tma_in, + llm, + tts, + transport.output(), + tma_out + ]) task = PipelineTask(pipeline) diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index 70c74f5e2..71f9a22ab 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -65,7 +65,13 @@ async def main(room_url: str, token): tma_in = LLMUserResponseAggregator(messages) tma_out = LLMAssistantResponseAggregator(messages) - pipeline = Pipeline([transport.input(), tma_in, llm, tts, tma_out, transport.output()]) + pipeline = Pipeline([ + transport.input(), + tma_in, + llm, + tts, + transport.output(), + tma_out]) task = PipelineTask(pipeline, allow_interruptions=True) diff --git a/examples/foundational/10-wake-word.py b/examples/foundational/10-wake-word.py index 430f76e8c..cc1829046 100644 --- a/examples/foundational/10-wake-word.py +++ b/examples/foundational/10-wake-word.py @@ -157,8 +157,16 @@ async def main(room_url: str, token): tma_out = LLMAssistantContextAggregator(messages) ncf = NameCheckFilter(["Santa Cat", "Santa"]) - pipeline = Pipeline([transport.input(), isa, ncf, tma_in, - llm, tma_out, tts, transport.output()]) + pipeline = Pipeline([ + transport.input(), + isa, + ncf, + tma_in, + llm, + tts, + transport.output(), + tma_out + ]) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py index 2515a4418..292a89052 100644 --- a/examples/foundational/11-sound-effects.py +++ b/examples/foundational/11-sound-effects.py @@ -111,8 +111,18 @@ async def main(room_url: str, token): fl = FrameLogger("LLM Out") fl2 = FrameLogger("Transcription In") - pipeline = Pipeline([transport.input(), tma_in, in_sound, fl2, llm, - tma_out, fl, tts, out_sound, transport.output()]) + pipeline = Pipeline([ + transport.input(), + tma_in, + in_sound, + fl2, + llm, + fl, + tts, + out_sound, + transport.output(), + tma_out + ]) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/foundational/12-describe-video.py b/examples/foundational/12-describe-video.py index 68033b77f..256580c07 100644 --- a/examples/foundational/12-describe-video.py +++ b/examples/foundational/12-describe-video.py @@ -89,8 +89,15 @@ async def main(room_url: str, token): transport.capture_participant_transcription(participant["id"]) image_requester.set_participant_id(participant["id"]) - pipeline = Pipeline([transport.input(), user_response, image_requester, - vision_aggregator, moondream, tts, transport.output()]) + pipeline = Pipeline([ + transport.input(), + user_response, + image_requester, + vision_aggregator, + moondream, + tts, + transport.output() + ]) task = PipelineTask(pipeline) diff --git a/examples/moondream-chatbot/bot.py b/examples/moondream-chatbot/bot.py index 6a43f617e..b09f4345d 100644 --- a/examples/moondream-chatbot/bot.py +++ b/examples/moondream-chatbot/bot.py @@ -168,11 +168,17 @@ async def main(room_url: str, token): ura = LLMUserResponseAggregator(messages) - pipeline = Pipeline([transport.input(), ura, llm, - ParallelPipeline( - [sa, ir, va, moondream], - [tf, imgf]), - tts, ta, transport.output()]) + pipeline = Pipeline([ + transport.input(), + ura, + llm, + ParallelPipeline( + [sa, ir, va, moondream], + [tf, imgf]), + tts, + ta, + transport.output() + ]) task = PipelineTask(pipeline) await task.queue_frame(quiet_frame) diff --git a/examples/simple-chatbot/bot.py b/examples/simple-chatbot/bot.py index e379ae049..c605414eb 100644 --- a/examples/simple-chatbot/bot.py +++ b/examples/simple-chatbot/bot.py @@ -138,8 +138,14 @@ async def main(room_url: str, token): ta = TalkingAnimation() - pipeline = Pipeline([transport.input(), user_response, - llm, tts, ta, transport.output()]) + pipeline = Pipeline([ + transport.input(), + user_response, + llm, + tts, + ta, + transport.output() + ]) task = PipelineTask(pipeline, allow_interruptions=True) await task.queue_frame(quiet_frame) diff --git a/examples/storytelling-chatbot/src/bot.py b/examples/storytelling-chatbot/src/bot.py index 24f02ccac..c5a75e949 100644 --- a/examples/storytelling-chatbot/src/bot.py +++ b/examples/storytelling-chatbot/src/bot.py @@ -133,8 +133,8 @@ async def main(room_url, token=None): story_processor, image_processor, tts_service, - llm_responses, - transport.output() + transport.output(), + llm_responses ]) main_task = PipelineTask(main_pipeline) diff --git a/examples/translation-chatbot/bot.py b/examples/translation-chatbot/bot.py index 376b3570e..7cd2cbd83 100644 --- a/examples/translation-chatbot/bot.py +++ b/examples/translation-chatbot/bot.py @@ -103,7 +103,16 @@ async def main(room_url: str, token): lfra = LLMFullResponseAggregator() ts = TranslationSubtitles("spanish") - pipeline = Pipeline([transport.input(), sa, tp, llm, lfra, ts, tts, transport.output()]) + pipeline = Pipeline([ + transport.input(), + sa, + tp, + llm, + lfra, + ts, + tts, + transport.output() + ]) task = PipelineTask(pipeline)