From 31db156dfcab9f295cce56ef006fb1755e37e01c Mon Sep 17 00:00:00 2001 From: Liza <65890040+lazeratops@users.noreply.github.com> Date: Thu, 25 Jan 2024 13:43:25 +0100 Subject: [PATCH 1/8] Local Whisper transcription (#10) * First pass at Whisper transcription * deletions * Revise based on feedback, add autopep8 --- requirements.txt | 3 +- src/dailyai/requirements.txt | 3 +- src/dailyai/services/ai_services.py | 37 +++++++++- .../services/daily_transport_service.py | 36 ++++++++-- src/dailyai/services/local_stt_service.py | 72 +++++++++++++++++++ src/dailyai/services/whisper_ai_services.py | 55 ++++++++++++++ .../foundational/07-whisper-transcription.py | 45 ++++++++++++ 7 files changed, 242 insertions(+), 9 deletions(-) create mode 100644 src/dailyai/services/local_stt_service.py create mode 100644 src/dailyai/services/whisper_ai_services.py create mode 100644 src/samples/foundational/07-whisper-transcription.py diff --git a/requirements.txt b/requirements.txt index 6188992d0..04c6a29a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +autopep8==2.0.4 build==1.0.3 packaging==23.2 -pyproject_hooks==1.0.0 +pyproject_hooks==1.0.0 \ No newline at end of file diff --git a/src/dailyai/requirements.txt b/src/dailyai/requirements.txt index 53d28d6fd..66ffbd0bb 100644 --- a/src/dailyai/requirements.txt +++ b/src/dailyai/requirements.txt @@ -1,2 +1,3 @@ Pillow==10.1.0 -typing_extensions==4.9.0 \ No newline at end of file +typing_extensions==4.9.0 +faster-whisper==0.10.0 \ No newline at end of file diff --git a/src/dailyai/services/ai_services.py b/src/dailyai/services/ai_services.py index 684912989..24c676709 100644 --- a/src/dailyai/services/ai_services.py +++ b/src/dailyai/services/ai_services.py @@ -1,5 +1,9 @@ +import array import asyncio +import io import logging +import math +import wave from dailyai.queue_frame import ( AudioQueueFrame, @@ -11,7 +15,7 @@ from dailyai.queue_frame import ( ) from abc import abstractmethod -from typing import AsyncGenerator, AsyncIterable, Iterable +from typing import AsyncGenerator, AsyncIterable, BinaryIO, Iterable from dataclasses import dataclass @@ -150,9 +154,40 @@ class ImageGenService(AIService): (url, image_data) = await self.run_image_gen(frame.text) yield ImageQueueFrame(url, image_data) +class STTService(AIService): + """STTService is a base class for speech-to-text services.""" + + _frame_rate: int + def __init__(self, frame_rate: int = 16000, **kwargs): + super().__init__(**kwargs) + self._frame_rate = frame_rate + + + @abstractmethod + async def run_stt(self, audio: BinaryIO) -> str: + """Returns transcript as a string""" + pass + + async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: + """Processes a frame of audio data, either buffering or transcribing it.""" + if not isinstance(frame, AudioQueueFrame): + return + + data = frame.data + content = io.BufferedRandom(io.BytesIO()) + ww = wave.open(self._content, "wb") + ww.setnchannels(1) + ww.setsampwidth(2) + ww.setframerate(self._frame_rate) + ww.writeframesraw(data) + ww.close() + content.seek(0) + text = await self.run_stt(content) + yield TextQueueFrame(text) @dataclass class AIServiceConfig: tts: TTSService image: ImageGenService llm: LLMService + stt: STTService diff --git a/src/dailyai/services/daily_transport_service.py b/src/dailyai/services/daily_transport_service.py index f1eac2bbd..aed5d9c9e 100644 --- a/src/dailyai/services/daily_transport_service.py +++ b/src/dailyai/services/daily_transport_service.py @@ -31,6 +31,10 @@ from daily import ( class DailyTransportService(EventHandler): _daily_initialized = False _lock = threading.Lock() + + speaker_enabled: bool + speaker_sample_rate: int + def __init__( self, room_url: str, @@ -38,6 +42,9 @@ class DailyTransportService(EventHandler): bot_name: str, duration: float = 10, min_others_count: int = 1, + start_transcription: bool = True, + speaker_enabled: bool = False, + speaker_sample_rate: int = 16000, ): super().__init__() self.bot_name: str = bot_name @@ -46,6 +53,7 @@ class DailyTransportService(EventHandler): self.duration: float = duration self.expiration = time.time() + duration * 60 self.min_others_count = min_others_count + self.start_transcription = start_transcription # This queue is used to marshal frames from the async send queue to the thread that emits audio & video. # We need this to maintain the asynchronous behavior of asyncio queues -- to give async functions @@ -61,6 +69,8 @@ class DailyTransportService(EventHandler): self.camera_width = 1024 self.camera_height = 768 self.camera_enabled = False + self.speaker_enabled = speaker_enabled + self.speaker_sample_rate = speaker_sample_rate self.send_queue = asyncio.Queue() self.receive_queue = asyncio.Queue() @@ -144,9 +154,11 @@ class DailyTransportService(EventHandler): "camera", width=self.camera_width, height=self.camera_height, color_format="RGB" ) - self.speaker: VirtualSpeakerDevice = Daily.create_speaker_device( - "speaker", sample_rate=16000, channels=1 - ) + if self.speaker_enabled: + self.speaker: VirtualSpeakerDevice = Daily.create_speaker_device( + "speaker", sample_rate=self.speaker_sample_rate, channels=1 + ) + Daily.select_speaker_device("speaker") self.image: bytes | None = None self.camera_thread = Thread(target=self.run_camera, daemon=True) @@ -156,8 +168,6 @@ class DailyTransportService(EventHandler): self.frame_consumer_thread = Thread(target=self.frame_consumer, daemon=True) self.frame_consumer_thread.start() - Daily.select_speaker_device("speaker") - self.client.set_user_name(self.bot_name) self.client.join(self.room_url, self.token, completion=self.call_joined) self.my_participant_id = self.client.participants()["local"]["id"] @@ -201,9 +211,20 @@ class DailyTransportService(EventHandler): } ) - if self.token: + if self.token and self.start_transcription: self.client.start_transcription(self.transcription_settings) + + def _receive_audio(self): + """Receive audio from the Daily call and put it on the receive queue""" + seconds = 1 + desired_frame_count = self.speaker_sample_rate * seconds + while True: + buffer = self.speaker.read_frames(desired_frame_count) + if len(buffer) > 0: + frame = AudioQueueFrame(buffer) + asyncio.run_coroutine_threadsafe(self.receive_queue.put(frame), self.loop) + async def get_receive_frames(self): while True: frame = await self.receive_queue.get() @@ -266,6 +287,9 @@ class DailyTransportService(EventHandler): def call_joined(self, join_data, client_error): self.logger.info(f"Call_joined: {join_data}, {client_error}") + if self.speaker_enabled: + t = Thread(target=self._receive_audio, daemon=True) + t.start() def on_error(self, error): self.logger.error(f"on_error: {error}") diff --git a/src/dailyai/services/local_stt_service.py b/src/dailyai/services/local_stt_service.py new file mode 100644 index 000000000..866d77cac --- /dev/null +++ b/src/dailyai/services/local_stt_service.py @@ -0,0 +1,72 @@ +import array +import io +import math +from typing import AsyncGenerator +import wave +from dailyai.queue_frame import AudioQueueFrame, QueueFrame, TextQueueFrame +from dailyai.services.ai_services import STTService + + +class LocalSTTService(STTService): + _content: io.BufferedRandom + _wave: wave.Wave_write + _current_silence_frames: int + + # Configuration + _min_rms: int + _max_silence_frames: int + _frame_rate: int + + def __init__(self, + min_rms: int = 400, + max_silence_frames: int = 3, + frame_rate: int = 16000, + **kwargs): + super().__init__(frame_rate, **kwargs) + self._current_silence_frames = 0 + self._min_rms = min_rms + self._max_silence_frames = max_silence_frames + self._frame_rate = frame_rate + self._new_wave() + + def _new_wave(self): + """Creates a new wave object and content buffer.""" + self._content = io.BufferedRandom(io.BytesIO()) + ww = wave.open(self._content, "wb") + ww.setnchannels(1) + ww.setsampwidth(2) + ww.setframerate(self._frame_rate) + self._wave = ww + + async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: + """Processes a frame of audio data, either buffering or transcribing it.""" + if not isinstance(frame, AudioQueueFrame): + return + + data = frame.data + # Try to filter out empty background noise + # (Very rudimentary approach, can be improved) + rms = self._get_volume(data) + if rms >= self._min_rms: + # If volume is high enough, write new data to wave file + self._wave.writeframesraw(data) + + # If buffer is not empty and we detect a 3-frame pause in speech, + # transcribe the audio gathered so far. + if self._content.tell() > 0 and self._current_silence_frames > self._max_silence_frames: + self._current_silence_frames = 0 + self._wave.close() + self._content.seek(0) + text = await self.run_stt(self._content) + self._new_wave() + yield TextQueueFrame(text) + # If we get this far, this is a frame of silence + self._current_silence_frames += 1 + + def _get_volume(self, audio: bytes) -> float: + # https://docs.python.org/3/library/array.html + audio_array = array.array('h', audio) + squares = [sample**2 for sample in audio_array] + mean = sum(squares) / len(audio_array) + rms = math.sqrt(mean) + return rms diff --git a/src/dailyai/services/whisper_ai_services.py b/src/dailyai/services/whisper_ai_services.py new file mode 100644 index 000000000..88bb6f5d4 --- /dev/null +++ b/src/dailyai/services/whisper_ai_services.py @@ -0,0 +1,55 @@ +"""This module implements Whisper transcription with a locally-downloaded model.""" +import asyncio +from enum import Enum +import logging +from typing import BinaryIO +from faster_whisper import WhisperModel +from dailyai.services.local_stt_service import LocalSTTService + + +class Model(Enum): + """Class of basic Whisper model selection options""" + TINY = "tiny" + BASE = "base" + MEDIUM = "medium" + LARGE = "large-v3" + DISTIL_LARGE_V2 = "Systran/faster-distil-whisper-large-v2" + DISTIL_MEDIUM_EN = "Systran/faster-distil-whisper-medium.en" + + +class WhisperSTTService(LocalSTTService): + """Class to transcribe audio with a locally-downloaded Whisper model""" + _model: WhisperModel + + # Model configuration + _model_name: Model + _device: str + _compute_type: str + + def __init__(self, model_name: Model = Model.DISTIL_MEDIUM_EN, + device: str = "auto", + compute_type: str = "default"): + + super().__init__() + self.logger: logging.Logger = logging.getLogger("dailyai") + self._model_name = model_name + self._device = device + self._compute_type = compute_type + self._load() + + def _load(self): + """Loads the Whisper model. Note that if this is the first time + this model is being run, it will take time to download.""" + model = WhisperModel( + self._model_name.value, + device=self._device, + compute_type=self._compute_type) + self._model = model + + async def run_stt(self, audio: BinaryIO = None) -> str: + """Transcribes given audio using Whisper""" + segments, _ = await asyncio.to_thread(self._model.transcribe, audio) + res: str = "" + for segment in segments: + res += f"{segment.text} " + return res diff --git a/src/samples/foundational/07-whisper-transcription.py b/src/samples/foundational/07-whisper-transcription.py new file mode 100644 index 000000000..52376fc6c --- /dev/null +++ b/src/samples/foundational/07-whisper-transcription.py @@ -0,0 +1,45 @@ +import argparse +import asyncio +from threading import Thread + +from dailyai.services.daily_transport_service import DailyTransportService +from dailyai.services.whisper_ai_services import WhisperSTTService + + +async def main(room_url: str): + global transport + global stt + + transport = DailyTransportService( + room_url, + None, + "Transcription bot", + ) + transport.mic_enabled = False + transport.camera_enabled = False + transport.speaker_enabled = True + stt = WhisperSTTService() + transcription_output_queue = asyncio.Queue() + + async def handle_transcription(): + print("`````````TRANSCRIPTION`````````") + while True: + item = await transcription_output_queue.get() + print(item.text) + + async def handle_speaker(): + await stt.run_to_queue( + transcription_output_queue, + transport.get_receive_frames() + ) + await asyncio.gather(transport.run(), handle_speaker(), handle_transcription()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") + parser.add_argument( + "-u", "--url", type=str, required=True, help="URL of the Daily room to join" + ) + + args, unknown = parser.parse_known_args() + asyncio.run(main(args.url)) From 795a33954265ea60c7300a1db2e806a9a7f50a56 Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Tue, 23 Jan 2024 21:07:27 -0500 Subject: [PATCH 2/8] Add InterruptibleConversationWrapper --- src/dailyai/async_processor/__init__.py | 0 .../async_processor/async_processor.py | 347 --------------- src/dailyai/conversation_wrappers.py | 76 ++++ src/dailyai/message_handler/__init__.py | 0 .../message_handler/message_handler.py | 127 ------ src/dailyai/orchestrator.py | 409 ------------------ src/dailyai/queue_aggregators.py | 21 +- src/dailyai/queue_frame.py | 7 +- src/dailyai/services/ai_services.py | 27 +- src/dailyai/services/azure_ai_services.py | 5 +- .../services/daily_transport_service.py | 25 +- src/dailyai/tests/test_asyncprocessor.py | 180 -------- src/dailyai/tests/test_message_handler.py | 147 ------- src/samples/foundational/07-interruptible.py | 98 +++++ 14 files changed, 226 insertions(+), 1243 deletions(-) delete mode 100644 src/dailyai/async_processor/__init__.py delete mode 100644 src/dailyai/async_processor/async_processor.py create mode 100644 src/dailyai/conversation_wrappers.py delete mode 100644 src/dailyai/message_handler/__init__.py delete mode 100644 src/dailyai/message_handler/message_handler.py delete mode 100644 src/dailyai/orchestrator.py delete mode 100644 src/dailyai/tests/test_asyncprocessor.py delete mode 100644 src/dailyai/tests/test_message_handler.py create mode 100644 src/samples/foundational/07-interruptible.py diff --git a/src/dailyai/async_processor/__init__.py b/src/dailyai/async_processor/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/dailyai/async_processor/async_processor.py b/src/dailyai/async_processor/async_processor.py deleted file mode 100644 index 4acd6cc46..000000000 --- a/src/dailyai/async_processor/async_processor.py +++ /dev/null @@ -1,347 +0,0 @@ -import json -import logging -import re - -from collections import defaultdict -from dataclasses import dataclass, field -from enum import Enum -from queue import Queue, PriorityQueue, Empty -from threading import Event, Semaphore, Thread -from typing import Any, Generator, Iterator, Optional, Type - -from dailyai.queue_frame import QueueFrame, FrameType -from dailyai.message_handler.message_handler import MessageHandler -from dailyai.services.ai_services import AIServiceConfig - -class AsyncProcessorState: - # Setting class variables, other synchronous activities - INIT = 0 - - # Making asynchronous requests to LLM and other services to render response - PREPARING = 1 - - # Ready to start presenting to user (but may not have all data yet) - READY = 2 - - # Playing response - PLAYING = 3 - - # An interrupt has been requested and the response is shutting down in-flight processing - INTERRUPTING = 4 - - # An interrupt has been requested and the response is finished stopping in-flight processing - INTERRUPTED = 5 - - # Response has been played or interrupted - DONE = 6 - - # Response is being finalized (updating records of speech, updating LLM context, etc.) - FINALIZING = 7 - - # Response is complete. This could mean that everything is updated, or that the response - # was interrupted. - FINALIZED = 8 - - state_transitions = { - INIT: [PREPARING, INTERRUPTING], - PREPARING: [READY, INTERRUPTING], - READY: [PLAYING, INTERRUPTING], - PLAYING: [DONE, INTERRUPTING], - INTERRUPTING: [INTERRUPTED], - INTERRUPTED: [DONE], - DONE: [FINALIZING], - FINALIZING: [FINALIZED], - FINALIZED: [FINALIZED], - } - - -@dataclass(order=True) -class StateTransitionItem: - state: int - evt: Event = field(compare=False) - -class AsyncProcessor: - def __init__( - self, - services: AIServiceConfig - ) -> None: - self.state = AsyncProcessorState.INIT - self.prepare_thread = None - self.play_thread = None - self.finalize_thread = None - - self.services: AIServiceConfig = services - - self.state_transition_semaphore = Semaphore() - self.waiting_for_state_changes = PriorityQueue() - self.state_queue = Queue() - - self.state_change_callbacks = defaultdict(list) - - self.was_interrupted = False - - self.logger: logging.Logger = logging.getLogger("dailyai") - - def set_state(self, state: int) -> None: - if state in AsyncProcessorState.state_transitions[self.state]: - self.state_transition_semaphore.acquire() - - self.state: int = state - self.state_transition_semaphore.release() - - # wake up any threads waiting for this state transition - try: - while True: - waiter = self.waiting_for_state_changes.get_nowait() - if waiter.state <= state: - waiter.evt.set() - else: - self.waiting_for_state_changes.put(waiter) - break - except Empty: - pass - - # make all the callbacks for this state - for callback in self.state_change_callbacks[state]: - callback(self) - else: - self.logger.error( - f"Invalid state transition from {self.state} to {state} in {self.__class__.__name__}" - ) - raise Exception(f"Invalid state transition from {self.state} to {state}") - - # - # This is used for state transitions that could be blocked by an interruption. - # If we are interrupted, we silently fail this call. Use only if you know that - # this state transition should fail if the processor has been interrupted. - # - - def maybe_set_state(self, state: int) -> bool: - if state in AsyncProcessorState.state_transitions[self.state]: - self.set_state(state) - return True - else: - return False - - def wait_for_state_transition(self, state: int) -> None: - if self.state >= state: - return - - self.state_transition_semaphore.acquire() - - evt = Event() - self.waiting_for_state_changes.put(StateTransitionItem(state, evt)) - self.state_transition_semaphore.release() - result = evt.wait(120.0) - if not result: - self.logger.error( - f"Timed out waiting for state transition to {state} from {self.state}" - ) - - def set_state_callback(self, state: int, callback: callable) -> None: - self.state_change_callbacks[state].append(callback) - - def prepare(self) -> None: - self.prepare_thread = Thread(target=self.async_prepare, daemon=True) - self.prepare_thread.start() - self.wait_for_state_transition(AsyncProcessorState.READY) - - def play(self) -> None: - self.wait_for_state_transition(AsyncProcessorState.READY) - self.play_thread = Thread(target=self.async_play, daemon=True) - self.play_thread.start() - self.wait_for_state_transition(AsyncProcessorState.PLAYING) - - def finalize(self) -> None: - # don't finalize until we're done playing. - self.wait_for_state_transition(AsyncProcessorState.DONE) - self.set_state(AsyncProcessorState.FINALIZING) - self.do_finalization() - self.set_state(AsyncProcessorState.FINALIZED) - - def interrupt(self) -> None: - # nothing to interrupt if we're already finalizing or finalized, no-op - if self.state in [ - AsyncProcessorState.FINALIZING, - AsyncProcessorState.FINALIZED, - ]: - return - - self.set_state(AsyncProcessorState.INTERRUPTING) - self.was_interrupted = True - self.do_interruption() - self.set_state(AsyncProcessorState.INTERRUPTED) - self.set_state(AsyncProcessorState.DONE) - - def async_play(self) -> None: - self.logger.info(f"Starting to play") - if self.maybe_set_state(AsyncProcessorState.PLAYING): - self.do_play() - self.maybe_set_state(AsyncProcessorState.DONE) - - def async_prepare(self) -> None: - self.set_state(AsyncProcessorState.PREPARING) - self.start_preparation() - self.set_state(AsyncProcessorState.READY) - self.continue_preparation() - self.logger.info(f"Preparation done for {self.__class__.__name__}") - self.preparation_done() - - def start_preparation(self) -> None: - pass - - def continue_preparation(self) -> None: - pass - - def preparation_done(self): - pass - - def get_preparation_iterator(self) -> Iterator: - yield None - - def process_chunk(self, chunk) -> None: - pass - - def do_interruption(self) -> None: - pass - - def do_play(self) -> None: - pass - - def do_finalization(self) -> None: - pass - -# A common class for responses that use a message queue and -# an output queue. - -class OrchestratorResponse(AsyncProcessor): - - def __init__( - self, - services, - message_handler, - output_queue, - ) -> None: - super().__init__(services) - - self.message_handler: MessageHandler = message_handler - self.output_queue: Queue = output_queue - - -class LLMResponse(OrchestratorResponse): - def __init__( - self, - services, - message_handler, - output_queue, - ) -> None: - super().__init__(services, message_handler, output_queue) - - self.has_sent_first_frame = False - - self.chunks_in_preparation = Queue() - - self.llm_responses: list[str] = [] - - def get_preparation_iterator(self) -> Iterator: - messages_for_llm = self.message_handler.get_llm_messages() - self.logger.debug(f"Messages for llm: {json.dumps(messages_for_llm, indent=2)}") - return self.clauses_from_chunks( - self.services.llm.run_llm_async(messages_for_llm) - ) - - def clauses_from_chunks(self, chunks) -> Iterator: - out = "" - for chunk in chunks: - if self.state not in [ - AsyncProcessorState.READY, - AsyncProcessorState.PLAYING, - ]: - break - - out += chunk - - if re.match(r"^.*[.!?]$", out): # it looks like a sentence - yield out.strip() - out = "" - - if out.strip(): - yield out.strip() - - def get_frames_from_tts_response(self, audio_frame) -> list[QueueFrame]: - return [QueueFrame(FrameType.AUDIO, audio_frame)] - - def get_frames_from_chunk(self, chunk) -> Generator[list[QueueFrame], Any, None]: - for audio_frame in self.services.tts.run_tts(chunk): - yield self.get_frames_from_tts_response(audio_frame) - - def start_preparation(self) -> None: - self.preparation_iterator = self.get_preparation_iterator() - - def continue_preparation(self) -> None: - for chunk in self.preparation_iterator: - if self.state not in [ - AsyncProcessorState.READY, - AsyncProcessorState.PLAYING, - ]: - break - - self.process_chunk(chunk) - - def process_chunk(self, chunk) -> None: - self.chunks_in_preparation.put((chunk, self.get_frames_from_chunk(chunk))) - - def preparation_done(self): - self.chunks_in_preparation.put((None, None)) - - def do_play(self) -> None: - while True: - if self.state not in [ - AsyncProcessorState.READY, - AsyncProcessorState.PLAYING, - ]: - break - prepared_chunk = self.chunks_in_preparation.get() - if prepared_chunk[0] == None: - return - - self.play_prepared_chunk(prepared_chunk) - - def play_prepared_chunk(self, prepared_chunk) -> None: - chunk, tts_generator = prepared_chunk - for frames in tts_generator: - if self.state not in [ - AsyncProcessorState.READY, - AsyncProcessorState.PLAYING, - ]: - break - - if not self.has_sent_first_frame: - self.output_queue.put(QueueFrame(FrameType.START_STREAM, None)) - self.has_sent_first_frame = True - - for frame in frames: - self.output_queue.put(frame) - - self.output_queue.join() - self.llm_responses.append(chunk) - - def do_finalization(self) -> None: - self.message_handler.add_assistant_messages(self.llm_responses) - - def do_interruption(self) -> None: - self.chunks_in_preparation.put((None, None)) - - if self.prepare_thread and self.prepare_thread.is_alive(): - self.prepare_thread.join() - - if self.play_thread and self.play_thread.is_alive(): - self.play_thread.join() - - -@dataclass(frozen=True) -class ConversationProcessorCollection: - introduction: Optional[Type[OrchestratorResponse]] = None - waiting: Optional[Type[OrchestratorResponse]] = None - response: Optional[Type[OrchestratorResponse]] = None - goodbye: Optional[Type[OrchestratorResponse]] = None diff --git a/src/dailyai/conversation_wrappers.py b/src/dailyai/conversation_wrappers.py new file mode 100644 index 000000000..79d751f36 --- /dev/null +++ b/src/dailyai/conversation_wrappers.py @@ -0,0 +1,76 @@ +import asyncio +import copy +import functools +from typing import AsyncGenerator, Awaitable, Callable +from dailyai.queue_aggregators import LLMContextAggregator +from dailyai.queue_frame import EndStreamQueueFrame, QueueFrame, TranscriptionQueueFrame + +class InterruptibleConversationWrapper: + + def __init__( + self, + frame_generator: Callable[[], AsyncGenerator[QueueFrame, None]], + runner: Callable[ + [str, LLMContextAggregator, LLMContextAggregator], Awaitable[None] + ], + interrupt: Callable[[], None], + my_participant_id: str|None, + llm_messages: list[dict[str, str]], + llm_context_aggregator_in=LLMContextAggregator, + llm_context_aggregator_out=LLMContextAggregator, + delay_before_speech_seconds: float = 1.0, + ): + self._frame_generator: Callable[[], AsyncGenerator[QueueFrame, None]] = frame_generator + self._runner: Callable[ + [str, LLMContextAggregator, LLMContextAggregator], Awaitable[None] + ] = runner + self._interrupt: Callable[[], None] = interrupt + self._my_participant_id = my_participant_id + self._messages: list[dict[str, str]] = llm_messages + self._delay_before_speech_seconds = delay_before_speech_seconds + self._llm_context_aggregator_in = llm_context_aggregator_in + self._llm_context_aggregator_out = llm_context_aggregator_out + + self._current_phrase = "" + + def update_messages(self, new_messages: list[dict[str, str]], task: asyncio.Task | None): + if task: + if not task.cancelled(): + self._current_phrase = "" + self._messages = new_messages + + async def speak_after_delay(self, user_speech, messages): + await asyncio.sleep(self._delay_before_speech_seconds) + tma_in = self._llm_context_aggregator_in( + messages, "user", self._my_participant_id, False + ) + tma_out = self._llm_context_aggregator_out( + messages, "assistant", self._my_participant_id + ) + + await self._runner(user_speech, tma_in, tma_out) + + async def run_conversation(self): + current_response_task = None + + async for frame in self._frame_generator(): + if isinstance(frame, EndStreamQueueFrame): + break + elif not isinstance(frame, TranscriptionQueueFrame): + continue + + if frame.participantId == self._my_participant_id: + continue + + if current_response_task: + current_response_task.cancel() + self._interrupt() + + self._current_phrase += " " + frame.text + current_llm_messages = copy.deepcopy(self._messages) + current_response_task = asyncio.create_task( + self.speak_after_delay(self._current_phrase, current_llm_messages) + ) + current_response_task.add_done_callback( + functools.partial(self.update_messages, current_llm_messages) + ) diff --git a/src/dailyai/message_handler/__init__.py b/src/dailyai/message_handler/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/dailyai/message_handler/message_handler.py b/src/dailyai/message_handler/message_handler.py deleted file mode 100644 index b9570a016..000000000 --- a/src/dailyai/message_handler/message_handler.py +++ /dev/null @@ -1,127 +0,0 @@ -import logging -import time - -from dataclasses import dataclass -from queue import Queue, Empty -from threading import Thread - -from dailyai.storage.search import SearchIndexer -from dailyai.services.ai_services import AIServiceConfig - - -@dataclass -class Message: - type: str - timestamp: float - message: str - - -class MessageHandler: - def __init__(self, intro): - self.messages: list[Message] = [Message("system", time.time(), intro)] - self.last_user_message_idx:int | None = None - self.finalized_user_message_idx: int | None = None - - def add_user_message(self, message) -> None: - if self.last_user_message_idx is not None and self.last_user_message_idx != self.finalized_user_message_idx: - previous_message: str = self.messages[self.last_user_message_idx].message - self.messages[self.last_user_message_idx] = Message( - "user", time.time(), ' '.join([previous_message, message]) - ) - self.messages = self.messages[: self.last_user_message_idx + 1] - else: - self.messages.append(Message("user", time.time(), message)) - - self.last_user_message_idx = len(self.messages) - 1 - - def add_assistant_message(self, message) -> None: - if self.messages[-1].type == "assistant": - self.messages[-1].message += " " + message - else: - self.messages.append(Message("assistant", time.time(), message)) - - def add_assistant_messages(self, messages) -> None: - self.messages.append(Message("assistant", time.time(), " ".join(messages))) - - def get_llm_messages(self) -> list[dict[str, str]]: - return [{"role": m.type, "content": m.message} for m in self.messages] - - def finalize_user_message(self) -> None: - self.finalized_user_message_idx = self.last_user_message_idx - - def shutdown(self) -> None: - pass - -class IndexingMessageHandler(MessageHandler): - def __init__( - self, intro, services: AIServiceConfig, indexer: SearchIndexer - ) -> None: - super().__init__(intro) - self.services = services - - self.search_indexer = indexer - - self.last_written_idx = 0 - self.storage_message_queue = Queue() - - self.index_writer_thread = Thread(target=self.storage_writer, daemon=True) - self.index_writer_thread.start() - - self.logger = logging.getLogger("dailyai") - - def shutdown(self): - self.finalize_user_message() - self.storage_message_queue.put(None) - self.index_writer_thread.join() - - def storage_writer(self) -> None: - while True: - try: - message_idx = self.storage_message_queue.get() - self.storage_message_queue.task_done() - - if message_idx is None: - return - - if message_idx <= self.last_written_idx: - continue - - self.last_written_idx = message_idx - - message = self.messages[message_idx] - content = message.message - if message.type == "user": - content = self.cleanup_user_message(content) - - # sometimes the LLM returns a string wrapped in quotes and sometimes it doesn't. - # if it didn't, wrap it in quotes - if content[0] != '"': - content = '"' + content + '"' - - self.search_indexer.index_text(content) - except Empty: - pass - - def cleanup_user_message(self, user_message) -> str: - return user_message - - def finalize_user_message(self): - super().finalize_user_message() - self.write_messages_to_storage() - - def write_messages_to_storage(self): - if self.finalized_user_message_idx is None: - return - - for idx in range(self.last_written_idx, len(self.messages)): - self.logger.info( - f"Writing to storage: {self.messages[idx].type} {self.messages[idx].message}" - ) - if ( - self.messages[idx].type == "user" - and idx > self.finalized_user_message_idx - ): - break - - if self.messages[idx].type != "system": - self.storage_message_queue.put(idx) diff --git a/src/dailyai/orchestrator.py b/src/dailyai/orchestrator.py deleted file mode 100644 index dcae4a78d..000000000 --- a/src/dailyai/orchestrator.py +++ /dev/null @@ -1,409 +0,0 @@ -import logging -import os -import time -import wave - -from dataclasses import dataclass -from enum import Enum -from queue import Queue, Empty -from opentelemetry import trace, context - -from dailyai.async_processor.async_processor import ( - AsyncProcessor, - AsyncProcessorState, - ConversationProcessorCollection, - OrchestratorResponse, - LLMResponse, -) -from dailyai.queue_frame import QueueFrame, FrameType -from dailyai.services.ai_services import AIServiceConfig -from dailyai.message_handler.message_handler import MessageHandler - -from threading import Thread, Semaphore, Event, Timer - -from opentelemetry import context -from opentelemetry.context.context import Context - -from daily import ( - EventHandler, - CallClient, - Daily, - VirtualCameraDevice, - VirtualMicrophoneDevice, - VirtualSpeakerDevice, -) - - -@dataclass -class OrchestratorConfig: - room_url: str - token: str - bot_name: str - expiration: float - -# Note that we use this as a default parameter value in the Orchestrator -# constructor. The dataclass is defined with Frozen=True, so this should -# be safe. -default_conversation_collection = ConversationProcessorCollection( - introduction=LLMResponse, - waiting=None, - response=LLMResponse, - goodbye=None, -) - - -class Orchestrator(EventHandler): - - def __init__( - self, - daily_config: OrchestratorConfig, - ai_service_config: AIServiceConfig, - message_handler: MessageHandler, - conversation_processors: ConversationProcessorCollection = default_conversation_collection, - tracer=None, - ): - self.bot_name: str = daily_config.bot_name - self.room_url: str = daily_config.room_url - self.token: str = daily_config.token - self.expiration: float = daily_config.expiration - - self.logger: logging.Logger = logging.getLogger("dailyai") - self.tracer = tracer or trace.get_tracer("orchestrator") - - self.ctx: Context = context.get_current() - - self.transcription = "" - self.last_fragment_at = None - self.talked_at = None - self.paused_at = None - - self.logger.info(f"Creating Response for introductions") - self.services: AIServiceConfig = ai_service_config - self.output_queue = Queue() - self.is_interrupted = Event() - self.stop_threads = Event() - self.story_started = False - - self.message_handler = message_handler - self.conversation_processors: ConversationProcessorCollection = conversation_processors - - if conversation_processors.introduction is not None: - intro = conversation_processors.introduction( - services=self.services, message_handler=self.message_handler, output_queue=self.output_queue - ) - intro.prepare() - intro.set_state_callback(AsyncProcessorState.DONE, self.on_intro_played) - intro.set_state_callback(AsyncProcessorState.FINALIZED, self.on_intro_finished) - self.logger.info(f"Introduction is preparing") - - self.current_response: AsyncProcessor = intro - self.can_interrupt = False - # self.response_event.set() - self.response_semaphore = Semaphore() - - self.speech_timeout = None - self.interrupt_time = None - - self.logger.info("Configuring daily") - self.configure_daily() - - def configure_daily(self): - Daily.init() - self.client = CallClient(event_handler=self) - - self.logger.info(f"Mic sample rate: {self.services.tts.get_mic_sample_rate()}") - self.mic: VirtualMicrophoneDevice = Daily.create_microphone_device( - "mic", sample_rate=self.services.tts.get_mic_sample_rate(), channels=1 - ) - self.speaker: VirtualSpeakerDevice = Daily.create_speaker_device( - "speaker", sample_rate=16000, channels=1 - ) - self.camera: VirtualCameraDevice = Daily.create_camera_device( - "camera", width=720, height=1280, color_format="RGB" - ) - - Daily.select_speaker_device("speaker") - - self.client.set_user_name(self.bot_name) - self.client.join(self.room_url, self.token, completion=self.call_joined) - - self.client.update_inputs( - { - "camera": { - "isEnabled": True, - "settings": { - "deviceId": "camera", - }, - }, - "microphone": { - "isEnabled": True, - "settings": { - "deviceId": "mic", - "customConstraints": { - "autoGainControl": {"exact": False}, - "echoCancellation": {"exact": False}, - "noiseSuppression": {"exact": False}, - }, - }, - }, - } - ) - - self.client.update_publishing( - { - "camera": { - "sendSettings": { - "maxQuality": "low", - "encodings": { - "low": { - "maxBitrate": 250000, - "scaleResolutionDownBy": 1.333, - "maxFramerate": 8, - } - }, - } - } - } - ) - - self.my_participant_id = self.client.participants()["local"]["id"] - - def start(self) -> None: - # TODO: this loop could, I think, be replaced with a timer and an event - self.participant_left = False - - try: - participant_count: int = len(self.client.participants()) - self.logger.info(f"{participant_count} participants in room") - while time.time() < self.expiration and not self.participant_left: - # all handling of incoming transcriptions happens in on_transcription_message - time.sleep(1) - except Exception as e: - self.logger.error(f"Exception {e}") - finally: - self.client.leave() - - def stop(self): - self.logger.info("Stop current response") - if self.current_response: - if self.current_response.state < AsyncProcessorState.INTERRUPTED: - self.current_response.interrupt() - - self.logger.info("Wait for state transition") - self.current_response.wait_for_state_transition(AsyncProcessorState.FINALIZED) - - self.stop_threads.set() - self.camera_thread.join() - self.logger.info("Camera thread stopped") - - self.logger.info("Put stop in output queue") - self.output_queue.put(QueueFrame(FrameType.END_STREAM, None)) - - self.frame_consumer_thread.join() - self.logger.info("Orchestrator stopped.") - - def on_intro_played(self, intro): - self.logger.info(f"Introduction has played") - self.can_interrupt = True - intro.finalize() - - def on_intro_finished(self, intro): - self.logger.info(f"Introduction has finished") - waiting = self.conversation_processors.waiting(self.services, self.message_handler, self.output_queue) - waiting.prepare() - waiting.play() - - def on_response_played(self, response): - response.finalize() - - def on_response_finished(self, response): - if not response.was_interrupted: - self.message_handler.finalize_user_message() - - def call_joined(self, join_data, client_error): - self.logger.info(f"Call_joined: {join_data}, {client_error}") - self.client.start_transcription( - { - "language": "en", - "tier": "nova", - "model": "2-conversationalai", - "profanity_filter": True, - "redact": False, - "extra": { - "endpointing": True, - "punctuate": False, - } - } - ) - - def on_participant_joined(self, participant): - with self.tracer.start_as_current_span("on_participant_joined", context=self.ctx): - self.logger.info(f"on_participant_joined: {participant}") - - # TODO: figure out the architecture to get the story id to the client - # self.client.send_app_message({"event": "story-id", "storyID": self.story_id}) - time.sleep(2) - - if not self.story_started: - self.action() - self.story_started = True - - def on_participant_left(self, participant, reason): - self.logger.info(f"Participant {participant} left") - if len(self.client.participants()) < 2: - self.participant_left = True - - def on_app_message(self, message, sender): - with self.tracer.start_as_current_span("on_app_message", context=self.ctx): - self.logger.info(f"on_app_message {message} from {sender}") - if "isSpeaking" in message and message["isSpeaking"] == True: - self.handle_user_started_talking() - - if "isSpeaking" in message and message["isSpeaking"] == False: - self.handle_user_stopped_talking() - - def on_transcription_message(self, message): - with self.tracer.start_as_current_span("on_transcription_message", context=self.ctx): - if message["session_id"] != self.my_participant_id: - self.handle_transcription_fragment(message['text']) - - def on_transcription_stopped(self, stopped_by, stopped_by_error): - self.logger.info(f"Transcription stopped {stopped_by}, {stopped_by_error}") - - def on_transcription_error(self, message): - self.logger.error(f"Transcription error {message}") - - def on_transcription_started(self, status): - self.logger.info(f"Transcription started {status}") - - def set_image(self, image: bytes): - self.image: bytes | None = image - - def run_camera(self): - try: - while not self.stop_threads.is_set(): - if self.image: - self.camera.write_frame(self.image) - - time.sleep(1.0 / 8.0) # 8 fps - except Exception as e: - self.logger.error(f"Exception {e} in camera thread.") - - def handle_user_started_talking(self): - # TODO: allow configuration of the timer timeout - self.logger.error("user started talking") - self.speech_timeout = Timer(1.0, self.utterance_interrupt) - - def handle_user_stopped_talking(self): - self.logger.error("user stopped talking, canceling utterance interrupt") - if self.speech_timeout: - self.speech_timeout.cancel() - - def utterance_interrupt(self): - self.logger.error("utterance interrupt") - self.is_interrupted.set() - - def handle_transcription_fragment(self, fragment): - if not self.can_interrupt: - return - - # start generating a new response. We'll do the fast parts of the interrupt - # now but wait for the state transition after we've kicked off the prepare - # on the new response. - if ( - self.current_response - and self.current_response.state < AsyncProcessorState.INTERRUPTED - ): - self.interrupt_time = time.perf_counter() - self.is_interrupted.set() - self.current_response.interrupt() - - self.message_handler.add_user_message(fragment) - - response_type: type[OrchestratorResponse] | type[LLMResponse] = self.conversation_processors.response or LLMResponse - new_response: OrchestratorResponse = response_type( - self.services, self.message_handler, self.output_queue - ) - new_response.set_state_callback( - AsyncProcessorState.DONE, self.on_response_played - ) - new_response.set_state_callback( - AsyncProcessorState.FINALIZED, self.on_response_finished - ) - new_response.prepare() - - self.response_semaphore.acquire() - if ( - self.current_response - and self.current_response.state < AsyncProcessorState.INTERRUPTED - ): - self.current_response.wait_for_state_transition( - AsyncProcessorState.FINALIZED - ) - - self.current_response = new_response - self.current_response.play() - - self.response_semaphore.release() - - def action(self): - self.logger.info("Starting camera thread") - self.image: bytes | None = None - self.camera_thread = Thread(target=self.run_camera, daemon=True) - self.camera_thread.start() - - self.logger.info("Starting frame consumer thread") - self.frame_consumer_thread = Thread(target=self.frame_consumer, daemon=True) - self.frame_consumer_thread.start() - - self.logger.info("Playing introduction") - self.can_interrupt = False - self.current_response.play() - - def frame_consumer(self): - self.logger.info("🎬 Starting frame consumer thread") - b = bytearray() - smallest_write_size = 3200 - all_audio_frames = bytearray() - while True: - try: - frame:QueueFrame = self.output_queue.get() - if frame.frame_type == FrameType.END_STREAM: - self.logger.info("Stopping frame consumer thread") - return - - # if interrupted, we just pull frames off the queue and discard them - if not self.is_interrupted.is_set(): - if frame: - if frame.frame_type == FrameType.AUDIO: - chunk = frame.frame_data - - all_audio_frames.extend(chunk) - - b.extend(chunk) - l = len(b) - (len(b) % smallest_write_size) - if l: - self.mic.write_frames(bytes(b[:l])) - b = b[l:] - elif frame.frame_type == FrameType.IMAGE: - self.set_image(frame.frame_data) - elif len(b): - self.mic.write_frames(bytes(b)) - b = bytearray() - else: - if self.interrupt_time: - self.logger.info(f"Lag to stop stream after interruption {time.perf_counter() - self.interrupt_time}") - self.interrupt_time = None - - if frame.frame_type == FrameType.START_STREAM: - self.is_interrupted.clear() - - self.output_queue.task_done() - except Empty: - try: - if len(b): - self.mic.write_frames(bytes(b)) - except Exception as e: - self.logger.error(f"Exception in frame_consumer: {e}, {len(b)}") - - b = bytearray() diff --git a/src/dailyai/queue_aggregators.py b/src/dailyai/queue_aggregators.py index ee6ca34bb..659af1cd7 100644 --- a/src/dailyai/queue_aggregators.py +++ b/src/dailyai/queue_aggregators.py @@ -25,23 +25,24 @@ class QueueTee: await queue.put(frame) class LLMContextAggregator(AIService): - def __init__(self, messages: list[dict], role:str, bot_participant_id=None): + def __init__(self, messages: list[dict], role:str, bot_participant_id=None, complete_sentences=True): self.messages = messages self.bot_participant_id = bot_participant_id self.role = role self.sentence = "" + self.complete_sentences = complete_sentences async def process_frame(self, frame:QueueFrame) -> AsyncGenerator[QueueFrame, None]: - content: str = "" - # TODO: split up transcription by participant if isinstance(frame, TextQueueFrame): - content = frame.text - - self.sentence += content - if self.sentence.endswith((".", "?", "!")): - self.messages.append({"role": self.role, "content": self.sentence}) - self.sentence = "" - yield LLMMessagesQueueFrame(self.messages) + if self.complete_sentences: + self.sentence += frame.text + if self.sentence.endswith((".", "?", "!")): + self.messages.append({"role": self.role, "content": self.sentence}) + self.sentence = "" + yield LLMMessagesQueueFrame(self.messages) + else: + self.messages.append({"role": self.role, "content": frame.text}) + yield LLMMessagesQueueFrame(self.messages) yield frame diff --git a/src/dailyai/queue_frame.py b/src/dailyai/queue_frame.py index d72345aaf..81b391f36 100644 --- a/src/dailyai/queue_frame.py +++ b/src/dailyai/queue_frame.py @@ -5,10 +5,13 @@ from typing import Any class QueueFrame: pass -class StartStreamQueueFrame(QueueFrame): +class ControlQueueFrame(QueueFrame): pass -class EndStreamQueueFrame(QueueFrame): +class StartStreamQueueFrame(ControlQueueFrame): + pass + +class EndStreamQueueFrame(ControlQueueFrame): pass @dataclass() diff --git a/src/dailyai/services/ai_services.py b/src/dailyai/services/ai_services.py index 24c676709..88b84bea7 100644 --- a/src/dailyai/services/ai_services.py +++ b/src/dailyai/services/ai_services.py @@ -7,6 +7,7 @@ import wave from dailyai.queue_frame import ( AudioQueueFrame, + ControlQueueFrame, EndStreamQueueFrame, ImageQueueFrame, LLMMessagesQueueFrame, @@ -28,11 +29,15 @@ class AIService: pass async def run_to_queue(self, queue: asyncio.Queue, frames, add_end_of_stream=False) -> None: - async for frame in self.run(frames): - await queue.put(frame) + try: + async for frame in self.run(frames): + await queue.put(frame) - if add_end_of_stream: - await queue.put(EndStreamQueueFrame()) + if add_end_of_stream: + await queue.put(EndStreamQueueFrame()) + except Exception as e: + print("Exception in run_to_queue", e) + raise e async def run( self, @@ -67,9 +72,8 @@ class AIService: @abstractmethod async def process_frame(self, frame:QueueFrame) -> AsyncGenerator[QueueFrame, None]: - # This is a trick for the interpreter (and linter) to know that this is a generator. - if False: - yield QueueFrame() + if isinstance(frame, ControlQueueFrame): + yield frame @abstractmethod async def finalize(self) -> AsyncGenerator[QueueFrame, None]: @@ -87,7 +91,9 @@ class LLMService(AIService): pass async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: - if isinstance(frame, LLMMessagesQueueFrame): + if isinstance(frame, ControlQueueFrame): + yield frame + elif isinstance(frame, LLMMessagesQueueFrame): async for text_chunk in self.run_llm_async(frame.messages): yield TextQueueFrame(text_chunk) @@ -110,9 +116,11 @@ class TTSService(AIService): yield bytes() async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: - if not isinstance(frame, TextQueueFrame): + if isinstance(frame, ControlQueueFrame): yield frame return + elif not isinstance(frame, TextQueueFrame): + return text: str | None = None if not self.aggregate_sentences: @@ -149,6 +157,7 @@ class ImageGenService(AIService): async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: if not isinstance(frame, TextQueueFrame): + yield frame return (url, image_data) = await self.run_image_gen(frame.text) diff --git a/src/dailyai/services/azure_ai_services.py b/src/dailyai/services/azure_ai_services.py index b723e77e4..46449f208 100644 --- a/src/dailyai/services/azure_ai_services.py +++ b/src/dailyai/services/azure_ai_services.py @@ -35,10 +35,7 @@ class AzureTTSService(TTSService): "" \ f"{sentence}" \ " " - try: - result = await asyncio.to_thread(self.speech_synthesizer.speak_ssml, (ssml)) - except Exception as e: - self.logger.error("Error in azure tts", e) + result = await asyncio.to_thread(self.speech_synthesizer.speak_ssml, (ssml)) self.logger.info("Got azure tts result") if result.reason == ResultReason.SynthesizingAudioCompleted: self.logger.info("Returning result") diff --git a/src/dailyai/services/daily_transport_service.py b/src/dailyai/services/daily_transport_service.py index aed5d9c9e..36b218f77 100644 --- a/src/dailyai/services/daily_transport_service.py +++ b/src/dailyai/services/daily_transport_service.py @@ -7,6 +7,7 @@ import types from functools import partial from queue import Queue, Empty +from typing import AsyncGenerator from dailyai.queue_frame import ( AudioQueueFrame, @@ -14,6 +15,7 @@ from dailyai.queue_frame import ( ImageQueueFrame, QueueFrame, StartStreamQueueFrame, + TextQueueFrame, TranscriptionQueueFrame, ) @@ -114,6 +116,7 @@ class DailyTransportService(EventHandler): handler(*args, **kwargs) except Exception as e: self.logger.error(f"Exception in event handler {event_name}: {e}") + raise e def add_event_handler(self, event_name: str, handler): if not event_name.startswith("on_"): @@ -214,7 +217,6 @@ class DailyTransportService(EventHandler): if self.token and self.start_transcription: self.client.start_transcription(self.transcription_settings) - def _receive_audio(self): """Receive audio from the Daily call and put it on the receive queue""" seconds = 1 @@ -223,9 +225,13 @@ class DailyTransportService(EventHandler): buffer = self.speaker.read_frames(desired_frame_count) if len(buffer) > 0: frame = AudioQueueFrame(buffer) - asyncio.run_coroutine_threadsafe(self.receive_queue.put(frame), self.loop) + if self.loop: + asyncio.run_coroutine_threadsafe(self.receive_queue.put(frame), self.loop) - async def get_receive_frames(self): + def interrupt(self): + self.is_interrupted.set() + + async def get_receive_frames(self) -> AsyncGenerator[QueueFrame, None]: while True: frame = await self.receive_queue.get() yield frame @@ -265,6 +271,7 @@ class DailyTransportService(EventHandler): await asyncio.sleep(1) except Exception as e: self.logger.error(f"Exception {e}") + raise e finally: self.client.leave() @@ -341,6 +348,7 @@ class DailyTransportService(EventHandler): time.sleep(1.0 / 8) # 8 fps except Exception as e: self.logger.error(f"Exception {e} in camera thread.") + raise e def frame_consumer(self): self.logger.info("🎬 Starting frame consumer thread") @@ -382,11 +390,11 @@ class DailyTransportService(EventHandler): self.mic.write_frames(bytes(b)) b = bytearray() else: - if self.interrupt_time: - self.logger.info( - f"Lag to stop stream after interruption {time.perf_counter() - self.interrupt_time}" - ) - self.interrupt_time = None + # if there are leftover audio bytes, write them now; failing to do so + # can cause static in the audio stream. + if len(b): + self.mic.write_frames(bytes(b)) + b = bytearray() if isinstance(frame, StartStreamQueueFrame): self.is_interrupted.clear() @@ -398,5 +406,6 @@ class DailyTransportService(EventHandler): self.mic.write_frames(bytes(b)) except Exception as e: self.logger.error(f"Exception in frame_consumer: {e}, {len(b)}") + raise e b = bytearray() diff --git a/src/dailyai/tests/test_asyncprocessor.py b/src/dailyai/tests/test_asyncprocessor.py deleted file mode 100644 index fcb2781e4..000000000 --- a/src/dailyai/tests/test_asyncprocessor.py +++ /dev/null @@ -1,180 +0,0 @@ -import time -import unittest - -from queue import Queue, Empty -from threading import Thread, Event -from typing import Generator - -from dailyai.async_processor.async_processor import ( - AsyncProcessor, - AsyncProcessorState, - LLMResponse, -) -from dailyai.message_handler.message_handler import MessageHandler -from dailyai.queue_frame import QueueFrame, FrameType -from dailyai.services.ai_services import ( - AIServiceConfig, - ImageGenService, - LLMService, - TTSService, -) -""" -class MockTTSService(TTSService): - def run_tts(self, sentence): - for word in sentence.split(' '): - time.sleep(0.1) - yield bytes(word, "utf-8") - -class MockLLMService(LLMService): - def run_llm_async(self, messages) -> Generator[str, None, None]: - for i in ["Hello ", "there.", "How are ", "you?", "I ", "hope ", "you ", "are ", "well."]: - time.sleep(0.1) - yield i - -class MockImageService(ImageGenService): - def run_image_gen(self, sentence) -> None: - return None - -class TestResponse(unittest.TestCase): - def test_base_state_transitions(self): - mock_tts_service = MockTTSService() - mock_llm_service = MockLLMService() - mock_image_service = MockImageService() - processor = AsyncProcessor(AIServiceConfig(tts=mock_tts_service, llm=mock_llm_service, image=mock_image_service)) - processor.prepare() - processor.play() - processor.finalize() - self.assertEqual(processor.state, AsyncProcessorState.FINALIZED) - - def test_state_transitions(self): - output_queue = Queue() - mock_tts_service = MockTTSService() - mock_llm_service = MockLLMService() - mock_image_service = MockImageService() - message_handler = MessageHandler("Hello World") - processor = LLMResponse( - AIServiceConfig( - tts=mock_tts_service, llm=mock_llm_service, image=mock_image_service - ), - message_handler, - output_queue, - ) - processor.prepare() - processor.play() - - # Consume the output from the output queue. It's necessary to mark these tasks as done for the - # play function to return. - expected_words = ["Hello", "there.", "How", "are", "you?", "I", "hope", "you", "are", "well."] - - # remove the "start_stream" message from the queue - output_queue.get() - output_queue.task_done() - - while expected_words: - actual_word:QueueFrame = output_queue.get() - word = expected_words.pop(0) - self.assertEqual(actual_word.frame_type, FrameType.AUDIO_FRAME) - self.assertEqual(actual_word.frame_data, bytes(word, "utf-8")) - output_queue.task_done() - - processor.finalize() - - self.assertEqual(processor.state, AsyncProcessorState.FINALIZED) - - def test_interrupt_preparation(self): - output_queue = Queue() - mock_tts_service = MockTTSService() - mock_llm_service = MockLLMService() - mock_image_service = MockImageService() - message_handler = MessageHandler("System Message") - processor = LLMResponse( - AIServiceConfig( - tts=mock_tts_service, llm=mock_llm_service, image=mock_image_service - ), - message_handler, - output_queue, - ) - processor.prepare() - interrupt_request_at = time.perf_counter() - processor.interrupt() - processor.finalize() - finalized_at = time.perf_counter() - self.assertTrue(0.1 < finalized_at - interrupt_request_at < 0.2) - print(f"delta: {interrupt_request_at, finalized_at}") - self.assertEqual(processor.state, AsyncProcessorState.FINALIZED) - - def test_interrupt_play(self): - output_queue = Queue() - mock_tts_service = MockTTSService() - mock_llm_service = MockLLMService() - mock_image_service = MockImageService() - message_handler = MessageHandler("System Message") - processor = LLMResponse( - AIServiceConfig( - tts=mock_tts_service, llm=mock_llm_service, image=mock_image_service - ), - message_handler, - output_queue, - ) - processor.prepare() - processor.play() - - stop_processing_output_queue = Event() - def process_output_queue_async(): - # Consume the output from the output queue. It's necessary to mark these tasks as done for the - # play function to return. - time.sleep(0.1) - expected_words = ["Hello", "there.", "How", "are", "you?", "I", "hope", "you", "are", "well."] - while expected_words and not stop_processing_output_queue.is_set(): - try: - actual_word:QueueFrame = output_queue.get_nowait() - if actual_word.frame_type == FrameType.AUDIO_FRAME: - time.sleep(0.1) - word = expected_words.pop(0) - self.assertEqual(actual_word.frame_type, FrameType.AUDIO_FRAME) - self.assertEqual(actual_word.frame_data, bytes(word, "utf-8")) - output_queue.task_done() - except Empty: - pass - - process_output_queue = Thread(target=process_output_queue_async, daemon=True) - process_output_queue.start() - - time.sleep(0.5) - processor.interrupt() - - stop_processing_output_queue.set() - process_output_queue.join() - - processor.finalize() - self.assertEqual(processor.state, AsyncProcessorState.FINALIZED) - - def test_statechange_callback(self): - mock_tts_service = MockTTSService() - mock_llm_service = MockLLMService() - mock_image_service = MockImageService() - processor = AsyncProcessor( - AIServiceConfig( - tts=mock_tts_service, llm=mock_llm_service, image=mock_image_service - ) - ) - is_finalized = False - def set_is_finalized(async_processor:AsyncProcessor): - nonlocal is_finalized - is_finalized = True - - processor.set_state_callback( - AsyncProcessorState.FINALIZED, set_is_finalized - ) - processor.prepare() - self.assertFalse(is_finalized) - processor.play() - self.assertFalse(is_finalized) - processor.finalize() - self.assertTrue(is_finalized) - self.assertEqual(processor.state, AsyncProcessorState.FINALIZED) - - -if __name__ == '__main__': - unittest.main() -""" diff --git a/src/dailyai/tests/test_message_handler.py b/src/dailyai/tests/test_message_handler.py deleted file mode 100644 index 9869755c2..000000000 --- a/src/dailyai/tests/test_message_handler.py +++ /dev/null @@ -1,147 +0,0 @@ -import time -import unittest - -from unittest.mock import MagicMock, call - -from dailyai.message_handler.message_handler import MessageHandler, IndexingMessageHandler -from dailyai.services.ai_services import ( - AIServiceConfig, - TTSService, - LLMService, - ImageGenService, -) -from ..storage.search import SearchIndexer - - -class TestMessageHandler(unittest.TestCase): - def test_simple_intro(self): - message_handler = MessageHandler("Hello world") - self.assertEqual( - message_handler.get_llm_messages(), - [{"role": "system", "content": "Hello world"}], - ) - - def test_simple_user_message(self): - message_handler = MessageHandler("System prompt") - message_handler.add_user_message("User message") - self.assertEqual( - message_handler.get_llm_messages(), - [ - {"role": "system", "content": "System prompt"}, - {"role": "user", "content": "User message"}, - ], - ) - - def test_simple_user_and_assistant_message(self): - message_handler = MessageHandler("System prompt") - message_handler.add_user_message("User message") - message_handler.add_assistant_message("Assistant message") - self.assertEqual( - message_handler.get_llm_messages(), - [ - {"role": "system", "content": "System prompt"}, - {"role": "user", "content": "User message"}, - {"role": "assistant", "content": "Assistant message"}, - ], - ) - - def test_user_message_overwrite(self): - message_handler = MessageHandler("System prompt") - message_handler.add_user_message("User message") - message_handler.add_assistant_message("Assistant message") - message_handler.add_user_message("plus something else") - self.assertEqual( - message_handler.get_llm_messages(), - [ - {"role": "system", "content": "System prompt"}, - {"role": "user", "content": "User message plus something else"}, - ], - ) - - def test_user_message_after_assistant(self): - message_handler = MessageHandler("System prompt") - message_handler.add_user_message("User message") - message_handler.add_assistant_message("Assistant message") - message_handler.finalize_user_message() - message_handler.add_user_message("other user message") - self.assertEqual( - message_handler.get_llm_messages(), - [ - {"role": "system", "content": "System prompt"}, - {"role": "user", "content": "User message"}, - {"role": "assistant", "content": "Assistant message"}, - {"role": "user", "content": "other user message"}, - ], - ) - - -class MockTTSService(TTSService): - def run_tts(self, sentence): - for word in sentence.split(" "): - time.sleep(0.1) - yield bytes(word, "utf-8") - - -class MockLLMService(LLMService): - def run_llm(self, messages) -> str: - return "Parsed user message." - -class MockImageService(ImageGenService): - def run_image_gen(self, sentence) -> None: - return None - - -class TestStorageMessageHandler(unittest.TestCase): - def test_user_message_finalized(self): - mock_tts_service = MockTTSService() - mock_llm_service = MockLLMService() - mock_image_service = MockImageService() - - service_config = AIServiceConfig( - tts=mock_tts_service, llm=mock_llm_service, image=mock_image_service - ) - - mock_indexer = MagicMock(spec=SearchIndexer) - - message_handler = IndexingMessageHandler( - "Hello world", service_config, mock_indexer - ) - message_handler.cleanup_user_message = MagicMock(return_value="Parsed user message.") - message_handler.add_user_message("User message") - message_handler.add_assistant_message("Assistant message will be ignored") - message_handler.add_user_message("plus something else") - message_handler.finalize_user_message() - message_handler.add_assistant_message( - "New assistant message will not be ignored" - ) - message_handler.add_user_message("User message second time") - message_handler.add_assistant_message("Assistant message second time") - message_handler.write_messages_to_storage() - - time.sleep(0.5) - message_handler.cleanup_user_message.assert_called_with("User message plus something else") - self.assertEqual( - mock_indexer.mock_calls, - [ - call.index_text('"Parsed user message."'), - call.index_text("New assistant message will not be ignored"), - ], - ) - - mock_indexer.reset_mock() - - message_handler.finalize_user_message() - - time.sleep(0.5) - - self.assertEqual( - mock_indexer.mock_calls, - [ - call.index_text('"Parsed user message."'), - call.index_text("Assistant message second time"), - ], - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/src/samples/foundational/07-interruptible.py b/src/samples/foundational/07-interruptible.py new file mode 100644 index 000000000..96b80325d --- /dev/null +++ b/src/samples/foundational/07-interruptible.py @@ -0,0 +1,98 @@ +import argparse +import asyncio +import requests +import time +import urllib.parse +from dailyai.conversation_wrappers import InterruptibleConversationWrapper + +from dailyai.queue_frame import StartStreamQueueFrame, TextQueueFrame +from dailyai.services.daily_transport_service import DailyTransportService +from dailyai.services.azure_ai_services import AzureLLMService +from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService + + +async def main(room_url:str, token): + global transport + global llm + global tts + + transport = DailyTransportService( + room_url, + token, + "Respond bot", + 5, + ) + transport.mic_enabled = True + transport.mic_sample_rate = 16000 + transport.camera_enabled = False + + llm = AzureLLMService() + tts = ElevenLabsTTSService(voice_id="ErXwobaYiN019PkySvjV") + + async def run_response(user_speech, tma_in, tma_out): + await tts.run_to_queue( + transport.send_queue, + tma_out.run( + llm.run( + tma_in.run( + [StartStreamQueueFrame(), TextQueueFrame(user_speech)] + ) + ) + ), + ) + + @transport.event_handler("on_first_other_participant_joined") + async def on_first_other_participant_joined(transport): + await tts.say("Hi, I'm listening!", transport.send_queue) + + async def run_conversation(): + messages = [ + {"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."}, + ] + + conversation_wrapper = InterruptibleConversationWrapper( + frame_generator=transport.get_receive_frames, + runner=run_response, + interrupt=transport.interrupt, + my_participant_id=transport.my_participant_id, + llm_messages=messages, + ) + await conversation_wrapper.run_conversation() + + transport.transcription_settings["extra"]["punctuate"] = False + await asyncio.gather(transport.run(), run_conversation()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") + parser.add_argument( + "-u", "--url", type=str, required=True, help="URL of the Daily room to join" + ) + parser.add_argument( + "-k", + "--apikey", + type=str, + required=True, + help="Daily API Key (needed to create token)", + ) + + args, unknown = parser.parse_known_args() + + # Create a meeting token for the given room with an expiration 1 hour in the future. + room_name: str = urllib.parse.urlparse(args.url).path[1:] + expiration: float = time.time() + 60 * 60 + + res: requests.Response = requests.post( + f"https://api.daily.co/v1/meeting-tokens", + headers={"Authorization": f"Bearer {args.apikey}"}, + json={ + "properties": {"room_name": room_name, "is_owner": True, "exp": expiration} + }, + ) + + if res.status_code != 200: + raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}") + + token: str = res.json()["token"] + + asyncio.run(main(args.url, token)) From c9c2e5f5615bd6578aa74d0402c4dd7c9cd66c6e Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Thu, 25 Jan 2024 11:18:55 -0500 Subject: [PATCH 3/8] Remove unnecessary try/except --- src/dailyai/services/ai_services.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/dailyai/services/ai_services.py b/src/dailyai/services/ai_services.py index 88b84bea7..d0f35adab 100644 --- a/src/dailyai/services/ai_services.py +++ b/src/dailyai/services/ai_services.py @@ -1,8 +1,6 @@ -import array import asyncio import io import logging -import math import wave from dailyai.queue_frame import ( @@ -29,15 +27,11 @@ class AIService: pass async def run_to_queue(self, queue: asyncio.Queue, frames, add_end_of_stream=False) -> None: - try: - async for frame in self.run(frames): - await queue.put(frame) + async for frame in self.run(frames): + await queue.put(frame) - if add_end_of_stream: - await queue.put(EndStreamQueueFrame()) - except Exception as e: - print("Exception in run_to_queue", e) - raise e + if add_end_of_stream: + await queue.put(EndStreamQueueFrame()) async def run( self, @@ -181,7 +175,7 @@ class STTService(AIService): """Processes a frame of audio data, either buffering or transcribing it.""" if not isinstance(frame, AudioQueueFrame): return - + data = frame.data content = io.BufferedRandom(io.BytesIO()) ww = wave.open(self._content, "wb") From f0d9b0613e630d4d2295017c683bc8a26b1121bb Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Thu, 25 Jan 2024 11:27:00 -0500 Subject: [PATCH 4/8] Add faster_whisper to module dependencies; remove unneeded import --- pyproject.toml | 3 ++- src/samples/foundational/07-whisper-transcription.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4dd677a8f..4f504960a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,8 @@ dependencies = [ "pyht", "opentelemetry-sdk", "aiohttp", - "fal" + "fal", + "faster_whisper" ] [tool.setuptools.packages.find] diff --git a/src/samples/foundational/07-whisper-transcription.py b/src/samples/foundational/07-whisper-transcription.py index 52376fc6c..268f400c7 100644 --- a/src/samples/foundational/07-whisper-transcription.py +++ b/src/samples/foundational/07-whisper-transcription.py @@ -1,6 +1,5 @@ import argparse import asyncio -from threading import Thread from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.whisper_ai_services import WhisperSTTService From 5fdda43bed7f67f753cf756437e1ba0cc678c629 Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Thu, 25 Jan 2024 12:12:46 -0500 Subject: [PATCH 5/8] Autopep linter fixes --- src/dailyai/conversation_wrappers.py | 3 +- src/dailyai/queue_aggregators.py | 11 ++++-- src/dailyai/queue_frame.py | 12 ++++++- src/dailyai/services/ai_services.py | 9 +++-- src/dailyai/services/azure_ai_services.py | 36 ++++++++++++------- .../services/daily_transport_service.py | 15 +++++--- src/dailyai/services/deepgram_ai_services.py | 7 ++-- src/dailyai/services/fal_ai_services.py | 6 ++-- src/dailyai/services/open_ai_services.py | 3 +- .../to_be_updated/cloudflare_ai_service.py | 14 ++++---- .../to_be_updated/deepgram_ai_service.py | 3 +- .../to_be_updated/google_ai_service.py | 16 ++++++--- .../to_be_updated/huggingface_ai_service.py | 14 ++++++-- .../services/to_be_updated/mock_ai_service.py | 6 ++-- .../to_be_updated/playht_ai_service.py | 11 +++--- src/dailyai/tests/test_ai_services.py | 3 ++ .../deprecated/simple-sample/simple-sample.py | 4 ++- .../deprecated/static-sprite/sprite-sample.py | 8 +++-- src/samples/foundational/01-say-one-thing.py | 1 + src/samples/foundational/01a-greet-user.py | 6 +++- .../foundational/02-llm-say-one-thing.py | 1 + src/samples/foundational/03-still-frame.py | 6 ++-- .../foundational/04-utterance-and-speech.py | 3 +- .../foundational/05-sync-speech-and-text.py | 3 +- .../foundational/06-listen-and-respond.py | 3 +- src/samples/foundational/07-interruptible.py | 2 +- src/samples/image-gen.py | 9 ++--- 27 files changed, 147 insertions(+), 68 deletions(-) diff --git a/src/dailyai/conversation_wrappers.py b/src/dailyai/conversation_wrappers.py index 79d751f36..bc7dd902d 100644 --- a/src/dailyai/conversation_wrappers.py +++ b/src/dailyai/conversation_wrappers.py @@ -5,6 +5,7 @@ from typing import AsyncGenerator, Awaitable, Callable from dailyai.queue_aggregators import LLMContextAggregator from dailyai.queue_frame import EndStreamQueueFrame, QueueFrame, TranscriptionQueueFrame + class InterruptibleConversationWrapper: def __init__( @@ -14,7 +15,7 @@ class InterruptibleConversationWrapper: [str, LLMContextAggregator, LLMContextAggregator], Awaitable[None] ], interrupt: Callable[[], None], - my_participant_id: str|None, + my_participant_id: str | None, llm_messages: list[dict[str, str]], llm_context_aggregator_in=LLMContextAggregator, llm_context_aggregator_out=LLMContextAggregator, diff --git a/src/dailyai/queue_aggregators.py b/src/dailyai/queue_aggregators.py index 659af1cd7..e2e5ff7fd 100644 --- a/src/dailyai/queue_aggregators.py +++ b/src/dailyai/queue_aggregators.py @@ -5,6 +5,7 @@ from dailyai.services.ai_services import AIService from typing import AsyncGenerator, List + class QueueTee: async def run_to_queue_and_generate( self, @@ -24,15 +25,21 @@ class QueueTee: for queue in output_queues: await queue.put(frame) + class LLMContextAggregator(AIService): - def __init__(self, messages: list[dict], role:str, bot_participant_id=None, complete_sentences=True): + def __init__( + self, + messages: list[dict], + role: str, + bot_participant_id=None, + complete_sentences=True): self.messages = messages self.bot_participant_id = bot_participant_id self.role = role self.sentence = "" self.complete_sentences = complete_sentences - async def process_frame(self, frame:QueueFrame) -> AsyncGenerator[QueueFrame, None]: + async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: # TODO: split up transcription by participant if isinstance(frame, TextQueueFrame): if self.complete_sentences: diff --git a/src/dailyai/queue_frame.py b/src/dailyai/queue_frame.py index 81b391f36..da30677ec 100644 --- a/src/dailyai/queue_frame.py +++ b/src/dailyai/queue_frame.py @@ -2,39 +2,49 @@ from enum import Enum from dataclasses import dataclass from typing import Any + class QueueFrame: pass + class ControlQueueFrame(QueueFrame): pass + class StartStreamQueueFrame(ControlQueueFrame): pass + class EndStreamQueueFrame(ControlQueueFrame): pass + @dataclass() class AudioQueueFrame(QueueFrame): data: bytes + @dataclass() class ImageQueueFrame(QueueFrame): url: str | None image: bytes + @dataclass() class TextQueueFrame(QueueFrame): text: str + @dataclass() class TranscriptionQueueFrame(TextQueueFrame): participantId: str timestamp: str + @dataclass() class LLMMessagesQueueFrame(QueueFrame): - messages: list[dict[str,str]] # TODO: define this more concretely! + messages: list[dict[str, str]] # TODO: define this more concretely! + class AppMessageQueueFrame(QueueFrame): message: Any diff --git a/src/dailyai/services/ai_services.py b/src/dailyai/services/ai_services.py index d0f35adab..3c5dba3c4 100644 --- a/src/dailyai/services/ai_services.py +++ b/src/dailyai/services/ai_services.py @@ -65,7 +65,7 @@ class AIService: raise e @abstractmethod - async def process_frame(self, frame:QueueFrame) -> AsyncGenerator[QueueFrame, None]: + async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: if isinstance(frame, ControlQueueFrame): yield frame @@ -75,6 +75,7 @@ class AIService: if False: yield QueueFrame() + class LLMService(AIService): @abstractmethod async def run_llm_async(self, messages) -> AsyncGenerator[str, None]: @@ -146,7 +147,7 @@ class ImageGenService(AIService): # Renders the image. Returns an Image object. @abstractmethod - async def run_image_gen(self, sentence:str) -> tuple[str, bytes]: + async def run_image_gen(self, sentence: str) -> tuple[str, bytes]: pass async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: @@ -157,15 +158,16 @@ class ImageGenService(AIService): (url, image_data) = await self.run_image_gen(frame.text) yield ImageQueueFrame(url, image_data) + class STTService(AIService): """STTService is a base class for speech-to-text services.""" _frame_rate: int + def __init__(self, frame_rate: int = 16000, **kwargs): super().__init__(**kwargs) self._frame_rate = frame_rate - @abstractmethod async def run_stt(self, audio: BinaryIO) -> str: """Returns transcript as a string""" @@ -188,6 +190,7 @@ class STTService(AIService): text = await self.run_stt(content) yield TextQueueFrame(text) + @dataclass class AIServiceConfig: tts: TTSService diff --git a/src/dailyai/services/azure_ai_services.py b/src/dailyai/services/azure_ai_services.py index 46449f208..6710534b9 100644 --- a/src/dailyai/services/azure_ai_services.py +++ b/src/dailyai/services/azure_ai_services.py @@ -15,6 +15,7 @@ from PIL import Image # See .env.example for Azure configuration needed from azure.cognitiveservices.speech import SpeechSynthesizer, SpeechConfig, ResultReason, CancellationReason + class AzureTTSService(TTSService): def __init__(self, speech_key=None, speech_region=None): super().__init__() @@ -23,18 +24,19 @@ class AzureTTSService(TTSService): speech_region = speech_region or os.getenv("AZURE_SPEECH_SERVICE_REGION") self.speech_config = SpeechConfig(subscription=speech_key, region=speech_region) - self.speech_synthesizer = SpeechSynthesizer(speech_config=self.speech_config, audio_config=None) + self.speech_synthesizer = SpeechSynthesizer( + speech_config=self.speech_config, audio_config=None) async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]: self.logger.info("Running azure tts") ssml = "" \ - "" \ - "" \ - "" \ - "" \ - f"{sentence}" \ - " " + "xmlns:mstts='http://www.w3.org/2001/mstts'>" \ + "" \ + "" \ + "" \ + "" \ + f"{sentence}" \ + " " result = await asyncio.to_thread(self.speech_synthesizer.speak_ssml, (ssml)) self.logger.info("Got azure tts result") if result.reason == ResultReason.SynthesizingAudioCompleted: @@ -47,6 +49,7 @@ class AzureTTSService(TTSService): if cancellation_details.reason == CancellationReason.Error: self.logger.info("Error details: {}".format(cancellation_details.error_details)) + class AzureLLMService(LLMService): def __init__(self, api_key=None, azure_endpoint=None, api_version=None, model=None): super().__init__() @@ -54,11 +57,13 @@ class AzureLLMService(LLMService): azure_endpoint = azure_endpoint or os.getenv("AZURE_CHATGPT_ENDPOINT") if not azure_endpoint: - raise Exception("No azure endpoint specified for Azure LLM, please set AZURE_CHATGPT_ENDPOINT in the environment or pass it to the AzureLLMService constructor") + raise Exception( + "No azure endpoint specified for Azure LLM, please set AZURE_CHATGPT_ENDPOINT in the environment or pass it to the AzureLLMService constructor") model: str | None = model or os.getenv("AZURE_CHATGPT_DEPLOYMENT_ID") if not model: - raise Exception("No model specified for Azure LLM, please set AZURE_CHATGPT_DEPLOYMENT_ID in the environment or pass it to the AzureLLMService constructor") + raise Exception( + "No model specified for Azure LLM, please set AZURE_CHATGPT_DEPLOYMENT_ID in the environment or pass it to the AzureLLMService constructor") self.model: str = model api_version = api_version or "2023-12-01-preview" @@ -90,9 +95,16 @@ class AzureLLMService(LLMService): else: return None + class AzureImageGenServiceREST(ImageGenService): - def __init__(self, image_size:str, api_key=None, azure_endpoint=None, api_version=None, model=None): + def __init__( + self, + image_size: str, + api_key=None, + azure_endpoint=None, + api_version=None, + model=None): super().__init__(image_size=image_size) self.api_key = api_key or os.getenv("AZURE_DALLE_KEY") self.azure_endpoint = azure_endpoint or os.getenv("AZURE_DALLE_ENDPOINT") @@ -103,7 +115,7 @@ class AzureImageGenServiceREST(ImageGenService): # TODO hoist the session to app-level async with aiohttp.ClientSession() as session: url = f"{self.azure_endpoint}openai/images/generations:submit?api-version={self.api_version}" - headers= { "api-key": self.api_key, "Content-Type": "application/json" } + headers = {"api-key": self.api_key, "Content-Type": "application/json"} body = { # Enter your prompt text here "prompt": sentence, diff --git a/src/dailyai/services/daily_transport_service.py b/src/dailyai/services/daily_transport_service.py index 36b218f77..2e2af9f87 100644 --- a/src/dailyai/services/daily_transport_service.py +++ b/src/dailyai/services/daily_transport_service.py @@ -30,6 +30,7 @@ from daily import ( VirtualSpeakerDevice, ) + class DailyTransportService(EventHandler): _daily_initialized = False _lock = threading.Lock() @@ -111,7 +112,8 @@ class DailyTransportService(EventHandler): if self.loop: asyncio.run_coroutine_threadsafe(handler(*args, **kwargs), self.loop) else: - raise Exception("No event loop to run coroutine. In order to use async event handlers, you must run the DailyTransportService in an asyncio event loop.") + raise Exception( + "No event loop to run coroutine. In order to use async event handlers, you must run the DailyTransportService in an asyncio event loop.") else: handler(*args, **kwargs) except Exception as e: @@ -126,8 +128,11 @@ class DailyTransportService(EventHandler): if event_name not in [method[0] for method in methods]: raise Exception(f"Event handler {event_name} not found") - if not event_name in self.event_handlers: - self.event_handlers[event_name] = [getattr(self, event_name), types.MethodType(handler, self)] + if event_name not in self.event_handlers: + self.event_handlers[event_name] = [ + getattr( + self, event_name), types.MethodType( + handler, self)] setattr(self, event_name, partial(self.patch_method, event_name)) else: self.event_handlers[event_name].append(types.MethodType(handler, self)) @@ -317,7 +322,7 @@ class DailyTransportService(EventHandler): def on_app_message(self, message, sender): pass - def on_transcription_message(self, message:dict): + def on_transcription_message(self, message: dict): if self.loop: participantId = "" if "participantId" in message: @@ -360,7 +365,7 @@ class DailyTransportService(EventHandler): frames_or_frame: QueueFrame | list[QueueFrame] = self.threadsafe_send_queue.get() if isinstance(frames_or_frame, QueueFrame): frames: list[QueueFrame] = [frames_or_frame] - elif isinstance(frames_or_frame, list): + elif isinstance(frames_or_frame, list): frames: list[QueueFrame] = frames_or_frame else: raise Exception("Unknown type in output queue") diff --git a/src/dailyai/services/deepgram_ai_services.py b/src/dailyai/services/deepgram_ai_services.py index 9daec8d3a..24fb2a604 100644 --- a/src/dailyai/services/deepgram_ai_services.py +++ b/src/dailyai/services/deepgram_ai_services.py @@ -7,13 +7,14 @@ import requests from collections.abc import AsyncGenerator from dailyai.services.ai_services import TTSService + class DeepgramTTSService(TTSService): def __init__(self, speech_key=None, voice=None): super().__init__() self.voice = voice or os.getenv("DEEPGRAM_VOICE") or "alpha-asteria-en-v2" self.speech_key = speech_key or os.getenv("DEEPGRAM_API_KEY") - + def get_mic_sample_rate(self): return 24000 @@ -22,8 +23,8 @@ class DeepgramTTSService(TTSService): base_url = "https://api.beta.deepgram.com/v1/speak" request_url = f"{base_url}?model={self.voice}&encoding=linear16&container=none&sample_rate=16000" headers = {"authorization": f"token {self.speech_key}"} - body = { "text": sentence } + body = {"text": sentence} async with aiohttp.ClientSession() as session: async with session.post(request_url, headers=headers, json=body) as r: async for data in r.content: - yield data \ No newline at end of file + yield data diff --git a/src/dailyai/services/fal_ai_services.py b/src/dailyai/services/fal_ai_services.py index 8527cb168..a88dc2241 100644 --- a/src/dailyai/services/fal_ai_services.py +++ b/src/dailyai/services/fal_ai_services.py @@ -8,6 +8,8 @@ from PIL import Image from dailyai.services.ai_services import LLMService, TTSService, ImageGenService # Fal expects FAL_KEY_ID and FAL_KEY_SECRET to be set in the env + + class FalImageGenService(ImageGenService): def __init__(self, image_size): super().__init__(image_size) @@ -18,9 +20,9 @@ class FalImageGenService(ImageGenService): handler = fal.apps.submit( "110602490-fast-sdxl", arguments={ - "prompt": sentence + "prompt": sentence }, - ) + ) print("past fal handler init, about to wait for iter_events...") for event in handler.iter_events(): if isinstance(event, fal.apps.InProgress): diff --git a/src/dailyai/services/open_ai_services.py b/src/dailyai/services/open_ai_services.py index ea6ea07ba..f10e62e12 100644 --- a/src/dailyai/services/open_ai_services.py +++ b/src/dailyai/services/open_ai_services.py @@ -49,8 +49,9 @@ class OpenAILLMService(LLMService): else: return None + class OpenAIImageGenService(ImageGenService): - def __init__(self, image_size:str, api_key=None, model=None): + def __init__(self, image_size: str, api_key=None, model=None): super().__init__(image_size=image_size) api_key = api_key or os.getenv("OPEN_AI_KEY") self.model = model or os.getenv("OPEN_AI_IMAGE_MODEL") or "dall-e-3" diff --git a/src/dailyai/services/to_be_updated/cloudflare_ai_service.py b/src/dailyai/services/to_be_updated/cloudflare_ai_service.py index c249da58a..b4a810bd5 100644 --- a/src/dailyai/services/to_be_updated/cloudflare_ai_service.py +++ b/src/dailyai/services/to_be_updated/cloudflare_ai_service.py @@ -4,6 +4,8 @@ from services.ai_service import AIService # Note that Cloudflare's AI workers are still in beta. # https://developers.cloudflare.com/workers-ai/ + + class CloudflareAIService(AIService): def __init__(self): super().__init__() @@ -19,11 +21,11 @@ class CloudflareAIService(AIService): return response.json() # https://developers.cloudflare.com/workers-ai/models/llm/ - def run_llm(self, messages, latest_user_message=None, stream = True): + def run_llm(self, messages, latest_user_message=None, stream=True): input = { "messages": [ - { "role": "system", "content": "You are a friendly assistant" }, - { "role": "user", "content": sentence } + {"role": "system", "content": "You are a friendly assistant"}, + {"role": "user", "content": sentence} ] } @@ -57,9 +59,9 @@ class CloudflareAIService(AIService): # https://developers.cloudflare.com/workers-ai/models/embedding/ def run_embeddings(self, texts, size="medium"): models = { - "small": "@cf/baai/bge-small-en-v1.5", # 384 output dimensions - "medium": "@cf/baai/bge-base-en-v1.5", # 768 output dimensions - "large": "@cf/baai/bge-large-en-v1.5" #1024 output dimensions + "small": "@cf/baai/bge-small-en-v1.5", # 384 output dimensions + "medium": "@cf/baai/bge-base-en-v1.5", # 768 output dimensions + "large": "@cf/baai/bge-large-en-v1.5" # 1024 output dimensions } return self.run(models[size], {"text": texts}) diff --git a/src/dailyai/services/to_be_updated/deepgram_ai_service.py b/src/dailyai/services/to_be_updated/deepgram_ai_service.py index 271feb2be..b4569e6cd 100644 --- a/src/dailyai/services/to_be_updated/deepgram_ai_service.py +++ b/src/dailyai/services/to_be_updated/deepgram_ai_service.py @@ -17,7 +17,8 @@ class DeepgramAIService(AIService): def run_tts(self, sentence): self.logger.info(f"Running deepgram tts for {sentence}") base_url = "https://api.beta.deepgram.com/v1/speak" - voice = os.getenv("DEEPGRAM_VOICE") or "alpha-apollo-en-v1" # move this to an environment variable + # move this to an environment variable + voice = os.getenv("DEEPGRAM_VOICE") or "alpha-apollo-en-v1" request_url = f"{base_url}?model={voice}&encoding=linear16&container=none" headers = {"authorization": f"token {self.api_key}"} diff --git a/src/dailyai/services/to_be_updated/google_ai_service.py b/src/dailyai/services/to_be_updated/google_ai_service.py index 8a742d79f..7272964f4 100644 --- a/src/dailyai/services/to_be_updated/google_ai_service.py +++ b/src/dailyai/services/to_be_updated/google_ai_service.py @@ -2,9 +2,12 @@ from services.ai_service import AIService import openai import os -# To use Google Cloud's AI products, you'll need to install Google Cloud CLI and enable the TTS and in your project: https://cloud.google.com/sdk/docs/install +# To use Google Cloud's AI products, you'll need to install Google Cloud +# CLI and enable the TTS and in your project: +# https://cloud.google.com/sdk/docs/install from google.cloud import texttospeech + class GoogleAIService(AIService): def __init__(self): super().__init__() @@ -15,11 +18,14 @@ class GoogleAIService(AIService): ) self.audio_config = texttospeech.AudioConfig( - audio_encoding = texttospeech.AudioEncoding.LINEAR16, - sample_rate_hertz = 16000 + audio_encoding=texttospeech.AudioEncoding.LINEAR16, + sample_rate_hertz=16000 ) def run_tts(self, sentence): - synthesis_input = texttospeech.SynthesisInput(text = sentence.strip()) - result = self.client.synthesize_speech(input=synthesis_input, voice=self.voice, audio_config=self.audio_config) + synthesis_input = texttospeech.SynthesisInput(text=sentence.strip()) + result = self.client.synthesize_speech( + input=synthesis_input, + voice=self.voice, + audio_config=self.audio_config) return result diff --git a/src/dailyai/services/to_be_updated/huggingface_ai_service.py b/src/dailyai/services/to_be_updated/huggingface_ai_service.py index 86db63bf4..7c4984067 100644 --- a/src/dailyai/services/to_be_updated/huggingface_ai_service.py +++ b/src/dailyai/services/to_be_updated/huggingface_ai_service.py @@ -1,7 +1,12 @@ from services.ai_service import AIService from transformers import pipeline -# These functions are just intended for testing, not production use. If you'd like to use HuggingFace, you should use your own models, or do some research into the specific models that will work best for your use case. +# These functions are just intended for testing, not production use. If +# you'd like to use HuggingFace, you should use your own models, or do +# some research into the specific models that will work best for your use +# case. + + class HuggingFaceAIService(AIService): def __init__(self): super().__init__() @@ -10,9 +15,12 @@ class HuggingFaceAIService(AIService): classifier = pipeline("sentiment-analysis") return classifier(sentence) - # available models at https://huggingface.co/Helsinki-NLP (**not all models use 2-character language codes**) + # available models at https://huggingface.co/Helsinki-NLP (**not all + # models use 2-character language codes**) def run_text_translation(self, sentence, source_language, target_language): - translator = pipeline(f"translation", model=f"Helsinki-NLP/opus-mt-{source_language}-{target_language}") + translator = pipeline( + f"translation", + model=f"Helsinki-NLP/opus-mt-{source_language}-{target_language}") return translator(sentence)[0]["translation_text"] diff --git a/src/dailyai/services/to_be_updated/mock_ai_service.py b/src/dailyai/services/to_be_updated/mock_ai_service.py index 27d8154bf..be608c9f5 100644 --- a/src/dailyai/services/to_be_updated/mock_ai_service.py +++ b/src/dailyai/services/to_be_updated/mock_ai_service.py @@ -4,6 +4,7 @@ import time from PIL import Image from services.ai_service import AIService + class MockAIService(AIService): def __init__(self): super().__init__() @@ -20,8 +21,7 @@ class MockAIService(AIService): time.sleep(1) return (image_url, image) - def run_llm(self, messages, latest_user_message=None, stream = True): + def run_llm(self, messages, latest_user_message=None, stream=True): for i in range(5): time.sleep(1) - yield({"choices": [{"delta": {"content": f"hello {i}!"}}]}) - + yield ({"choices": [{"delta": {"content": f"hello {i}!"}}]}) diff --git a/src/dailyai/services/to_be_updated/playht_ai_service.py b/src/dailyai/services/to_be_updated/playht_ai_service.py index d38c59b72..4ba9ddc86 100644 --- a/src/dailyai/services/to_be_updated/playht_ai_service.py +++ b/src/dailyai/services/to_be_updated/playht_ai_service.py @@ -8,6 +8,7 @@ from pyht.protos.api_pb2 import Format from services.ai_service import AIService + class PlayHTAIService(AIService): def __init__(self, **kwargs): super().__init__(**kwargs) @@ -23,8 +24,7 @@ class PlayHTAIService(AIService): voice="s3://voice-cloning-zero-shot/820da3d2-3a3b-42e7-844d-e68db835a206/sarah/manifest.json", sample_rate=16000, quality="higher", - format=Format.FORMAT_WAV - ) + format=Format.FORMAT_WAV) def close(self): super().close() @@ -43,14 +43,15 @@ class PlayHTAIService(AIService): fh = io.BytesIO(b) fh.seek(36) (data, size) = struct.unpack('<4sI', fh.read(8)) - self.logger.info(f"first attempt: data: {data}, size: {hex(size)}, position: {fh.tell()}") + self.logger.info( + f"first attempt: data: {data}, size: {hex(size)}, position: {fh.tell()}") while data != b'data': fh.read(size) (data, size) = struct.unpack('<4sI', fh.read(8)) - self.logger.info(f"subsequent data: {data}, size: {hex(size)}, position: {fh.tell()}, data != data: {data != b'data'}") + self.logger.info( + f"subsequent data: {data}, size: {hex(size)}, position: {fh.tell()}, data != data: {data != b'data'}") self.logger.info("position: ", fh.tell()) in_header = False else: if len(chunk): yield chunk - diff --git a/src/dailyai/tests/test_ai_services.py b/src/dailyai/tests/test_ai_services.py index 8bb1e2b00..28423515b 100644 --- a/src/dailyai/tests/test_ai_services.py +++ b/src/dailyai/tests/test_ai_services.py @@ -6,10 +6,12 @@ from typing import AsyncGenerator, Generator from dailyai.services.ai_services import AIService from dailyai.queue_frame import EndStreamQueueFrame, QueueFrame, TextQueueFrame + class SimpleAIService(AIService): async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: yield frame + class TestBaseAIService(unittest.IsolatedAsyncioTestCase): async def test_async_input(self): service = SimpleAIService() @@ -18,6 +20,7 @@ class TestBaseAIService(unittest.IsolatedAsyncioTestCase): TextQueueFrame("hello"), EndStreamQueueFrame() ] + async def iterate_frames() -> AsyncGenerator[QueueFrame, None]: for frame in input_frames: yield frame diff --git a/src/samples/deprecated/simple-sample/simple-sample.py b/src/samples/deprecated/simple-sample/simple-sample.py index 27ae3fece..6a770030a 100644 --- a/src/samples/deprecated/simple-sample/simple-sample.py +++ b/src/samples/deprecated/simple-sample/simple-sample.py @@ -15,11 +15,12 @@ from dailyai.services.ai_services import AIServiceConfig from dailyai.services.azure_ai_services import AzureImageGenService, AzureTTSService, AzureLLMService from dailyai.services.deepgram_ai_services import DeepgramTTSService + def add_bot_to_room(room_url, token, expiration) -> None: # A simple prompt for a simple sample. message_handler = MessageHandler( - """ + """ You are a sample bot in a WebRTC session. You'll receive input as transcriptions of user's speech, and your responses will be converted to audio via a TTS service. Answer user's questions and be friendly, and if you can, give some ideas about how someone @@ -62,6 +63,7 @@ def add_bot_to_room(room_url, token, expiration) -> None: services.tts.close() services.llm.close() + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") parser.add_argument("-u", "--url", type=str, required=True, help="URL of the Daily room") diff --git a/src/samples/deprecated/static-sprite/sprite-sample.py b/src/samples/deprecated/static-sprite/sprite-sample.py index 8cfcbadcd..e8f905542 100644 --- a/src/samples/deprecated/static-sprite/sprite-sample.py +++ b/src/samples/deprecated/static-sprite/sprite-sample.py @@ -20,6 +20,7 @@ from dailyai.message_handler.message_handler import MessageHandler from dailyai.services.ai_services import AIServiceConfig from dailyai.services.azure_ai_services import AzureImageGenService, AzureTTSService, AzureLLMService + class StaticSpriteResponse(OrchestratorResponse): def __init__( @@ -29,8 +30,8 @@ class StaticSpriteResponse(OrchestratorResponse): output_queue ) -> None: super().__init__(services, message_handler, output_queue) - self.image_bytes:bytes | None = None - self.filenames = None # override this in subclasses + self.image_bytes: bytes | None = None + self.filenames = None # override this in subclasses def start_preparation(self) -> None: full_path = os.path.join(os.path.dirname(__file__), "sprites/", self.filename) @@ -82,7 +83,7 @@ def add_bot_to_room(room_url, token, expiration) -> None: # A simple prompt for a simple sample. message_handler = MessageHandler( - """ + """ You are a sample bot in a WebRTC session. You'll receive input as transcriptions of user's speech, and your responses will be converted to audio via a TTS service. Answer user's questions and be friendly, and if you can, give some ideas about how someone @@ -143,6 +144,7 @@ def add_bot_to_room(room_url, token, expiration) -> None: services.image.close() services.llm.close() + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") parser.add_argument("-u", "--url", type=str, required=True, help="URL of the Daily room") diff --git a/src/samples/foundational/01-say-one-thing.py b/src/samples/foundational/01-say-one-thing.py index ac0778fbc..d55d605ff 100644 --- a/src/samples/foundational/01-say-one-thing.py +++ b/src/samples/foundational/01-say-one-thing.py @@ -4,6 +4,7 @@ import asyncio from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService + async def main(room_url): # create a transport service object using environment variables for # the transport service's API key, room url, and any other configuration. diff --git a/src/samples/foundational/01a-greet-user.py b/src/samples/foundational/01a-greet-user.py index dfd33fee3..9bf3434c1 100644 --- a/src/samples/foundational/01a-greet-user.py +++ b/src/samples/foundational/01a-greet-user.py @@ -7,6 +7,7 @@ from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.azure_ai_services import AzureTTSService from dailyai.services.deepgram_ai_services import DeepgramTTSService + async def main(room_url): # create a transport service object using environment variables for # the transport service's API key, room url, and any other configuration. @@ -33,17 +34,20 @@ async def main(room_url): # Register an event handler so we can play the audio when the participant joins. print("settting up handler") + @transport.event_handler("on_participant_joined") async def on_participant_joined(transport, participant): print(f"participant joined: {participant['info']['userName']}") if participant["info"]["isLocal"]: return - audio_generator: AsyncGenerator[bytes, None] = tts.run_tts(f"Hello there, {participant['info']['userName']}!") + audio_generator: AsyncGenerator[bytes, None] = tts.run_tts( + f"Hello there, {participant['info']['userName']}!") async for audio in audio_generator: transport.output_queue.put(QueueFrame(FrameType.AUDIO, audio)) print("setting up call state handler") + @transport.event_handler("on_call_state_updated") async def on_call_joined(transport, state): print(f"call state callback: {state}") diff --git a/src/samples/foundational/02-llm-say-one-thing.py b/src/samples/foundational/02-llm-say-one-thing.py index a54c0ecb7..0da365d42 100644 --- a/src/samples/foundational/02-llm-say-one-thing.py +++ b/src/samples/foundational/02-llm-say-one-thing.py @@ -6,6 +6,7 @@ from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.azure_ai_services import AzureLLMService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService + async def main(room_url): meeting_duration_minutes = 1 transport = DailyTransportService( diff --git a/src/samples/foundational/03-still-frame.py b/src/samples/foundational/03-still-frame.py index 5ffdcc5ac..8ea025b3b 100644 --- a/src/samples/foundational/03-still-frame.py +++ b/src/samples/foundational/03-still-frame.py @@ -8,6 +8,7 @@ from dailyai.services.open_ai_services import OpenAIImageGenService local_joined = False participant_joined = False + async def main(room_url): meeting_duration_minutes = 1 transport = DailyTransportService( @@ -23,8 +24,9 @@ async def main(room_url): imagegen = OpenAIImageGenService(image_size="1024x1024") image_task = asyncio.create_task( - imagegen.run_to_queue(transport.send_queue, [TextQueueFrame("a cat in the style of picasso")]) - ) + imagegen.run_to_queue( + transport.send_queue, [ + TextQueueFrame("a cat in the style of picasso")])) @transport.event_handler("on_participant_joined") async def on_participant_joined(transport, participant): diff --git a/src/samples/foundational/04-utterance-and-speech.py b/src/samples/foundational/04-utterance-and-speech.py index 32844cb6f..8da2147bc 100644 --- a/src/samples/foundational/04-utterance-and-speech.py +++ b/src/samples/foundational/04-utterance-and-speech.py @@ -7,7 +7,8 @@ from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.queue_frame import EndStreamQueueFrame, LLMMessagesQueueFrame from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService -async def main(room_url:str): + +async def main(room_url: str): global transport global llm global tts diff --git a/src/samples/foundational/05-sync-speech-and-text.py b/src/samples/foundational/05-sync-speech-and-text.py index 7e6239288..c08161566 100644 --- a/src/samples/foundational/05-sync-speech-and-text.py +++ b/src/samples/foundational/05-sync-speech-and-text.py @@ -7,6 +7,7 @@ from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.fal_ai_services import FalImageGenService + async def main(room_url): meeting_duration_minutes = 5 transport = DailyTransportService( @@ -98,7 +99,7 @@ async def main(room_url): await transport.run() -if __name__=="__main__": +if __name__ == "__main__": parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") parser.add_argument( "-u", "--url", type=str, required=True, help="URL of the Daily room to join" diff --git a/src/samples/foundational/06-listen-and-respond.py b/src/samples/foundational/06-listen-and-respond.py index 22337f414..8dbaea1e2 100644 --- a/src/samples/foundational/06-listen-and-respond.py +++ b/src/samples/foundational/06-listen-and-respond.py @@ -8,7 +8,8 @@ from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.queue_aggregators import LLMContextAggregator -async def main(room_url:str, token): + +async def main(room_url: str, token): global transport global llm global tts diff --git a/src/samples/foundational/07-interruptible.py b/src/samples/foundational/07-interruptible.py index 96b80325d..cd18804af 100644 --- a/src/samples/foundational/07-interruptible.py +++ b/src/samples/foundational/07-interruptible.py @@ -11,7 +11,7 @@ from dailyai.services.azure_ai_services import AzureLLMService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService -async def main(room_url:str, token): +async def main(room_url: str, token): global transport global llm global tts diff --git a/src/samples/image-gen.py b/src/samples/image-gen.py index 110d0c4f5..fc2ddbaaa 100644 --- a/src/samples/image-gen.py +++ b/src/samples/image-gen.py @@ -11,7 +11,8 @@ from dailyai.queue_frame import QueueFrame, FrameType from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService -async def main(room_url:str, token): + +async def main(room_url: str, token): global transport global llm global tts @@ -32,7 +33,6 @@ async def main(room_url:str, token): tts = AzureTTSService() img = FalImageGenService() - async def handle_transcriptions(): print("handle_transcriptions got called") @@ -41,7 +41,7 @@ async def main(room_url:str, token): print(f"transcription message: {message}") if message["session_id"] == transport.my_participant_id: continue - finder = message["text"].find("start over") + finder = message["text"].find("start over") print(f"finder: {finder}") if finder >= 0: async for audio in tts.run_tts(f"Resetting."): @@ -69,7 +69,8 @@ async def main(room_url:str, token): if participant["info"]["isLocal"]: return async for audio in tts.run_tts("Describe an image, and I'll create it."): - audio_generator = tts.run_tts(f"Hello, {participant['info']['userName']}! Describe an image and I'll create it. To start over, just say 'start over'.") + audio_generator = tts.run_tts( + f"Hello, {participant['info']['userName']}! Describe an image and I'll create it. To start over, just say 'start over'.") async for audio in audio_generator: transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio)) From 95fc8026077d387315784348e1810a904f39ceba Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Fri, 26 Jan 2024 09:15:29 -0500 Subject: [PATCH 6/8] Speaking / waiting images --- src/dailyai/queue_aggregators.py | 33 ++++- src/dailyai/services/ai_services.py | 8 +- src/samples/foundational/06a-image-sync.py | 134 +++++++++++++++++++++ src/samples/foundational/speaking.png | Bin 0 -> 33854 bytes src/samples/foundational/waiting.png | Bin 0 -> 30719 bytes 5 files changed, 166 insertions(+), 9 deletions(-) create mode 100644 src/samples/foundational/06a-image-sync.py create mode 100644 src/samples/foundational/speaking.png create mode 100644 src/samples/foundational/waiting.png diff --git a/src/dailyai/queue_aggregators.py b/src/dailyai/queue_aggregators.py index e2e5ff7fd..3b0ffbc7e 100644 --- a/src/dailyai/queue_aggregators.py +++ b/src/dailyai/queue_aggregators.py @@ -1,6 +1,6 @@ import asyncio -from dailyai.queue_frame import LLMMessagesQueueFrame, QueueFrame, TextQueueFrame +from dailyai.queue_frame import LLMMessagesQueueFrame, QueueFrame, TextQueueFrame, TranscriptionQueueFrame from dailyai.services.ai_services import AIService from typing import AsyncGenerator, List @@ -32,16 +32,24 @@ class LLMContextAggregator(AIService): messages: list[dict], role: str, bot_participant_id=None, - complete_sentences=True): + complete_sentences=True, + pass_through=False): self.messages = messages self.bot_participant_id = bot_participant_id self.role = role self.sentence = "" self.complete_sentences = complete_sentences + self.pass_through = pass_through async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: # TODO: split up transcription by participant if isinstance(frame, TextQueueFrame): + + # Ignore transcription frames from the bot + if isinstance(frame, TranscriptionQueueFrame): + if frame.participantId == self.bot_participant_id: + return + if self.complete_sentences: self.sentence += frame.text if self.sentence.endswith((".", "?", "!")): @@ -52,4 +60,23 @@ class LLMContextAggregator(AIService): self.messages.append({"role": self.role, "content": frame.text}) yield LLMMessagesQueueFrame(self.messages) - yield frame + if self.pass_through: + yield frame + else: + yield frame + +class LLMUserContextAggregator(LLMContextAggregator): + def __init__(self, + messages: list[dict], + bot_participant_id=None, + complete_sentences=True): + super().__init__(messages, "user", bot_participant_id, complete_sentences, pass_through=False) + + +class LLMAssistantContextAggregator(LLMContextAggregator): + def __init__( + self, messages: list[dict], bot_participant_id=None, complete_sentences=True + ): + super().__init__( + messages, "assistan", bot_participant_id, complete_sentences, pass_through=True + ) diff --git a/src/dailyai/services/ai_services.py b/src/dailyai/services/ai_services.py index 3c5dba3c4..263cf186e 100644 --- a/src/dailyai/services/ai_services.py +++ b/src/dailyai/services/ai_services.py @@ -86,9 +86,7 @@ class LLMService(AIService): pass async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: - if isinstance(frame, ControlQueueFrame): - yield frame - elif isinstance(frame, LLMMessagesQueueFrame): + if isinstance(frame, LLMMessagesQueueFrame): async for text_chunk in self.run_llm_async(frame.messages): yield TextQueueFrame(text_chunk) @@ -111,11 +109,9 @@ class TTSService(AIService): yield bytes() async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: - if isinstance(frame, ControlQueueFrame): + if not isinstance(frame, TextQueueFrame): yield frame return - elif not isinstance(frame, TextQueueFrame): - return text: str | None = None if not self.aggregate_sentences: diff --git a/src/samples/foundational/06a-image-sync.py b/src/samples/foundational/06a-image-sync.py new file mode 100644 index 000000000..47bb025de --- /dev/null +++ b/src/samples/foundational/06a-image-sync.py @@ -0,0 +1,134 @@ +import argparse +import asyncio +from typing import AsyncGenerator +import requests +import time +import urllib.parse + +from PIL import Image +from dailyai.queue_frame import ImageQueueFrame, QueueFrame + +from dailyai.services.daily_transport_service import DailyTransportService +from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService +from dailyai.services.ai_services import AIService +from dailyai.queue_aggregators import LLMAssistantContextAggregator, LLMUserContextAggregator +from dailyai.services.fal_ai_services import FalImageGenService + + +class ImageSyncAggregator(AIService): + def __init__(self, speaking_path:str, waiting_path:str): + self._speaking_image = Image.open(speaking_path) + self._speaking_image_bytes = self._speaking_image.tobytes() + + self._waiting_image = Image.open(waiting_path) + self._waiting_image_bytes = self._waiting_image.tobytes() + + async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: + yield ImageQueueFrame(None, self._speaking_image_bytes) + yield frame + yield ImageQueueFrame(None, self._waiting_image_bytes) + +async def main(room_url: str, token): + global transport + global llm + global tts + + transport = DailyTransportService( + room_url, + token, + "Respond bot", + 5, + ) + transport.camera_enabled = True + transport.camera_width = 1024 + transport.camera_height = 1024 + transport.mic_enabled = True + transport.mic_sample_rate = 16000 + + llm = AzureLLMService() + tts = AzureTTSService() + img = FalImageGenService(image_size="1024x1024") + + async def get_images(): + get_speaking_task = asyncio.create_task( + img.run_image_gen("An image of a cat speaking") + ) + get_waiting_task = asyncio.create_task( + img.run_image_gen("An image of a cat waiting") + ) + + (speaking_data, waiting_data) = await asyncio.gather( + get_speaking_task, get_waiting_task + ) + + return speaking_data, waiting_data + + @transport.event_handler("on_first_other_participant_joined") + async def on_first_other_participant_joined(transport): + await tts.say("Hi, I'm listening!", transport.send_queue) + + async def handle_transcriptions(): + messages = [ + {"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."}, + ] + + tma_in = LLMUserContextAggregator( + messages, transport.my_participant_id + ) + tma_out = LLMAssistantContextAggregator( + messages, transport.my_participant_id + ) + image_sync_aggregator = ImageSyncAggregator( + "/Users/moishe/src/daily-ai-sdk/src/samples/foundational/speaking.png", + "/Users/moishe/src/daily-ai-sdk/src/samples/foundational/waiting.png", + ) + await tts.run_to_queue( + transport.send_queue, + image_sync_aggregator.run( + tma_out.run( + llm.run( + tma_in.run( + transport.get_receive_frames() + ) + ) + ) + ) + ) + + transport.transcription_settings["extra"]["punctuate"] = True + await asyncio.gather(transport.run(), handle_transcriptions()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") + parser.add_argument( + "-u", "--url", type=str, required=True, help="URL of the Daily room to join" + ) + parser.add_argument( + "-k", + "--apikey", + type=str, + required=True, + help="Daily API Key (needed to create token)", + ) + + args, unknown = parser.parse_known_args() + + # Create a meeting token for the given room with an expiration 1 hour in the future. + room_name: str = urllib.parse.urlparse(args.url).path[1:] + expiration: float = time.time() + 60 * 60 + + res: requests.Response = requests.post( + f"https://api.daily.co/v1/meeting-tokens", + headers={"Authorization": f"Bearer {args.apikey}"}, + json={ + "properties": {"room_name": room_name, "is_owner": True, "exp": expiration} + }, + ) + + if res.status_code != 200: + raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}") + + token: str = res.json()["token"] + + asyncio.run(main(args.url, token)) diff --git a/src/samples/foundational/speaking.png b/src/samples/foundational/speaking.png new file mode 100644 index 0000000000000000000000000000000000000000..d7bab6fd7f7aa0f7003752e65c16171ef11787ef GIT binary patch literal 33854 zcmeFZhhJ0Mx-J}$DkuU9A__Ai)b0@8!1Km-J&M5Kh?tDuN26aM?xx2}z+m7Tr0 zE8Lk)*BQko?uhRp7ex1JQqv|AF{l{gHq^T=?&6pdLt6SX5kASVUG- zl1*4zRzzA>SOWA-KUo$iqI6V)x`04}9E2Yt&s;@!5a>ds%@adcLv1Zt3%I?Yxh4FC zm7u4+BcUe{*i#m`w6}6KXY;hTb8wOMlxP2|hAePRxGlua_E!~GTX}XvZC$npaAzwv zNkL&jVRnU!Y;0^`XG?2YJ!RE@nghSdv%hq8b(9qn^6>Bw^biw-JHv!TWMpK7ghho! zMFoHw0xn(-uI8Qs4lW#jck-`(l&xGWoNXLkZQu@UgnrFmz};Ns+1UvL{nzL3=XABP z{_l|-T>hCBFhLMX60$~zceG1{IAV`cl2M=`acHqGF(O)m)?`H%P zo_x`Ou;YG>*wVk2LHuhEyE`%A109z6)^dF1{b*_KYOZYVYNbFp1Omb$0-};nfI}cIDJvl?AS@wEc)$O@y8ok} z2hLVN5C5T;zmC$s_wsKw|5G1u=O=Kuox*=Q+kZdBU+w>?^-l+2A;Ql7x1Ij?G5YH+ zaE=r%0xkb_CKWDTS{OisKuREW<$F&&iPk5)>H-Gq6di|MP4*UU%q_j;d>p2BJEd4x znO7qRtt)CxaXbEFkZzvdyZHCao&0V#SLjj@wspgfJ*^9qQ)^3mr+)rR$i30E^*yHE zZ9hc}>s0$vt>S6f;d4xd!cyIoZArz!)Y)MG{_xt_6*i?n5D_s6#T7Q)2h;B*-WS9Z zuK&4t#!5sP=>A_dls?1+l6=s$;=B8I3qk>9pyZ$Of8OAq_w#3e{5cDsm@aTom#R3Zi6Tf5w-c?1KgI;?A~MgW$rX=gXabTM6;DSo{(EUtmZzq;-6X!} zJMRtZSD%)6_1He&@*b|8xditdXvATU+N62|O8kEp@Fb zxV1*yTE61>ib2C}f(u6e@N5r*AF68j4aXnADNpcwig+=&o5{kuiH$Z$qT=gn}WC)Mj!uy}B7o4Ur|hUy{q_I@BC&w!-7cYk--V_bMnrI* z@o`}BP2DzZqRzxGn8frgIn>NgGE**=}Zj^vBNUGQ_<7&rz$8aO=cPNYZ^2MlDb!?^#Ao9BIhuS?VOHv!zkeJh=O$*ZRj_uwbgx%#>v87EFUH2BAK?4a@CrvlyE#D=%y^gOc9+emVSp^3QU= zb0pOs!@m}qFht7dCoxY2VwAwa0lP1ARF#pJabKz?WigQY@rg~zN&4D28q1EK3`Tab z9P7Z@dU(_6yG?$^(C_r@AcG;cP7Y$=W-VCQ^C?x0H1PmKht#I*FiM9-OLDhqW8Y`~ zp&h4EVAC^N;0-3EI%n7#O#|D}9Qpec`1)9QJxqn=xZ`2NqC`nb=gahY8V^*LNtD9L z@>$yCc{lyCEWC1QLaKg68_k5<+`<)qORJ(NvLI%L*ExyK7kh03$9z&&|M`#Gz!xE-S6UOuzSyhBsN>1LYDcZ_G7*qCONwaka^ zn;k{L#eC%%O*K3*yuYoRN_+-CE0a*{(|)CX$;x$lC0T8nuoclfndzf7CJ#p)TQ3nL zhL+R`YdT*mv-_A$K4Zi7o8js6PpY!QNMA`J-Ac)GaowWaOY8eVy4#41vKBCpHP+}N4-hc;;yM;ls%jy*%!{qHF$l6QL{p}Vi zq4g6xx&%N@9d;dy2NINvx{RpCT@My#X3o8Yy(e>EgeQ|jnCorv$|d$aZ21{=@xgn8 zx)*}d!PPfdq0LpI-{hzum;9Z3P;thN#gd1wcM*(*^j@St^jixa+aBw6+gcPzCsf>c z)h=HXs!a?M^CJa~NK%B^DV31N2^1Fdzb$Vlu`q+zC@IxL z8%_aPa@=Lwiy&_}-+=S1btHZ-K-JB&034-&fN&!$yDj^rO@sLtrGE2+f~{@e%?T3_ zYCPM8<9;s@ge6->nok9OE#SNaYk=CYhu7jx)(DEvv~tYY@a%gOdTO@=MJ-di->>O{ z_R2`e(91_nud|}(4b-IpQM5KW&n<-}om$nes#y?#r~HQPDPgbG?Y^<%t>CIS07S$z z^VCtBLF36_vlKfOd44C|{?g%t&)f+Ksbep*BOUzKjDT_}ows$}3aI4rGeFdx9t)l= zKUC~YV#kPU1*2DqS6b9g(Q{NTVqzTB2HWE`-P}44&At%Si1(!$+@N60cA>~u|DVV8 zc2;ViQWg76y8ITSz5P#&wOClvxBVqe)CK?*yWK>tAgsx0Mdor@&KU!>)8fqNlH01=Q#hNUS(+k|SC715kah}sblx6t z)(%$RxJToyTUx}7ohIuVwMu(-ItD0#Fo^H$94p>)C|z48L4a3B z(r55Fl?L5*7N7Nwgoa{3l`76qLf);mL@(9ZJVW7V{I`Qs0;Y;9d^wyyk&G4;gvzg* zAEGdRxEWTQ`H;Qcc1907Fb>&WXeVG?r1{b=N`5gmY)G?FLGx5$ox1`~@&h^t+V>Vr zJnfIoI;HjGD$2S|Q{D`kQw2E_A9J7HRx)!Z=0SSC;K=G^?Q;Fnv$?((l^!_fUR7%% zJN6Ja5AmGs@M2{;o>FdGN$$?vc%#Tc6X$tCP*iQq@bqRL%@eLPVfN<7_do4n6a%h! zMbsf}?6q8-N3D|TPX>ujez|6u8s*E4+2wZ+T*;ed7q9boGB?6sEzfD?2WLa$?Z zWGf&gMKDKmo>B$412c1t*IJvs6$%PcAlAeU4lE5L{3xzmzsQM&jxb63N@cuSVa)vf zDQ#wjj=KRkIOig)q+%YAFpVd!)-*0mSdK$kkWX z_&P?JC->2FQF5Cctj8VnnMdF24=3g1-|QWv3C_~aeuLcFK^4z(+XDj>yRCbd^$-<%D2ozvrGfi)L(XTI)L zKGTNP*2q`dotX!r0%s0zLe9-lsI=6jUfH#x@5_%%^01w1%${@1Q^|Z<+&}RxyY|{E z&OCRMss{CqxKg_5V6D_Vl)hwr3&RsF7Zx{1J+D&HSWyuX`;nelQXT-C(Jn~&?#j2d zllzM$7B8yLc9>6(_;#DApDo{gqRPt|$YT7I>@eSTwEyWDmp$a}Z@^!X{`UJ9if&>4 z@&hE|XJc6zCL!N>V8Ls(AfNk1nuEkS!2uZn_J@|tvI~>NIwFm-s_}e-o;1pDp{({o zOlPNdbVx*`GuP~t>wvZ*aJVYcG@SF>{E+)U4~r^r9-#g&$jMD?fUU7FzDln6xA{f%)JCUclZ`-n8lV@gT*gE#o7^dex=^7Gph3+SpDomWNt4%d4kbC+q8# zOIxms2!eqIn69;AYaVxbIz{Nu6@gu&gGfFL{MO@F*$!S5$$eArh2??>oDyvz1Z@#ph`?kn=^Ut2}9@wOQA>T8r73Z*`wbs9t z@iv91@t6Ys$k5jw37_iaBM~w;&kSEi3_qoqT!q)KBMhX=95g_6v zDIIGrj4@7|5R)WAm59G~PwN=yu=NG_2*jSP$S)>akR5goU&`3e=Gn(=udTR$qH&^A zSNvT7>B)Ib$&sLpH-}`!_2FBCF|#V0CYEEi&Ve~hh?|P2?^MsXUk4l~1*|}~m7AEP z5}0+N#UhGKUU7Iqn_@@eYJz*ca(&aQOkHx&^?q7du#bYWIsf}DzgfwRqPx9jAq=985C8?9cP%MP{ol^wZr ze0@c#Mf|+t>0HYYZBRsB{bBu9MhYmBWz7|lUnnsboO#?0vsO87`+TpEm8)4fuiu)? ze0b_K{0&*0nLbWZZXgyZA*a5_ib zUdGnhQUGpA^+6{s-H~-rUU?QtGgEseRc{MFbWe(j7U(*VfUHe$qQi9eHac>Si!Jh|Q=C)ZCcUw(Ie^A)qF|5=djmY-Iz0`?J@83~+6_spV{37lt39Mut;_eIEHk4GTYRRech^**7JX^;By66nOk)f_6k6p)E+{B%UbJ39h3=+;f)cKFBu&u}5sO{xC>8d5 zo$uOX4Q=Jmhz*)MwvZXnOx>Q2RBxg*?H{e%PI{J=&OcqYR$ObXXV-H0T}u7h+oafG zCR?PLY(@`L41yOK?*3#c&`Vi~J!j_LNn^)~j?6;57RJys!=-BeDwKhDj$L$3d^lZ+ zhgK^qy&0nS?z=o?UUsY;9ST$Saxrn5HFW1904yjz`VFwzP@RZhbHPHPMe;RX3bz?nVn}9we{GGesq= zb^p$fR-|cG)0lEASh=Wv%If8d2wZyl9#3aT7R1NjDwFT> zc&4Ri+VTokldqM)+$!zL}mS_fTFlpfY%+ zNH>M2^tYnjfmV{EcXe&5hM+W&w`dI4Etkz+CjZ1xK`E_%qQ)HD?*dNHPWPRXgNZU- z#c3j%jw%ZWC+~*m1te@1Xi+*l_2`{W;j$vOU%f=TDqZW_NGpuP3bf;H{YEa_g`C5q zslWULzlfHSk`ffXQrMTiSYb)2gnw4X6J!H8v3Xo8(j?D_SK53wT{6dUr}A6TvsnCM zw5w|OQl%363V(l6A8{Cq{&ueto`g+4g9)ZM`60=*5H<;uBUd|1nvwF8T_GQ&FQ=x5 z^SdBFamg2;4*L~nQw(+rGF;W9vN9vCK3jbmS#&Vs`ONq&*(C;eiU*hG34GISK331g zXaYNo3Ptdje2-y+_%D&S9E-Jx#rQt$4J}Vk!T&i)2HFnL6uA8}E#qbU z%}P^FPi?Jqa(nJ=`I!*`OSQ zTW1?#e5i^+7iz=JV#`^$fm{Ac-oR~5oyepbnqnEvA#1~DnNyy0fvAdgiJ%1nIx+_@ zucFX;%r=)yf8Rk(QLza0oImIhDh%b_Zf7}Qf zEANt8X5BFY)4KQ#0K}GNlq1!+EYM0x-B#(ol*g`Uo@58D zV(ol47OS62sgFP13wVXQ1#NPj1*Co@z2xVcGA-)5VykxZQ0D@sH~o9BAF_PJw|&n8 zU_9T=WCM-i7C64jQ1Q^n`=oZx^;kKDrfYNTiDbrJ5sWfy&l@0Nrek7xN8hwUZLLfwtYWN&JhqNRhf-L+f5hB zDy?hawZu~@(csC;OgX=h24ncIVK}i1rLhvZkMXnNLJq9L3Q%Q5lN^)UTptNd!6An%0D?kizU?7;5pJY?W4_& z_OAxtayH`4juYj|Tf(_>C;<>TC#XwxLzx6uqJ;B@rK=}atG~l8cc=a2n`mQVMy$N` zdjFU>&s7c90mqHIgfxY~AXR=Z-tlXS&Nz{=LaxUytTGds&LSh0976MQ5%KUV}3jP$kpUD*|*QL7T4rHtWmEl9G&N03xeR?yX8p7TfgQFVZ|5nB2tV-S+e8E zORb9?hiUt#My36S^T)=;B1J2(*Yxu75|m0Ob>rmdS{r6)CJ*Gm;X0L*i57}U7CF6V65eNg_TFitcEz7wM%NkMfwI2 zF%>D@pjGt`3)a(-@9$P(y6zmDJ?+)qYGYk|Lk{gvR7S6mD(-cQi5Yly2~>4^9%56s zE<%!A=ikTy5=CgLVVh$^fN3p`>9SXd{-RPeR?H%^bXN;mBfC2hFJEWkp{U0w<2igb z9812A65)Hl*rt-masw5RsIDH#?>&BdXLEYgoQfvSVeEiTol3EPByMHSk1R$!N_f(! zU_h#F<`UGYl(sI9D91<^(ry%QTW6a*k?)MZ3$aPoXY?F<<-NQIPu*{^)p2ZG7K#CU z@t;txDeV`dB~KdZ8M#YLuEh)H0rr3=*R?!aWx z+Gm}i^DPc|RY1)24^D5INBEKT;WbT9`iuAZ+qL#HD^tle#PjY%IIS)mt6A$?niTLN z`S^mk3$7n`-4>|hd%*HsY3Pb=H!o&?svrW#iWC&u?djaA!Ycr_!ek}J7aRKp}xO@DyGR`eD-P58!%-R}F9D6i|tJK2FZ!r$k$ z%8490E+zQIHY-Bed7;bB&sARua14}MzT!B+tZG3Q4iZXy7oJ7nf>U^^CGtulK4wG! z@a{0V*u_t4Uz!iF9un9bTPceeABw6^Mx{Q+H! zL_LU0u$(#>suEmci|iilaGl&=^U!v>eE*f$USrwkr*ro@y6h@`4R8fPmJan42S*eq zh&Y&!r#ZrL)*V8K*e@ci81sR-n218BgPqCFt@3Uv4bxUn)ULxI}66jOk1bxpbj25$CrfZBcuFs*}pC|5@k_rC+(=Fe}<#oARc3e9HSAXHN zSrIDxC7*q@RXo248UOKdGx|i05CCKjI?{uDTgJ(pnup#>6gfw|ZR>4DBO@>VgMUAFF%$^j*oT zTwgyJ4OzU({Xr|qoh*hBIASz*yvJ~?j`!GH_aJSL9N=l|q}^Tr>I#BPVp4_WDVrkL zD@g{za%*OP1)<^Age6S%5Lnj3w#l zSE9PGjB7_DS3Y9y0tT}hKBG(*;ynOJ0W7t8U9F=@wsKXNH#1Cq$-^R0cP`j(QX^vg z`m{pws{BVUK@N%J1aX)0tDNd8#OGL}>m(=8o4m~Q3riZMnCrYRGG81_@law2!G`id zg}#8xX(Wr=tu&2G@>?k@zs@nG-UnJw>b>=nlBJL-8Cq@(w&s^s*e%(t#%=@T)_nt9I-fdV6wY8N z(L$jrjJ(gRujF&+>MIfJR<2(2ow%w-ex{8-K|I7SJ`XslEDJfyqCwv<h0n#ubf3@ZpOExe#aYr7D<=w!*}3*aFRuSfNkz1;oXYu>jaPE?^JnmZH?nuvIDq}f8*=shxz8IOH?s>Bh_Imw`yWl*N4dJ@=2uWDRFf1d46lm% z1W;dM(?F34&kgKR`k154D+}SmyMm)iLLMXefq^eUAjJoSzXiZ>E+A6rl7%0=<-1yqY&;NOZ;-hGic;QgQxmi*1ow_BJN)JuyzUv~HX*6YB@?(sW z2etCqS=i=|;F`%wtEA^weCVrjNd{)b$qeLTzlzgrEM=w!u#GBxn%`7d? z{)2WG5%OEEKM8QFZPy)qc9iU0eh$SrtEk=SuV4kA0%p_G2%Pp_#RJ6E4Vhd&kO zFq*hNO1vO@G#za4xwA&lPTLOS@o~pCpKmKwqfgYGuaPd_`3^Y{({KjN>-o1)N@gKS zNOZ!xFSf_WA%1{~z(*evc{Cp(WnYTshU&@Nes;~^Y1NElvHRLgW+GQ!1=G^>A3KdW zcX;vc%+(Lvs-`L^7N_Pkf6(mYY%yO^;B}{6}v6l4w+4Qj%VXfwJR^NT-{pieem`-9Pol$yEhW6>K4QR5SRx5 z{7q;|)z)MWMl1lj)2VevVSy<|HlV4z>h|xJ7ReKShRwQP0w|w>j>#l%@l$#>;g7&p zzpuo~Bm9glK%;ELOaCR)u{&KL;n?xTfpXHo&5!j@S1u0hJ(L~o>gd`m6CuSwGUeMx zcx+AGf4q@-w)rfA+M%BonOQuLJ1lnF8Cim@Xzg^sI2Jvikr(GHuN302>7LkhuLA6Y z86*3Or&+7>_tnY9SfM)}Qb3HEBN{9k?$fKS*{@g+#1UZcU7UXvGrvSVNC;9QFg<+w zx8E~5UVE<8fQz)0HZ7+9WwS1<9=5AgGkR{FJm2D*sH7c4OTYN){vC;+m5ZEo_SBEm zqz=|uzBeB7Knx2OEfm#XETw@d5Q3?Tl*Tku|f(Ujg;gMRqJ`}ERM6O`A&Wa z`S&SoU2Hi01{l;p;7-lpPCAw5K0wTwI<|-V7_l!#Y<1p3g6SM`S}!KPm@J@QnEGAp zq5Tkdfn_@o)rU8$!j`CcrxR)JDM%bPdTm4V_@%f*x+g~0J}NnoEWnPYyo8(&;qD^d z`q$T1U6VbS0E@gy6d>Uw5{>fSy~ue2CmDrC%FzoNbaJ&eGZQ(z=uw~j%n3qZm0qTi ze6hWH??q>X1FKAf3@yYWS^ISy8a7lZ3)fJf=JX{Z-fZMecEXqdT$t0rBE5 zDmklDQwZ^XaSF{jji@um(GMV%ZLBJHSHJ0Yymj3COC?)QCbVZRSO2r1>ho>KIRb$X z{KM;L>WvVcdLw|z))I&y*VKj1)GRcwBsSmn;6d)j=UWnGwMp>oPXWY3#BK3nNl1~; z55DK$UVHT47_NLZrBR}6@o^A)F)bFR7Y=TDF<4qHOiT)h9a27%*rW$+9-tfdV8v{O z_L7Tbz58_ZQ&RWO>IScKeqDaURfX;jg?^8nshN+{tPy0(o~%uLt0g{)97vb;aZ|W^ z^qNYr!}GA1ohQzxndi?!TQRqd zm<=y$6&BN<3G|W*h6i6dr2K`fTF?tgrV3Q)S}%?#4;!SyIpnyBi01$xw9RFYmpGh) zGhz}T6*2pzuXH~zc%-x;Eo8F76jHCYlL|NW-T7g1!7Vk@XufCvNK%$oteBT?()HFX zA+~Sv%0iXr5NeUXHZ3-W97O|=@Dg+DHvm*UwY-MsMan;Df{7DbMuc~J@T{H9Q(8q@ zO7!qcDFO5&p$Mn5R95sx5>{ySt`9g^=J&chIe01E#hL!YT3sa&2s|U!Cwk`Z0xM#7 zJRP4>2!_M+6a=MT6|9=tm)hV)Dz5L&b9?J^=XONVX8o z|Lkr0g8sPc_|qFtkyGG-Q0fT&@JJF0aRA`CbQ4Ys;8(Y8kQKk&=H%pS^V6L>0Sv7A zEJ)LOPJ%9wq!X&?9~Zau=!Ms_0ayCK?|c9khP>8+Kc`AM+)Q%lq&vH$w8__iI0jY8 zq~+XrI#JR$3T9DL&A0rP%I^+%>uV*A{)v_w8X&q*2{KEVB#a#u8+r?Fp%)Jzn{cML zAc2+LIMl+t)Px-LqemJ-z50EzHzqTbOMZ z_fa|DU%}Fy+FT!?_uu66RIjb4fE=TJu-R)9c96u19}GsWksTUWQgS7kvp1L@Iy5Gj zYZ9C(m$O(!uOlTSMW0&;NOB0Q&AJgXD85w(tz98lV0DpvxT){^e?fg|62+V;{eaJ9 z9h*YdhpmdEUZQkzsp|I2@bTENf-P-gm)uf!YA1ge57zZNWa^EC0-QQFBE!SgCt>>G zN@T*8F{&0S2U@5uibYkSEq;wS=@oz*p*+_Ced51M1cKH#h}dO29mK+OS&Z zK*EG8q6d~HuJTYSv4<(hRUN8VIlL>?g$z4Dd7a1Yt4zdg(BI!a1kch&vWcP%RRtv+ zYXr^a4}Pr^JUjk(;=2=e*jI&LZcwVbcf`MK6H62Qwp%-O?l9b(R4#uEEB%S_T@=$i z`Q^Nsek~n&^qpt%c6%hk`dJr&oT@5k<6AjqbR4zhG;NZrGeBT1$GLfI{B-fz#S=&$ zDIZ0?Y(2s3jYSfOJ9L2x8V9$9^OTTeG03hq>8+bm9^++`Ua~^@gQg0*&s?^FK)D}O zQeiGibu%VX{YmAhb()sR@|d|g0LOksXAZ=7Dd3Lg#JE4kvROF>ycH1ew&yjW3$&Cr z@!=lu1o)JD0I#;tHWYw?D^?!#>`xVofSKNY0{~j7HvGIX)4PO-Ksh6?VBk=8Ox`D&1fGMxMy&tpKQmmY&gUBVh6|A>0^?*J;Xhz6a!fEII zRggRr3TmelBUAI8Ow&G+8j?e zs^H)&Y;jE0nOpa>FG*dAdz$x7*XKq&tV#?X(TZx}4_5-+AQ|_?|E5PD~eb7f)(bZr=XrDj$ zmS|;upIH6&FjS+wPnDTV8V96_q?q%Vz|f?e`mdkNf)$pHi| zbzpDvz$tQx%gWN1JApfXUG|rm&)d7X^zvvU2Hj`n@5Od`&IdLwT%OX-$h*rN{G9Bv z5xm~5tv<)Nl_ttJmQ$Dp?BB{f8MAg%=A4b^6!s;z|D z=8&)A?0OaYMFd@x);7oZmZx)I>IUEBB9$Dr?3_I^$H|o4%MK6MKUEF}{_tpVUw8k@ z)0>uGAuKBWOF@3>$*lVy$QOnmSwrWCANi*(XvSvxp2l6zkzbNd@rzZ#)RgXEeQo*Y zQ`={(_AT&Tl{^G1!jsMsXLxj5*d-{WD@>oB+q&a-7KbmLpe5!k1#@@TYvi%)m$Fpd z#2QSv^6Ifg`W=A)-Mw?jix|O!2q*;Nl@;R(UaItYQ4-?qRRt@E0kyihbAWR1zT zstP&fZNZ-JO@Ra0Xa_~wZz9YED3u5qC5s|x%0|7Id#iD=WSOVfCDCsZ7j~F~apa*r zxsw3XsWb>?w>d^Z}Nif1;jQWkn{6vLOF=vn+ds)$&)bWQYG`B{| ztA5=VMTERunS<iFnK^fU*k}!^-AGxp`B5sT(Ll2MHTL>%*};5%{dkLq3*vU;i6FpueFGo8AK0V; zcux8nlU{rmbC&r1%qY3`x(if%!WoIVuxNS7|neN5orbBGKid$MH^v4q; z-f1%1MoeQ!d1yNkcW!XJ(Kq_2zv|u3k0@X2g@`t-ObVlutFZc=u6fOo4aLNH=9nyd zkBnX*ZH1kU=qEJ=f0Yv14v#zf<$D2u&R!GXa2n4U5GMASbI+;lvew(|?mGwCp)ZC+ zJ5TaAu6)6e(j205t+4?}fmAm|`p=iEAiL~`L~$2#!u|SGWZLC4i^YO>UEX7@nS_80 z9I4?HTA{}nB4^WCCXA=5noDdj1wUiTpxK_|!cJ$cU1sMXx3#^ZwcH_}=rw7S_NKJJ z_?s_%^tm4w_0^pNi%w+{8p3>y%6>g{a8uZgjlOx7$XsM8+|MJY2~zDGQ3IIu1IHY) zwQDYx#A`W9Ek)l2q0S0b#G(h@&t~k#?*_aT>*s)VV`oDanfNCs16n-9sO33^#q3=H z{_W9e#EDv9oK5&ZIao;kUx=D9o zqKK<50bnP4{h4kMmN#mP^|>F81vb>oon{v|>QCd<@j%qcWfV8G2oNRR=HEV0{hIDE z={1)i7=AFXqcU`gxt>rEt$*WA?kV0W+Q;~9gJo>i_ciEin?5MfgUMJSV&t3PUH%*r zB@RaH&EtZDwqF*UVz*?%u}g_9BaBR~`-fXVIv;_j;7o2dqwdn|72!1GhNkanD=dA} zp5B+psluz)c5LbYYM8fFP`~X%w4p#cUdk=$48?wOwQUV#ZhiW&E`mQ2ZBfQqyb1D=eDrdxYS% z>or(HeTXpuI4Qf?J|AR&`(Vx14rH3dTo(t3AX`Y5ft{Lt?pTd0F-06f&uXqAmiD2u zE{-+E0S>n;4e!4Wqt=7KPdJmlEyG-DfGDV%NVJFs#>yJY&AP^PNKJzrk=^?&79)$& zZ&T#NGc`zjVPrzal3#=M49$ZGCZAY${SvV*3XeGEL4DRz?)^0~B6;J1_iM2}b`fLb zg#?hdxovk}e@(w@Ekm@%Xe~n$_3td2woP{M`<#d7cibtO!S2ko z&4fScX^Ezu`u!Rd1f@9Z3AtrbR?XecsjJWCfffSt9p^zyNFRq63;d(V+`D^>Rcq1b z#-GkewLz(0`TL9jP2GwB(h_g?CWaWh9HyWBCWMrzH*PVp@^EW@XdKXy>ygv+ADpMX z4-HlD(vf8>TMZz7ee<>o^BF^@%H`Rx?=GnI?;@1<^kSGIdow+}T<~d{8fFsJrSfl1h+ z+0^=z+YlXg!jtmDk}qi}ztF{8VmJbrR6EwFY2+|{D3D$nc!z?Lgd@g+TU?7S&KvF0 zyc<7D5+GxMxiDrQc`;{nsW6qAM-f`bXD>vGmXdFDj9anNd)$ztPRcBBn-#qk!v9!K zk8R~K!zEi^-}!tXIns;{09OH5P18)DIIMm*RPvDtdtVL_G+m?%4wFaNY28eehF_mq zpm8Mr$v+uajR%rUp*U?&h9dfS4_p{t+j6Z<4fK`~K_QXb5gaPo!O45?{(BN~ULE!> zWyDIZj&BN`wjb%o8PffL*k>IJ3$l-?!UD^vU&_XniBzJ-u#eDJQjP74WrXN1d{HAo z9K=v*?y1=_x&F)x?9*J%EYz@2gW*FqZts}B5>Vtd+buuamn!E8qL}By6LL4A$)TTO zRTJ_lU=dR-Rd#=6BZ*PdT6^?GX_;j(su|eMcAZ&LV?KOsW#e}m`M~OZJ(7#fDE6mI z-LsBCZpj{qcXqp6OR3-8ikNgxMW~PKs7}o9GDt(WHZvNTk+oa0-JUT0DAi)a@BB7=oz&8V+s$}xn#(z>4f02kbNvY5Om0He#Lo&KQrNx65{3*jPlC$2Y~%YmKy!n9|#Kq(!AX)2-dN?JZJ{U4|zW1 znK4^KYjP{Y!e&b#h47U-5c^h(ot!WPEN5BvXDWE|T95EzpG$qS5*!>|7w-FD& zoNw|padOA=h*|*381?dc?>ILHB#S;B^q#6Sc+y0T-SJsJQ{!GXTJ zni%?h0;&Mcb(sGsy01uc6~htou)&stz@d);b^m|uU1?L3R}@Y{5C{|yunmh5nX=e| zc4)PtQX;iw9ncSK;|S>pbSRZkO9BebLM+T6n}Se^4WY^)j;7d@vXz3i!xq+I)s`Zg zh>3tfV1R`5+&3njary)L;r@W!H@WxR^PcB<&T~>UdSAQE&#dYn4NEv+ZKz9}`TC+f z)O%%UFx1Dp&*Az^qjXh+FkmQGS$N9mAsAUvTQ<8n5x9@nSLvWmARl_}@R~GUgIpNU z5hO=Ion9icnL=(z?&kB70S#|!x!jL2X$NLpAVya!+K6!+Q}>ZkK;HzsQPkN1{IeP(uB zM5R(s8e2v($TI%ke5q8st;0?QSJn_zFa2D!4gz|M{5#JKpVO(p^{*l27hy%w7#mSZ zfiUpkf!g}1=2=8bU~d802i!Q1kg^QAn1yMIeNq%py&^TAN4ntToBQBV@sJxlg@<=) zkK%&hPD{h$Z<~zd{-cPDZ<4s=FZSFP&z1AIjFB`TDbEYvIN)!0R%rj1qd_WRJ!e&P zzi3wcB(qY-aMx!`nZG{m7NEfbUe&`sP2Fc)YmUHa&z|EzXv63S822CnRdddTDWV~p zT5bEr+_)LFEA*FL(?({BsLlP=R5lQqYq!JP+({;J2G-b&eI2gCnBwmRP8B?|p7-3vkxf=)B$ zPtLpOp+UMKwRJ5`rUxLI6bdV~u~QE&V#wqr_22)Zj|x7CE=tnFlhz0(DZQmm|I1l! zj-<@0GCi(X+F?nWSoF}BQ26n9^L5oE4n2JF2$~QIYn*m)($!7Z#jhOv!hKbL2ynT9 zi>+78H!d}Cp{-LVFwDRpODA1nh=~EVPS>QI>CoKn`@XLGy081Z&hvG? z&evTxk2_eeUbT4@1Oi!YbM(+j2t*QWN7IxD50HlPb$_BrD@I7tg zXKxSL0k&5{q{P-iBt)lxKM=9akQKkTArKcam49!0if#M*9MGrOE{Le-4dBNwA>jAL zKubt8_yHd=1JPgq`aT{I;d?po94a_e^LS{mCd_b;;a+fDGDY%tAM3$(**&p;u76Mb zU;Rl!?n(T!4bFoY8yWA08|{J{@6|Ng2j8_1ZnOvT(j^-X4#|WZbqj+)47Ef*ViAuJ z;Sh+#>wr_IQK#+g;1_}e4bS-mpZ7J42n-SR1Tl|*gUvu+)H%(Fz{^2l@Cb{ozt4b! zZP9M%R?Xk1pe|W#J#Bwn^I&kOujXDuBSWLDma8;1HO)hP{NN`K9r?RC_|0OgKMEBB zheFY4v>_U17#xa(?lLtsg&G+{jg5DLGj@hyf>7rob_RuM{ng38`Z?qqb|Ew%1Qie* zq$%q6-1*?ED2uIIMHl+_=dW?10{s5DQc&36w*_txDmnt)WoQKbZ_Ruo0{)j~q9gxn zGcZU0zODc3ViD&;{%Z7lGX5H)xo8A%x0AkM!I!U!?!^*j^!o$dCHgY|J;ML%_&@je zSKHJ7FKz$x`2TGCPgCch0Fh^$It*@@JxP>cGurc{&U}doG+Rv_-}J| zI}{M;8x$tG!(G56&7uGG(0^@u`hRcwABX;I&N9=K?J{~zc6kA4n@`hp(*OE15d(tq~yA7}ozK7vC}1qWZY{BLXf zuTlJN|MyvccVG?`S@wTe`d`cF_g=7$ELVY+|5}rltD3f+UkQO&L2M4~KNTT1+JnBg z=kU3`A@+7jU)LUL5I?E4#$eO>)UCH}ZTREXW|a@86(!cX-#uOM^2i0NmlwojH8-w* z=Hix+s1=UD5GI7o?aj2Z7{+3>gsM%4z-$X7F?Kk_h zZfPvI^rwcHxa7(;npUw8G4a5a{QW8or;h!zM^aE13%RHE_i=Ed|4}Q5|KqX^8Q}E4 z+JWP%myiFRzvcN^wvXlYvFsm~{o`_cSdNd&_~19LTE<7q_;48?FXQ9C0>X0sw46UK z=g-Uh!7_ic%pWcDXChp*jE|S`@iIPM#>dO}SVWjC=LgIA!E%1^|7U(c`Lbq>=J98g zU>vW@OSsrs|A<$Q;MXJO<2Tf-oj?eu5p@f#!8c0#F1l8KcP@?m(svm5hw_)8Hh|9{30GVQ390$vZqGM=bhSj5G zWLuj`9z{Z-hGoI|hNeu-5ks4|m*SR27}FzW&Wq8G(LWW@U4fNkiOf3Te19^o(X4k@ zNdAZi3!b1JIqa7kxHp8lL%8j%uwDVd@7EAUu&#Q}Rdw3izWLS~)Um*ZcP&(uV#gNf zH~~Foww&fw(^)${(`0QKdia4ii}T$E3{Gl=WUQ3_*;MznU%lg3(@a@;^ynd&#EM@3 z2e)T?qc|I?rgK;djcVxbmyPOXAz};u411HFtDR?EhxZ>U*&?QywJT(`+4!3a#ZvO8 zmo#+FV`s~WWBNm@gCpwW6)Yoavwm757Tb7Cp@1mh77<+2{63MG4k8zpT}@eKxk%(y zz_*Br!(;}?90#+p-TVygYc0^a==p3N+#jZ2^Tve&ln_;w)g{FcPnLh|s z&bpM$K!nJxeO4l0Pi#HUCZ_W+Fbfms>bU8w^H9)_+41Eu!=7(wls4 z_FdS_-(7xHG-C8da}P|JHU#zatH_e}Z1G5<%2T&iUWp!H^Dcu{!x}EJS0T-=z!lWMeOfKI|Cl=%$FzY*1k!j*awBC zBbFG5+V&ECVJZ!M?-eUbuH3%V-6xOU$9|Z;I9`gx`85|l$ooi` zuDT-&tueY3OGCYu6Kl4**H>)JkeTLhVW=z~vw%D69`(t2Z&!4-Lb|Acrwbc!ZhUqw z+o|JBYX;Zegg#Sx#$D+MxK^{IWbAKyvRn1AE(4O%`-QSJgCbA2mzX+?*cCMW%umlU zo}W{|&DWd%nSBA$Br|;@E{@4hM=Yean%O)9LTVU-~vy zzB$aY9tmZ=?h499Z$P#c(KKo&ZwGr=pG9TTL@T#=hnSeRvvWe?mKR&a+0MaL>_Um! zz8YDBV98up$o!Ymm<$;RcBzKEZphv3`S5;)$m6}#FKDVAZ+iXth>f92hk?Y-vcjZxPA_XRBCR4g=RGEq3=CEODd5_&>p5`RBt=T$2q&Mwam zfX8h0p~~lFOWfoJk~pxo7Vu*DO9ZX+H{0&1->1aYuITKbLyyvKS5PAbyJtSl0nfqmK zhCNqAw|J57n5hl_5i7B{vG-&ApEa@Ji_FdYjA3ii4ob;4CDM&0A~e^pj_$Pn?Y+G1 zWmiHjRV{j$#|5k~hV%VQs=ra%@DeF_h=jD=!igh2+7NC}q_7;pEAQ$a+6x@ob2EBo zrF0(f6^uHId(+wv2uf#BCzwuuwu#}_ro}1I9ov;SNLTlQZgj>e)oTXBA31z zX~y|XEP3=cm`yk^i%*qz1y%d%`U{w3JQ=elwR6sMp*?e|lf=sBr4#u+KXT1RU(}1d zdYnY`A<YXX+#Mng&S$Sa&ENcdbntrWs`|2xA5C1eJ&v`Fi3e*1WCQoz3#VL`c;!0ND zYBJs9;gus%Cq56aDnrpJ)#8PuW;-pK_`Q$o2-3`{%u3bgPNas7?%bWH@skMU6?fI6 zS;~do_>Xe++qXJy2?C+q5%YSxLd)Z#_2ydl1k)glTi5Zj5_aIfOq3Is40yZ`hwL$- z3C~j~k2~6bwNFiRlCD$m;s~&dME)3-=)DTBT z07ekGNfaY1T8%yBH%1_rvOWlBK-1`)E+J<9geYcBH&G=V>PVcm`&q8K;W3;jCU3~3 z&h%wD9~Rv3%T{ynTw+jB7?F9aYk@^1b{hDsJ1p6rtU5Cfqq2eXjPz3}GQcz*n5gUy zZJF>}+&{}*{l^)uwb~22IMkfeC9kDluCSq<(8Qy;j#0K5gWpcMJZTPlddlVE#oWNx zHd-GKynOQBSMamg%Wsgm7tZEUBbj@xR#4ob!v{QTBAO^e9wK{K(RN7U@4B3dtb zrmK|5LUlV+W~_pvy1ESc{>>KbJ4N)}At8QSm_MXUHtqACf580^nyRrhZ0%?<^Vx3i zpYCatg38;BI=KVt@3dn^K|qQct8iU+qUuVRjrhV$99war$9!_wLuvq$&AY~7 zEbmL| zXC*u4#~|U+02awluC-R(Wf|IWn_G~-Kt+bC!CVzTfye|%~$LGqt-r`2a3TMy>WaJg6I7Jtj-gwIdm&(Y~)3uQDeTNk6FD=@OYfxh%^$(#=5G;7VxUtNzk~ zxFaP-_WHpk*)yvc$@Jn}56?4gjQrVG7PDV3@V)Gf&mCEFRiKx-%i&4lRc zrN^oiqYekTdY8BEZTT@2HwA@9`2DC>F77s&*{fFAokaDYpX?0lG>uwZtj6Ez&N}%4 zIo#Ex1hXvSFh(_6uY)zyXRVx_%4<}J{vnYJAU26BXhfub%DA2^kKj*;5Opwi#68{f zgT}YhWUJnw$x7haQqPpnKG%z}a8Cb@`&zG_shWKzcODgh8#2GClZ+j`K;#eT>3JJ0 ztWrE-ym<^NoPr7my#zX`-T_Me?vg8agMhZ%xo1^Zm6hLWzxNZd2MS*nf`v?>ShJVr zcJ>k5SZ?8AKgn#6z@&-E5%j3T*HKAu#2FM)D(YdKWysfpXIt!}O%zqsr?VSt08aM` z)y41Ydxr=!+gq3B`6JPjThRT|Ga}ZaeAUBXg7p66i1%8(z?1u&nJxC>n8%=K=FdK^ z|AQvs_Z@!>z&PPtf+hDpX$g__BWo5fyx%Ap**zemCn1FogRe~q8BIe z2T9lNuTB6fJ>n|jnoGrfJJ2w-GTf`{mUKxGaSXk{cPWGv5Fu_p5PE)A0H8ZKeYJ zh~wcTNd3Q#}|<$=SFD(t(;GMTo;>J*IOysW~?^cwZYA z$#~y`?R)ThAOqwm=HvKo6#WH{Mr^rgzVIq+$VzHAQf{tTOj8Mw5*WJLj8hX<{zKa&=LiY(?UD!dc8YiOBPLtiE9zqeT zL%bajwvGwSPY#=(3rb-Q8lv(1cO0Q$q)srx^@i7dt%S+)6;MrO?-s$b>jgByC~oYM zSM>yWo&z1*!DbKbvC|`VwbB{Y(E(}bXwSCab`?~UsXYQWpr&A0o-?iB{ z)02c$+u~5`8uOU8F}W}e9QM4{(ooy6eTn%#KVuzZrqVHIjIX?&Br2~WJ+g~1-UBHfq5mDocP6;@j+ol0~NkR8!t@Rv#*?u3bc}FIY!iX0)t8d73OxO3E zyydE`J#gY6^GO*ZS^VT${C&&Fx3h;*r3F+zk1W)dj-dg_|5+PkiWK3|#XqeG;>u6b zT=Ql#c4F`Jo#V>ZWmnH&M+yei!wc2sp3kN?u4N~3dssAIIp%iao3w7#R@aTaRCSHa zLAulIBvC?z^f8CWBwR?JdhRh1R6Q0xkZKLkHk!b$uwMu&a3yENIr2u!=~mW8oUy8j zce2tYpR5_OK1N?`J*U#lhyTQ|Y{Ocd9x;hewk!V>4qGE4_el(R#(0%O9ryJ*boK$} zWGag3FXF4&$szI2A#65;5hM$l$v_DcDGvd zexR0$gf2I_eYTyDl3ji3iuzEVhydtt#C*3~XqDr{?Xj8Gc;}Qqbr&^^jMVWkKByk# z)8tYTu6fb{lPa%eb#E_oV3I=i8FSB%>gtv$h_F9JRUPwjYP#TSLnstXUbyP#vC+NdlX^m*`L@wksnFu=GAKARUCJvTOMLhjuBD@z@#dzQdj$Z+Sa;)${Y zAHo(_VH+RSr9q4adH&_6k@=fY`8yKReZIBQlA8%`=`Vwo)476a&p)PqkjLbjXc0+d z$8yA=TFk~>f}kl!=E}!O!jEN1^-i!xmK#c|KUK;w!29WN0W#bw4cS{>&Ch8*iI9Na zY3eVnVVyfMW5uPSP> zG`hT9?Sa9O-GjF*7TW9(!!`Le2fdpcR}#F4U-5d{_~czL8N?Q)qkAmaHjd4QdIyO+ z^pvxo5%e4wd}P`hGb0O&N@4jFed5+ZhLSd2zbq!ms~}%seg{B}pt$=wAAT8<+SmGf zXwLPN!;$0m;c&p@xx8~jwdNTx@Np&tWE0J#yApNBqxnEVSoHqjKUyi-j}AlI#jxUTiZF$Xi3X1bn}V{CU@b-mil>jQ(bGnip=gv4UTH? zfZn>p?T=JA?Cz$c50k`y@wb^dwcoAeFm5i9*tu61%{0sg@|RSlPZ!lNeMgtTx~G&`>1(L4gmv6yT0Gi zbnq_Ow2_hbeisNPJH1yP@0odql8jst6cVp8`lkO5R-*_xji*QZwaFg}>YBS68(MDO zU&Ny!ULve#YjZEpZ>b+PCne`guf<}ERnyZ7YZMXR=a7+6%KKCMl?1kxp*cqx=%JfS zZk4Z~miDS_I(vn2`PNEG7Hkgvv$9=I%4O}C-Sa_ZvT%u`_dYYKdCdHH)mUJ#`#8QO zTxx!>j1{^=Z;REvV_>N?Uv%CEmWmJcO3;J?_c9G+*?oPh*^6>=pB#xXZEFwpKclX* z5OGZN^w0|tId6xXlIq9ZZ*G%*n6#Ze#fr~#t>BN@G z=Z$eMXWW?6KNF$j^WQn>)dp?p@=j9c91-<H#aNYDvKI#c}N6L&SC zZ?i*#7VUJJttw1uE<|>bnjJuqYmB75?N6mAY6Je1Wc>y(O2{O8*0<>!52k5m!QR=& zb!E6c6bGReRAt;ekV#Wznrz2#pT@;0DGeN3JhF9=b&|>-;q_d7fxy1+<~SB1luRQ} zPXgk9?!BU;YWAio`I4s{#G@SC4;{4awyTjs2{G^aYs$yw05@9!)gE|qsyDrMAYFlv zI$)c>bAXCPSWKJ+4ECo#Vj-Un?9|ta3ySOBOg|V)M#ns&FMb0_mx#q>-CjT$7z>Ix z9e(g`_u$&AbEUJRt&=udm=&CNbZe+e@1@dOc=a*T&HNG7HyQa|!tLyL3>udH_<=?` zimOO8p6E}rHT>vwS2ElDUQ>9a@EZ}gQHVDa%wLlsPMGA(=C*y3JrA5AsSd8J8rNd0x@ z=CA4fFmyQ#^6&!ogEDfU^$mgJ(~9MiE0U(d*s_Jdt|*Ssz6TyB;gjrU*4Y>AVU|r3 zASwunhE?0tmTFSHKes8I^(@TvNsG^#P$ZY_j@z9^a-C~0(PlPZAjZi+7ND0Ff`wz% zuWLr3@rjbLmKr>n-G(5A?uD$cyBa6ApLTHz7D}!KkSmN?1%vo3yh_v|+)X`R0cQcsX8F+ccaZEQ1UC&wLNIkJfiQuny9`Pj8kdssOEB z1KBMhZZGRod$;C#B^2@yox7Gw&>u#AZuQMN+YwOtQ8(A1>N1X{1XJH$xzns@+HYM* zp*UJ9)+e4EAh(cl*AOZ;%qQW)3$-#bj)#DUWR(n$IFsK@pQy#qlRG zKX*08;DSDl*G$vOX?KV9pDaCd(riPWyQt=MF-S;~vLH#UTyuH^ zP~K-s_jfvalf*F+ofNzGpe%H^<0f;{?Iz+^(?wWi7lSoE@*W-ITLWV@qj~QRvWnvm zrE=REk=b9HydRELGGorO?&1oW;lG{~SEt@YQa;Z59; z8Yrf3*X${+6kn_9($ak-o4Qae_u3cJp)cgEUmas9n>PUOblWAbMc2qAj%!`{%`cah zI=67UW^4o$=#G%2aFH9_()B8P%}O=XS1sO{M5!ZQ!kDdAV4dDvS_peJ{t1vnCu2}= zS18HVXc-{&OKbQ)C&K->>QSRF6I4Tw`fXCsAPpCh8gg@&-1003o4ZoOmfSLT+wS&5 zGCwCCrn6(=k}f548pKz9{AUu!wu@@Tu@EElNj>csA0&+@bT6DRuB@1jIR{?^BuDs> znIzkcwG+ho_j+&z+Aela(5tp1{qe}2ChZ>v-x(=Z797?6lY)hFpIo^*Iu5BIsW187O;l`8r*V5N$z6b{e^$K@%NggFyvOTOdkj;H+ z7S+xUN4_O?A5(SQrdpd-Ib9h>9k1q)^Dws;f@l+fbgm8?Ky@&wiZ>@OVPU3QV6nW& z6QqXxO4%Y5Aq)NjRyV~hLSS=xaaCyL_jRD+;j_tE;M@PfgnVUITOw%)EsqjN8B(mj ztdyg_x0l9x$9^(iTgD%L_sy)uyr@)B^Z4yp$Z>n;9V_D=$#(y8sciP<%9U}<8$f+0 z*TnNSX9N^#nANvKS!Rw9xTEnBt@zkoOBBoT;T*AdVIRaN^g!V@E@45}g!FNpl-A(Q zMp5RNWbn&u%pcd@0N^A_$YbJ?EoPf!lNkM8q;Glku0I)H3{nyfU}2+S-!R#)Lm!}~ zypC^tai(=&$!$p7%r@_J6Or%d7?LiDR^#r$l|A8D*K~Kmg301;lc92mKeH?sZ=2FK zhGpRajWj2UH>|hpPd@8E%@+pr)@L03gz*XYTMYit$HjiHL_BJlCz6X+Ex4#}AMg_b zGW~ZvycaLuTrF!%_Ttph%O{s)ZY^{syeKeXr?XrQ&vG-GV#FmoZh->1%f;ra>rU^F zOA;z?+L_)ZbmPZxr`;6v9t6RmzPs=GF9h#x?csXZu^fGno^Q=!Dt-Z}MA*V$7Fr#j z_|Ya*STW5Y6O@f7LEd6few!x`60ho>u^_;Vn`z9xNEqDJu4B?w#$BK5IW7N3* znKNg+B*uBocX^7R>3|f9Q#BjJs2+*6dJeGX^F7ZzB>mPLwGQ_tUmn-;TP^cI09vVh$)i0(YN!QuA~wwk;g2wY|`8 zM2%kZ`Z7uzWf$i8*K-MKnQNU|pnfqrOR%%A#Ku-xHsjVyTpHDdpyaQ*$GlZcQ-oyV z)RBP3b^*YxBgffLyj;fTb1UT*-fgA-t8rG5<;;`G)$=zs-mvY1s|F6w={aQV+N+$g z`(+rHHNW-{+!3Gm31b)T*GC{ z<4eeNYwPHk#7pK04HrumZt0f{=SDE*GPchlsh(cPiBoc-YF3woxliFN- zl8Hva72P)I9h`7PEO95-@NdSKN3&`?FJ&xLW0hd}*HShtLaT-gZeH3X#3w4#&!}@$ zdz)J4Yi8dBM;;{S?m7%~8Z3|TrQ{Or6FbgQ`kqa(b2c{h7fZI0_ojcyq>|P-#;yCT zM6L_Ax2*!n>X5!BgxbmYwzAkcuGl%P!HSppyarHS<-HEZRvY`+2Iu_V`Yq7nRye}z zzEr|4%)wSY=A3UwHn-Fn@9YD0;}P?N|nuVm_mHUd0=52C6zI;>n*bwKuRTEO2b}K|# zvZKzbgE^Y}E*BJ2%<^q2vCsOEI(bRS#0$!y)wRL1J~vFj3a(V-_B^ixj8N7dP;k#S zXKvb*LNJw?-uz>C)r;=AR@6n=J+w3Sm&WIllvN@&Wn|_B?3~_tiiRgmOG-!CJOn| zi21i5rT+8yP}3~7>e`Zc;KW#dk7X(15U(uc4jq*oq_zb~UQP8V~InS+Or)q;dgM zfAeB!s$|D?EBG5_ICIY78K^``W7C6R?=irMji{8>y|ShI4&-6`=V9ATD;><}<{q~| z=^=bbNxEvjo-Qrr(6w;F^Z7(hXUzx6Sckmka6hRDIYmyax8NQrccSQOT^>ejT z8Kry>9LyM%s&=tf@xU*{XOuL76mQD2iyXxw6@5j!@3IY_-T9rtvvrCN$?fXay`*?m z{VQ6>bf3gksr(D5{J>W*Shn=gyx;l3nhs~G=IzRw!JS2Uq0N+r`Nz6(2A8V)@0iVI zY;$t+1jS_JANEW!r*Zo9vj@qRi$HXyF<*M2*~v5s688%PwqMUqi^)(mUo3D0trORo zYB>UXDL8%c`Qc>x83AUnqaeXvHRYoOUc%^QSPlZ#ioUm={pe8(neWSE*ow&8xdNry z$H99Wc4t+|-(kCxj1f!1_#eTzYWb4;ImD(hp|r*Der$62NvZ+j>hS%9p)Fl$iFwj1 z@deXBCKv3j-je1xo(S??Jx$1G*P>&GyxHQUC1L^uhDB)Mj9rR24Ye@B#h*V`@BONI zFik45K(R4dpnAIH@$~#8jY|ibmg+cS3$Z&V^S5V@yGy=baPJ?uV|` zH1YyW3pt;5#5+*dN0nbvJq{0f7J6>7>@+fS)Z-0qA*wy_805{+o>J4LC&o#7UdqEY z^2CcQ0@~r%u$ivyJplt~kJ^J6g$Y@uK$Ats{Zu4o4+?oP=VwMWLYm4GMWVbg<#KXl zb9K9GTIXeiU@}>w>7(){5!G9$NJLA}xv@Ghn>&C@tXq6mm%|G!&=35{2Vxy-FgA;y zya4fMCp6L?`hIEdZ08L{N!e&U$$Gn}}=%|Xv%PH>%ew@!5-a7_l zLB^L9ETINbbJfhHSZVVg4M1a)jN|uD$|RVCW??PJEdikDJ>GKSO4I4-Bsby{BHZKa zFNA$g4-Rkw;g!{^&CfSK$PndJ$_96@#+4l#EzrfppLR*Zetw`n^SQ|K%X{WwlB!j# z)%K8zDTOYeglT#PX_v{5NjkQ1gqlXsAIOx)-p~e`HH=d}4yYkppLc9m@QgSeBjP2! zs$OF&D;bAvnI?1o#K|NiShwmDl^?^V?=!W%iAGA3wnPH zZJ~;amo*dm`p0&fyT5ba>fxEa^Wo0-FB2effR4x!6BBn`*VGaVOA>Gyj&<$5`3U3F z+Y}*-F7jgw?Q+#&0-&?7fbKoU3T-PL?EN*dDgQ7%C5QnMjTFHm#s8MaM`dK*(+q<>6*D<^A}J(vu5VBJR!d+jM9dJL{Y+;sN`U&KLlkl*RS1t-3Jxl- z7rbe-FibQU9KmR&ED$gaZ$o~8S1(59W=IAkt9x_TUdZq7na-c12FpNpzS8&Aejy1=RK}lt;GSO`7Z%E#1k4Jj8g_PLOuGHv zkLL-4`J`YD5Qwm)20ejj(*bJkYH9TTd>d>aRu0*0v{Ym<7Es?n59|chE(?u$iuBa% zD=ys_1E2F}w9$LM$xVH%|Hj#+uqtFIyJ0=M73w zu#}n{?b9O}E7Cr?{qY0}NxGGyj_cj$PDTMa!iU}SUo_H%Z;Vy;(f=^^N490*uA@J9 z5;Ju0eMI2`5ytZAfWkqL2nWKv@Ly+aWbE`)&T*9yMyMiD@cUcHf<(Z!t$EZg+LJ$F>BI)>nPau&I zaCzz5rh-wa*i!%OZSOn;RHOW;o)J&wy4@kU7(>M8?>6h!!Ri!MAsSVw_m-MVvtmahOie!st4BLs?bi-Mta_Nx?OI~ z8hH7~>4A$2tp<8jeu_y=-)aR+)4?Mx^L|w!%>L`|;|GO5&-kEQC08ban78XXsHy-O zm=Mn$RT>LJuIEi9blb(}yYnPC!)k;r0X{jOj9clq?|e+V7aYmR1>#d)a^@SnAdz70 z7ZR82%)|OML{qlpwUTA)SIL12z(~GxDGabZi zeD%r!L(cfP1)oBAj}G7m34t-E#RYNq>^ZKU16s zAE1`XwObwlRhEJB=Z_f@#;kqms_k(tXS@2!pZ7b$Ml^5gC~ONYg)=r}>b|&BiV0lu zSp128Q?J&3fCe8a-W}`YV5*Zy^?NtB`%zOp-cIx5zJh^-ZUOP=Q~Zrybed0;*gb{D zn18s}Y~w@B(N4;4>;Ir`wOR5>AX_5q^}GnGy>kB&$5JC~`lMp>Q%D(tc%G?{M6 zs{z_m^S*0};>Jd=s?1&Cr%CzUJ#=23u@~+&T7+6RoB8RuD}PJ}-scO&W&_xl`%a%Hd9BE>tbVr*-YduxNdK zYWPNk;t5a!c3mA^JMA96PV;y=NEw_0cgTu~>l>!@eocyHVk60kFUAKSs- zvRCOz8PseyTgrGO%<@RZu?u`PV+8iXv)1eWwEF2*Sel1ww;^m6$; z@y|yO>+6b%Pp@qvbawaC>lX4#Mhw_e;HA#I11X6(gj*KXH9 zFORKn$XKTdsCseAtrzg^e~DDR#NCc<;{aN8rdadUx)I;2f=mJcTW7?=ATEJl-WZ8q zaCp4`hd#g>H}&$rRu1*%M~&lMFH&FT;l=V+xbtERZ7clN-ZvHGIjz-vu~EbtoxW6p z7YS=SlB_nSv_3ThLR2WcbRXwUQ>Y-!V}$*Otl@k&%d&I9-R2oAd(o^e8+~lx`Xz$Kx8D*y zaVw`U&tHh-$1h4&l=)xwOz)n1hAH)6uI7eh(14~6o9jZ0R3e zfcNi`4rqINr#R+dcX(G|CwdQJiR}nzc@yNIfGknSBtAlbV zkGTjQq&-mI+dEgjTc-)^yJ}-oOaJWmOQmsH2ZG3B>RG?gOY z&es7SqKf*oU*w+`X(6{SdGzl}#!!!_1nv$|7u7xPfVzjnS^eJ+w^$?29VpU7n71VI z%d^=feuI)1A7Glf_cAuB6%hd|&7Y2fuDxp3R```}zS5_T=@ZE(ny}DfM zH9SD?QC6U0+JIjE;)XZBxWa=@`f(st5LsWw#`*Rv@?E)!*5H`19|>i4tiYhKTv6GX zT^juQn_nk+w!$oHlo}|Ysy00lm1_Hw5rX1!TJv|IL{V#fg28GOc)Q}tk4>VX-vvWI zbyeR$WS3y*a+3%U7V`lXmP2=hUBT#CefZ@=$`|G6yELKWtvd>A#OoHnl(uu~03`!7 zpWCC8wUnVU5HfR3jS0Gh(5?&oB{0`abudB}^I zcq?5uxHFio@`Q=;Rj!1oSEVP+!YnH3pM)ZM$T8|WqpS@RY9_PB-wC{_SKgbE^V>e- zt;U(i*W;p>iJYWPf4}ZIa;9^!x{g=P6_mex1)JX}CSC!!`HGI>-;+hTxW4I9Wce9k zs2h>>icp%-_Ho&}4D6BNYH zBWvGNwaE;^I2Y)nE&&CYM#KW8xag|6nE1cn?Gc&a_ccw&1$G12eidf6b_%;D?4}rd zkqM|$I6(Lc`+mgQsd=ibDMt@tkH79HRkro`;v9IP=1V!5bvp?4AP|rZS!csPZWeZ3F1PPh zdGWLJmo)G?2Y~pnVDpAQe$O}ADvbi(|C+Xi*6!v=2tbK2ODDdJ&~R}27Bl*;*b=@i z3P_A`Ad7S8L@=U7uhfj3T&=fgD>c%KiR6XNS@n$c|F)2CcuFfcNr%O+300_^<1TiD!V9B{E!J{QSM5eqaB; zj0LM>L=Nu=29hepR>ix9nR) zu6Wtyf+)EhiA2$JIY5hE>jj`{8MiFMIp7_{L>oLBz4dEfWj>--!i*7w@V+_#e*djYfr~wqXZ-UEm7We^X Date: Fri, 26 Jan 2024 09:54:51 -0500 Subject: [PATCH 7/8] little more cleanup --- src/dailyai/conversation_wrappers.py | 6 +-- src/dailyai/queue_aggregators.py | 48 +++++++++++--------- src/samples/foundational/07-interruptible.py | 1 + 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/dailyai/conversation_wrappers.py b/src/dailyai/conversation_wrappers.py index bc7dd902d..bb83f1272 100644 --- a/src/dailyai/conversation_wrappers.py +++ b/src/dailyai/conversation_wrappers.py @@ -2,7 +2,7 @@ import asyncio import copy import functools from typing import AsyncGenerator, Awaitable, Callable -from dailyai.queue_aggregators import LLMContextAggregator +from dailyai.queue_aggregators import LLMAssistantContextAggregator, LLMContextAggregator, LLMUserContextAggregator from dailyai.queue_frame import EndStreamQueueFrame, QueueFrame, TranscriptionQueueFrame @@ -17,8 +17,8 @@ class InterruptibleConversationWrapper: interrupt: Callable[[], None], my_participant_id: str | None, llm_messages: list[dict[str, str]], - llm_context_aggregator_in=LLMContextAggregator, - llm_context_aggregator_out=LLMContextAggregator, + llm_context_aggregator_in=LLMUserContextAggregator, + llm_context_aggregator_out=LLMAssistantContextAggregator, delay_before_speech_seconds: float = 1.0, ): self._frame_generator: Callable[[], AsyncGenerator[QueueFrame, None]] = frame_generator diff --git a/src/dailyai/queue_aggregators.py b/src/dailyai/queue_aggregators.py index 3b0ffbc7e..545c86728 100644 --- a/src/dailyai/queue_aggregators.py +++ b/src/dailyai/queue_aggregators.py @@ -33,7 +33,7 @@ class LLMContextAggregator(AIService): role: str, bot_participant_id=None, complete_sentences=True, - pass_through=False): + pass_through=True): self.messages = messages self.bot_participant_id = bot_participant_id self.role = role @@ -42,28 +42,32 @@ class LLMContextAggregator(AIService): self.pass_through = pass_through async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: - # TODO: split up transcription by participant - if isinstance(frame, TextQueueFrame): - - # Ignore transcription frames from the bot - if isinstance(frame, TranscriptionQueueFrame): - if frame.participantId == self.bot_participant_id: - return - - if self.complete_sentences: - self.sentence += frame.text - if self.sentence.endswith((".", "?", "!")): - self.messages.append({"role": self.role, "content": self.sentence}) - self.sentence = "" - yield LLMMessagesQueueFrame(self.messages) - else: - self.messages.append({"role": self.role, "content": frame.text}) - yield LLMMessagesQueueFrame(self.messages) - - if self.pass_through: - yield frame - else: + # We don't do anything with non-text frames, pass it along to next in the pipeline. + if not isinstance(frame, TextQueueFrame): yield frame + return + + # The common case for "pass through" is receiving frames from the LLM that we'll + # use to update the "assistant" LLM messages, but also passing the text frames + # along to a TTS service to be spoken to the user. + if self.pass_through: + yield frame + + # Ignore transcription frames from the bot + if isinstance(frame, TranscriptionQueueFrame): + if frame.participantId == self.bot_participant_id: + return + + # TODO: split up transcription by participant + if self.complete_sentences: + self.sentence += frame.text # type: ignore -- the linter thinks this isn't a TextQueueFrame, even though we check it above + if self.sentence.endswith((".", "?", "!")): + self.messages.append({"role": self.role, "content": self.sentence}) + self.sentence = "" + yield LLMMessagesQueueFrame(self.messages) + else: + self.messages.append({"role": self.role, "content": frame.text}) # type: ignore -- the linter thinks this isn't a TextQueueFrame, even though we check it above + yield LLMMessagesQueueFrame(self.messages) class LLMUserContextAggregator(LLMContextAggregator): def __init__(self, diff --git a/src/samples/foundational/07-interruptible.py b/src/samples/foundational/07-interruptible.py index cd18804af..927a5670f 100644 --- a/src/samples/foundational/07-interruptible.py +++ b/src/samples/foundational/07-interruptible.py @@ -25,6 +25,7 @@ async def main(room_url: str, token): transport.mic_enabled = True transport.mic_sample_rate = 16000 transport.camera_enabled = False + transport.start_transcription = True llm = AzureLLMService() tts = ElevenLabsTTSService(voice_id="ErXwobaYiN019PkySvjV") From ead655fe238fca6462f0b3b73052dcadd08e2d58 Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Fri, 26 Jan 2024 10:07:16 -0500 Subject: [PATCH 8/8] some more fixup --- src/dailyai/conversation_wrappers.py | 4 ++-- src/dailyai/queue_aggregators.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/dailyai/conversation_wrappers.py b/src/dailyai/conversation_wrappers.py index bb83f1272..7f688477c 100644 --- a/src/dailyai/conversation_wrappers.py +++ b/src/dailyai/conversation_wrappers.py @@ -43,10 +43,10 @@ class InterruptibleConversationWrapper: async def speak_after_delay(self, user_speech, messages): await asyncio.sleep(self._delay_before_speech_seconds) tma_in = self._llm_context_aggregator_in( - messages, "user", self._my_participant_id, False + messages, self._my_participant_id, complete_sentences=False ) tma_out = self._llm_context_aggregator_out( - messages, "assistant", self._my_participant_id + messages, self._my_participant_id ) await self._runner(user_speech, tma_in, tma_out) diff --git a/src/dailyai/queue_aggregators.py b/src/dailyai/queue_aggregators.py index 545c86728..55461e29c 100644 --- a/src/dailyai/queue_aggregators.py +++ b/src/dailyai/queue_aggregators.py @@ -47,17 +47,17 @@ class LLMContextAggregator(AIService): yield frame return + # Ignore transcription frames from the bot + if isinstance(frame, TranscriptionQueueFrame): + if frame.participantId == self.bot_participant_id: + return + # The common case for "pass through" is receiving frames from the LLM that we'll # use to update the "assistant" LLM messages, but also passing the text frames # along to a TTS service to be spoken to the user. if self.pass_through: yield frame - # Ignore transcription frames from the bot - if isinstance(frame, TranscriptionQueueFrame): - if frame.participantId == self.bot_participant_id: - return - # TODO: split up transcription by participant if self.complete_sentences: self.sentence += frame.text # type: ignore -- the linter thinks this isn't a TextQueueFrame, even though we check it above