wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #34 from daily-co/rename-frames

Browse Source 
Remove Queue in frame names
This commit is contained in:
Moishe Lettvin
2024-03-06 14:10:56 -05:00
committed by GitHub
parent b9556716dd 62fd371b97
commit d3e76c4fd6
25 changed files with 241 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
src/dailyai/pipeline/aggregators.py
View File

@@ -5,13 +5,13 @@ from tblib import Frame
from dailyai.pipeline.frame_processor import FrameProcessor from dailyai.pipeline.frame_processor import FrameProcessor
from dailyai.pipeline.frames import ( from dailyai.pipeline.frames import (
ControlQueueFrame, ControlFrame,
EndParallelPipeQueueFrame, EndPipeFrame,
EndStreamQueueFrame, EndFrame,
LLMMessagesQueueFrame, LLMMessagesQueueFrame,
LLMResponseEndQueueFrame, LLMResponseEndFrame,
QueueFrame, Frame,
TextQueueFrame, TextFrame,
TranscriptionQueueFrame, TranscriptionQueueFrame,
) )
from dailyai.pipeline.pipeline import Pipeline from dailyai.pipeline.pipeline import Pipeline
@@ -38,10 +38,10 @@ class LLMContextAggregator(AIService):
self.pass_through = pass_through self.pass_through = pass_through
async def process_frame( async def process_frame(
self, frame: QueueFrame self, frame: Frame
) -> AsyncGenerator[QueueFrame, None]: ) -> AsyncGenerator[Frame, None]:
# We don't do anything with non-text frames, pass it along to next in the pipeline. # We don't do anything with non-text frames, pass it along to next in the pipeline.
if not isinstance(frame, TextQueueFrame): if not isinstance(frame, TextFrame):
yield frame yield frame
return return
@@ -71,7 +71,7 @@ class LLMContextAggregator(AIService):
self.messages.append({"role": self.role, "content": frame.text}) self.messages.append({"role": self.role, "content": frame.text})
yield LLMMessagesQueueFrame(self.messages) yield LLMMessagesQueueFrame(self.messages)
async def finalize(self) -> AsyncGenerator[QueueFrame, None]: async def finalize(self) -> AsyncGenerator[Frame, None]:
# Send any dangling words that weren't finished with punctuation. # Send any dangling words that weren't finished with punctuation.
if self.complete_sentences and self.sentence: if self.complete_sentences and self.sentence:
self.messages.append({"role": self.role, "content": self.sentence}) self.messages.append({"role": self.role, "content": self.sentence})
@@ -106,18 +106,18 @@ class SentenceAggregator(FrameProcessor):
self.aggregation = "" self.aggregation = ""
async def process_frame( async def process_frame(
self, frame: QueueFrame self, frame: Frame
) -> AsyncGenerator[QueueFrame, None]: ) -> AsyncGenerator[Frame, None]:
if isinstance(frame, TextQueueFrame): if isinstance(frame, TextFrame):
m = re.search("(.*[?.!])(.*)", frame.text) m = re.search("(.*[?.!])(.*)", frame.text)
if m: if m:
yield TextQueueFrame(self.aggregation + m.group(1)) yield TextFrame(self.aggregation + m.group(1))
self.aggregation = m.group(2) self.aggregation = m.group(2)
else: else:
self.aggregation += frame.text self.aggregation += frame.text
elif isinstance(frame, EndStreamQueueFrame): elif isinstance(frame, EndFrame):
if self.aggregation: if self.aggregation:
yield TextQueueFrame(self.aggregation) yield TextFrame(self.aggregation)
yield frame yield frame
else: else:
yield frame yield frame
@@ -128,12 +128,12 @@ class LLMFullResponseAggregator(FrameProcessor):
self.aggregation = "" self.aggregation = ""
async def process_frame( async def process_frame(
self, frame: QueueFrame self, frame: Frame
) -> AsyncGenerator[QueueFrame, None]: ) -> AsyncGenerator[Frame, None]:
if isinstance(frame, TextQueueFrame): if isinstance(frame, TextFrame):
self.aggregation += frame.text self.aggregation += frame.text
elif isinstance(frame, LLMResponseEndQueueFrame): elif isinstance(frame, LLMResponseEndFrame):
yield TextQueueFrame(self.aggregation) yield TextFrame(self.aggregation)
self.aggregation = "" self.aggregation = ""
else: else:
yield frame yield frame
@@ -143,20 +143,20 @@ class StatelessTextTransformer(FrameProcessor):
def __init__(self, transform_fn): def __init__(self, transform_fn):
self.transform_fn = transform_fn self.transform_fn = transform_fn
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, TextQueueFrame): if isinstance(frame, TextFrame):
result = self.transform_fn(frame.text) result = self.transform_fn(frame.text)
if isinstance(result, Coroutine): if isinstance(result, Coroutine):
result = await result result = await result
yield TextQueueFrame(result) yield TextFrame(result)
else: else:
yield frame yield frame
class ParallelPipeline(FrameProcessor): class ParallelPipeline(FrameProcessor):
def __init__(self, pipeline_definitions: List[List[FrameProcessor]]): def __init__(self, pipeline_definitions: List[List[FrameProcessor]]):
self.sources = [asyncio.Queue() for _ in pipeline_definitions] self.sources = [asyncio.Queue() for _ in pipeline_definitions]
self.sink: asyncio.Queue[QueueFrame] = asyncio.Queue() self.sink: asyncio.Queue[Frame] = asyncio.Queue()
self.pipelines: list[Pipeline] = [ self.pipelines: list[Pipeline] = [
Pipeline( Pipeline(
pipeline_definition, pipeline_definition,
@@ -166,10 +166,10 @@ class ParallelPipeline(FrameProcessor):
for source, pipeline_definition in zip(self.sources, pipeline_definitions) for source, pipeline_definition in zip(self.sources, pipeline_definitions)
] ]
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
for source in self.sources: for source in self.sources:
await source.put(frame) await source.put(frame)
await source.put(EndParallelPipeQueueFrame()) await source.put(EndPipeFrame())
await asyncio.gather(*[pipeline.run_pipeline() for pipeline in self.pipelines]) await asyncio.gather(*[pipeline.run_pipeline() for pipeline in self.pipelines])
@@ -186,7 +186,7 @@ class ParallelPipeline(FrameProcessor):
seen_ids.add(id(frame)) seen_ids.add(id(frame))
# Skip passing along EndParallelPipeQueueFrame, because we use them for our own flow control. # Skip passing along EndParallelPipeQueueFrame, because we use them for our own flow control.
if not isinstance(frame, EndParallelPipeQueueFrame): if not isinstance(frame, EndPipeFrame):
yield frame yield frame
class GatedAggregator(FrameProcessor): class GatedAggregator(FrameProcessor):
@@ -194,9 +194,9 @@ class GatedAggregator(FrameProcessor):
self.gate_open_fn = gate_open_fn self.gate_open_fn = gate_open_fn
self.gate_close_fn = gate_close_fn self.gate_close_fn = gate_close_fn
self.gate_open = start_open self.gate_open = start_open
self.accumulator: List[QueueFrame] = [] self.accumulator: List[Frame] = []
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if self.gate_open: if self.gate_open:
if self.gate_close_fn(frame): if self.gate_close_fn(frame):
self.gate_open = False self.gate_open = False

12
src/dailyai/pipeline/frame_processor.py
View File

@@ -1,7 +1,7 @@
from abc import abstractmethod from abc import abstractmethod
from typing import AsyncGenerator from typing import AsyncGenerator
from dailyai.pipeline.frames import ControlQueueFrame, QueueFrame from dailyai.pipeline.frames import ControlFrame, Frame
""" """
This is the base class for all frame processors. Frame processors consume a frame This is the base class for all frame processors. Frame processors consume a frame
@@ -20,16 +20,16 @@ be closed, del'd, etc.
class FrameProcessor: class FrameProcessor:
@abstractmethod @abstractmethod
async def process_frame( async def process_frame(
self, frame: QueueFrame self, frame: Frame
) -> AsyncGenerator[QueueFrame, None]: ) -> AsyncGenerator[Frame, None]:
if isinstance(frame, ControlQueueFrame): if isinstance(frame, ControlFrame):
yield frame yield frame
@abstractmethod @abstractmethod
async def finalize(self) -> AsyncGenerator[QueueFrame, None]: async def finalize(self) -> AsyncGenerator[Frame, None]:
# This is a trick for the interpreter (and linter) to know that this is a generator. # This is a trick for the interpreter (and linter) to know that this is a generator.
if False: if False:
yield QueueFrame() yield Frame()
@abstractmethod @abstractmethod
async def interrupted(self) -> None: async def interrupted(self) -> None:

45
src/dailyai/pipeline/frames.py
View File

@@ -2,72 +2,73 @@ from dataclasses import dataclass
from typing import Any from typing import Any
class QueueFrame: class Frame:
pass
class ControlFrame(Frame):
# Control frames should contain no instance data, so
# equality is based solely on the class.
def __eq__(self, other): def __eq__(self, other):
return isinstance(other, self.__class__) return type(other) == self.__class__
class ControlQueueFrame(QueueFrame): class StartFrame(ControlFrame):
pass pass
class StartStreamQueueFrame(ControlQueueFrame): class EndFrame(ControlFrame):
pass
class EndPipeFrame(ControlFrame):
pass pass
class EndStreamQueueFrame(ControlQueueFrame): class LLMResponseStartFrame(ControlFrame):
pass
class EndParallelPipeQueueFrame(ControlQueueFrame):
pass pass
class LLMResponseStartQueueFrame(QueueFrame): class LLMResponseEndFrame(ControlFrame):
pass
class LLMResponseEndQueueFrame(QueueFrame):
pass pass
@dataclass() @dataclass()
class AudioQueueFrame(QueueFrame): class AudioFrame(Frame):
data: bytes data: bytes
@dataclass() @dataclass()
class ImageQueueFrame(QueueFrame): class ImageFrame(Frame):
url: str | None url: str | None
image: bytes image: bytes
@dataclass() @dataclass()
class SpriteQueueFrame(QueueFrame): class SpriteFrame(Frame):
images: list[bytes] images: list[bytes]
@dataclass() @dataclass()
class TextQueueFrame(QueueFrame): class TextFrame(Frame):
text: str text: str
@dataclass() @dataclass()
class TranscriptionQueueFrame(TextQueueFrame): class TranscriptionQueueFrame(TextFrame):
participantId: str participantId: str
timestamp: str timestamp: str
@dataclass() @dataclass()
class LLMMessagesQueueFrame(QueueFrame): class LLMMessagesQueueFrame(Frame):
messages: list[dict[str, str]] # TODO: define this more concretely! messages: list[dict[str, str]] # TODO: define this more concretely!
class AppMessageQueueFrame(QueueFrame): class AppMessageQueueFrame(Frame):
message: Any message: Any
participantId: str participantId: str
class UserStartedSpeakingFrame(QueueFrame): class UserStartedSpeakingFrame(Frame):
pass pass
class UserStoppedSpeakingFrame(QueueFrame): class UserStoppedSpeakingFrame(Frame):
pass pass

18
src/dailyai/pipeline/pipeline.py
View File

@@ -2,7 +2,7 @@ import asyncio
from typing import AsyncGenerator, List from typing import AsyncGenerator, List
from dailyai.pipeline.frame_processor import FrameProcessor from dailyai.pipeline.frame_processor import FrameProcessor
from dailyai.pipeline.frames import EndParallelPipeQueueFrame, EndStreamQueueFrame, QueueFrame from dailyai.pipeline.frames import EndPipeFrame, EndFrame, Frame
""" """
This class manages a pipe of FrameProcessors, and runs them in sequence. The "source" This class manages a pipe of FrameProcessors, and runs them in sequence. The "source"
@@ -17,19 +17,19 @@ class Pipeline:
self, self,
processors: List[FrameProcessor], processors: List[FrameProcessor],
source: asyncio.Queue | None = None, source: asyncio.Queue | None = None,
sink: asyncio.Queue[QueueFrame] | None = None, sink: asyncio.Queue[Frame] | None = None,
): ):
self.processors = processors self.processors = processors
self.source: asyncio.Queue[QueueFrame] | None = source self.source: asyncio.Queue[Frame] | None = source
self.sink: asyncio.Queue[QueueFrame] | None = sink self.sink: asyncio.Queue[Frame] | None = sink
def set_source(self, source: asyncio.Queue[QueueFrame]): def set_source(self, source: asyncio.Queue[Frame]):
self.source = source self.source = source
def set_sink(self, sink: asyncio.Queue[QueueFrame]): def set_sink(self, sink: asyncio.Queue[Frame]):
self.sink = sink self.sink = sink
async def get_next_source_frame(self) -> AsyncGenerator[QueueFrame, None]: async def get_next_source_frame(self) -> AsyncGenerator[Frame, None]:
if self.source is None: if self.source is None:
raise ValueError("Source queue not set") raise ValueError("Source queue not set")
yield await self.source.get() yield await self.source.get()
@@ -52,9 +52,9 @@ class Pipeline:
async for frame in frame_generator: async for frame in frame_generator:
await self.sink.put(frame) await self.sink.put(frame)
if isinstance( if isinstance(
frame, EndStreamQueueFrame frame, EndFrame
) or isinstance( ) or isinstance(
frame, EndParallelPipeQueueFrame frame, EndPipeFrame
): ):
return return
except asyncio.CancelledError: except asyncio.CancelledError:

58
src/dailyai/services/ai_services.py
View File

@@ -6,14 +6,14 @@ import wave
from dailyai.pipeline.frame_processor import FrameProcessor from dailyai.pipeline.frame_processor import FrameProcessor
from dailyai.pipeline.frames import ( from dailyai.pipeline.frames import (
AudioQueueFrame, AudioFrame,
EndStreamQueueFrame, EndFrame,
ImageQueueFrame, ImageFrame,
LLMMessagesQueueFrame, LLMMessagesQueueFrame,
LLMResponseEndQueueFrame, LLMResponseEndFrame,
LLMResponseStartQueueFrame, LLMResponseStartFrame,
QueueFrame, Frame,
TextQueueFrame, TextFrame,
TranscriptionQueueFrame, TranscriptionQueueFrame,
) )
@@ -33,14 +33,14 @@ class AIService(FrameProcessor):
await queue.put(frame) await queue.put(frame)
if add_end_of_stream: if add_end_of_stream:
await queue.put(EndStreamQueueFrame()) await queue.put(EndFrame())
async def run( async def run(
self, self,
frames: Iterable[QueueFrame] frames: Iterable[Frame]
| AsyncIterable[QueueFrame] | AsyncIterable[Frame]
| asyncio.Queue[QueueFrame], | asyncio.Queue[Frame],
) -> AsyncGenerator[QueueFrame, None]: ) -> AsyncGenerator[Frame, None]:
try: try:
if isinstance(frames, AsyncIterable): if isinstance(frames, AsyncIterable):
async for frame in frames: async for frame in frames:
@@ -55,7 +55,7 @@ class AIService(FrameProcessor):
frame = await frames.get() frame = await frames.get()
async for output_frame in self.process_frame(frame): async for output_frame in self.process_frame(frame):
yield output_frame yield output_frame
if isinstance(frame, EndStreamQueueFrame): if isinstance(frame, EndFrame):
break break
else: else:
raise Exception("Frames must be an iterable or async iterable") raise Exception("Frames must be an iterable or async iterable")
@@ -76,12 +76,12 @@ class LLMService(AIService):
async def run_llm(self, messages) -> str: async def run_llm(self, messages) -> str:
pass pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, LLMMessagesQueueFrame): if isinstance(frame, LLMMessagesQueueFrame):
yield LLMResponseStartQueueFrame() yield LLMResponseStartFrame()
async for text_chunk in self.run_llm_async(frame.messages): async for text_chunk in self.run_llm_async(frame.messages):
yield TextQueueFrame(text_chunk) yield TextFrame(text_chunk)
yield LLMResponseEndQueueFrame() yield LLMResponseEndFrame()
else: else:
yield frame yield frame
@@ -103,8 +103,8 @@ class TTSService(AIService):
# yield empty bytes here, so linting can infer what this method does # yield empty bytes here, so linting can infer what this method does
yield bytes() yield bytes()
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if not isinstance(frame, TextQueueFrame): if not isinstance(frame, TextFrame):
yield frame yield frame
return return
@@ -119,16 +119,16 @@ class TTSService(AIService):
if text: if text:
async for audio_chunk in self.run_tts(text): async for audio_chunk in self.run_tts(text):
yield AudioQueueFrame(audio_chunk) yield AudioFrame(audio_chunk)
async def finalize(self): async def finalize(self):
if self.current_sentence: if self.current_sentence:
async for audio_chunk in self.run_tts(self.current_sentence): async for audio_chunk in self.run_tts(self.current_sentence):
yield AudioQueueFrame(audio_chunk) yield AudioFrame(audio_chunk)
# Convenience function to send the audio for a sentence to the given queue # Convenience function to send the audio for a sentence to the given queue
async def say(self, sentence, queue: asyncio.Queue): async def say(self, sentence, queue: asyncio.Queue):
await self.run_to_queue(queue, [TextQueueFrame(sentence)]) await self.run_to_queue(queue, [TextFrame(sentence)])
class ImageGenService(AIService): class ImageGenService(AIService):
@@ -141,13 +141,13 @@ class ImageGenService(AIService):
async def run_image_gen(self, sentence: str) -> tuple[str, bytes]: async def run_image_gen(self, sentence: str) -> tuple[str, bytes]:
pass pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if not isinstance(frame, TextQueueFrame): if not isinstance(frame, TextFrame):
yield frame yield frame
return return
(url, image_data) = await self.run_image_gen(frame.text) (url, image_data) = await self.run_image_gen(frame.text)
yield ImageQueueFrame(url, image_data) yield ImageFrame(url, image_data)
class STTService(AIService): class STTService(AIService):
@@ -164,9 +164,9 @@ class STTService(AIService):
"""Returns transcript as a string""" """Returns transcript as a string"""
pass pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
"""Processes a frame of audio data, either buffering or transcribing it.""" """Processes a frame of audio data, either buffering or transcribing it."""
if not isinstance(frame, AudioQueueFrame): if not isinstance(frame, AudioFrame):
return return
data = frame.data data = frame.data
@@ -187,8 +187,8 @@ class FrameLogger(AIService):
super().__init__(**kwargs) super().__init__(**kwargs)
self.prefix = prefix self.prefix = prefix
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, (AudioQueueFrame, ImageQueueFrame)): if isinstance(frame, (AudioFrame, ImageFrame)):
self.logger.info(f"{self.prefix}: {type(frame)}") self.logger.info(f"{self.prefix}: {type(frame)}")
else: else:
print(f"{self.prefix}: {frame}") print(f"{self.prefix}: {frame}")

50
src/dailyai/services/base_transport_service.py
View File

@@ -12,12 +12,12 @@ from typing import AsyncGenerator
from enum import Enum from enum import Enum
from dailyai.pipeline.frames import ( from dailyai.pipeline.frames import (
AudioQueueFrame, AudioFrame,
EndStreamQueueFrame, EndFrame,
ImageQueueFrame, ImageFrame,
QueueFrame, Frame,
SpriteQueueFrame, SpriteFrame,
StartStreamQueueFrame, StartFrame,
TranscriptionQueueFrame, TranscriptionQueueFrame,
UserStartedSpeakingFrame, UserStartedSpeakingFrame,
UserStoppedSpeakingFrame UserStoppedSpeakingFrame
@@ -159,7 +159,7 @@ class BaseTransportService():
self._stop_threads.set() self._stop_threads.set()
await self.send_queue.put(EndStreamQueueFrame()) await self.send_queue.put(EndFrame())
await async_output_queue_marshal_task await async_output_queue_marshal_task
await self.send_queue.join() await self.send_queue.join()
self._frame_consumer_thread.join() self._frame_consumer_thread.join()
@@ -182,7 +182,7 @@ class BaseTransportService():
pipeline.set_sink(self.send_queue) pipeline.set_sink(self.send_queue)
pipeline_task = asyncio.create_task(pipeline.run_pipeline()) pipeline_task = asyncio.create_task(pipeline.run_pipeline())
async def yield_frame(frame:QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def yield_frame(frame:Frame) -> AsyncGenerator[Frame, None]:
yield frame yield frame
async def post_process(post_processor): async def post_process(post_processor):
@@ -194,7 +194,7 @@ class BaseTransportService():
print("post-processing frame: ", frame.__class__.__name__) print("post-processing frame: ", frame.__class__.__name__)
await post_processor.process_frame(frame) await post_processor.process_frame(frame)
if isinstance(frame, EndStreamQueueFrame): if isinstance(frame, EndFrame):
break break
post_process_task = asyncio.create_task(post_process(post_processor)) post_process_task = asyncio.create_task(post_process(post_processor))
@@ -214,7 +214,7 @@ class BaseTransportService():
async for frame in frame_generator: async for frame in frame_generator:
await source_queue.put(frame) await source_queue.put(frame)
if isinstance(frame, EndStreamQueueFrame): if isinstance(frame, EndFrame):
break break
await asyncio.gather(pipeline_task, post_process_task) await asyncio.gather(pipeline_task, post_process_task)
@@ -303,20 +303,20 @@ class BaseTransportService():
async def _marshal_frames(self): async def _marshal_frames(self):
while True: while True:
frame: QueueFrame | list = await self.send_queue.get() frame: Frame | list = await self.send_queue.get()
self._threadsafe_send_queue.put(frame) self._threadsafe_send_queue.put(frame)
self.send_queue.task_done() self.send_queue.task_done()
if isinstance(frame, EndStreamQueueFrame): if isinstance(frame, EndFrame):
break break
def interrupt(self): def interrupt(self):
self._is_interrupted.set() self._is_interrupted.set()
async def get_receive_frames(self) -> AsyncGenerator[QueueFrame, None]: async def get_receive_frames(self) -> AsyncGenerator[Frame, None]:
while True: while True:
frame = await self.receive_queue.get() frame = await self.receive_queue.get()
yield frame yield frame
if isinstance(frame, EndStreamQueueFrame): if isinstance(frame, EndFrame):
break break
def _receive_audio(self): def _receive_audio(self):
@@ -329,13 +329,13 @@ class BaseTransportService():
while not self._stop_threads.is_set(): while not self._stop_threads.is_set():
buffer = self.read_audio_frames(desired_frame_count) buffer = self.read_audio_frames(desired_frame_count)
if len(buffer) > 0: if len(buffer) > 0:
frame = AudioQueueFrame(buffer) frame = AudioFrame(buffer)
asyncio.run_coroutine_threadsafe( asyncio.run_coroutine_threadsafe(
self.receive_queue.put(frame), self._loop self.receive_queue.put(frame), self._loop
) )
asyncio.run_coroutine_threadsafe( asyncio.run_coroutine_threadsafe(
self.receive_queue.put(EndStreamQueueFrame()), self._loop self.receive_queue.put(EndFrame()), self._loop
) )
def _set_image(self, image: bytes): def _set_image(self, image: bytes):
@@ -363,18 +363,18 @@ class BaseTransportService():
all_audio_frames = bytearray() all_audio_frames = bytearray()
while True: while True:
try: try:
frames_or_frame: QueueFrame | list[QueueFrame] = ( frames_or_frame: Frame | list[Frame] = (
self._threadsafe_send_queue.get() self._threadsafe_send_queue.get()
) )
if isinstance(frames_or_frame, QueueFrame): if isinstance(frames_or_frame, Frame):
frames: list[QueueFrame] = [frames_or_frame] frames: list[Frame] = [frames_or_frame]
elif isinstance(frames_or_frame, list): elif isinstance(frames_or_frame, list):
frames: list[QueueFrame] = frames_or_frame frames: list[Frame] = frames_or_frame
else: else:
raise Exception("Unknown type in output queue") raise Exception("Unknown type in output queue")
for frame in frames: for frame in frames:
if isinstance(frame, EndStreamQueueFrame): if isinstance(frame, EndFrame):
self._logger.info("Stopping frame consumer thread") self._logger.info("Stopping frame consumer thread")
self._threadsafe_send_queue.task_done() self._threadsafe_send_queue.task_done()
if self._loop: if self._loop:
@@ -386,7 +386,7 @@ class BaseTransportService():
# if interrupted, we just pull frames off the queue and discard them # if interrupted, we just pull frames off the queue and discard them
if not self._is_interrupted.is_set(): if not self._is_interrupted.is_set():
if frame: if frame:
if isinstance(frame, AudioQueueFrame): if isinstance(frame, AudioFrame):
chunk = frame.data chunk = frame.data
all_audio_frames.extend(chunk) all_audio_frames.extend(chunk)
@@ -398,9 +398,9 @@ class BaseTransportService():
if truncated_length: if truncated_length:
self.write_frame_to_mic(bytes(b[:truncated_length])) self.write_frame_to_mic(bytes(b[:truncated_length]))
b = b[truncated_length:] b = b[truncated_length:]
elif isinstance(frame, ImageQueueFrame): elif isinstance(frame, ImageFrame):
self._set_image(frame.image) self._set_image(frame.image)
elif isinstance(frame, SpriteQueueFrame): elif isinstance(frame, SpriteFrame):
self._set_images(frame.images) self._set_images(frame.images)
elif len(b): elif len(b):
self.write_frame_to_mic(bytes(b)) self.write_frame_to_mic(bytes(b))
@@ -418,7 +418,7 @@ class BaseTransportService():
self.write_frame_to_mic(bytes(b[:truncated_length])) self.write_frame_to_mic(bytes(b[:truncated_length]))
b = bytearray() b = bytearray()
if isinstance(frame, StartStreamQueueFrame): if isinstance(frame, StartFrame):
self._is_interrupted.clear() self._is_interrupted.clear()
self._threadsafe_send_queue.task_done() self._threadsafe_send_queue.task_done()

6
src/dailyai/services/local_stt_service.py
View File

@@ -4,7 +4,7 @@ import math
import time import time
from typing import AsyncGenerator from typing import AsyncGenerator
import wave import wave
from dailyai.pipeline.frames import AudioQueueFrame, QueueFrame, TranscriptionQueueFrame from dailyai.pipeline.frames import AudioFrame, Frame, TranscriptionQueueFrame
from dailyai.services.ai_services import STTService from dailyai.services.ai_services import STTService
@@ -39,9 +39,9 @@ class LocalSTTService(STTService):
ww.setframerate(self._frame_rate) ww.setframerate(self._frame_rate)
self._wave = ww self._wave = ww
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
"""Processes a frame of audio data, either buffering or transcribing it.""" """Processes a frame of audio data, either buffering or transcribing it."""
if not isinstance(frame, AudioQueueFrame): if not isinstance(frame, AudioFrame):
return return
data = frame.data data = frame.data

76
src/dailyai/tests/test_aggregators.py
View File

@@ -9,13 +9,13 @@ from dailyai.pipeline.aggregators import (
StatelessTextTransformer, StatelessTextTransformer,
) )
from dailyai.pipeline.frames import ( from dailyai.pipeline.frames import (
AudioQueueFrame, AudioFrame,
EndStreamQueueFrame, EndFrame,
ImageQueueFrame, ImageFrame,
LLMResponseEndQueueFrame, LLMResponseEndFrame,
LLMResponseStartQueueFrame, LLMResponseStartFrame,
QueueFrame, Frame,
TextQueueFrame, TextFrame,
) )
from dailyai.pipeline.pipeline import Pipeline from dailyai.pipeline.pipeline import Pipeline
@@ -27,46 +27,46 @@ class TestDailyFrameAggregators(unittest.IsolatedAsyncioTestCase):
expected_sentences = ["Hello, world.", " How are you?", " I am fine "] expected_sentences = ["Hello, world.", " How are you?", " I am fine "]
aggregator = SentenceAggregator() aggregator = SentenceAggregator()
for word in sentence.split(" "): for word in sentence.split(" "):
async for sentence in aggregator.process_frame(TextQueueFrame(word + " ")): async for sentence in aggregator.process_frame(TextFrame(word + " ")):
self.assertIsInstance(sentence, TextQueueFrame) self.assertIsInstance(sentence, TextFrame)
if isinstance(sentence, TextQueueFrame): if isinstance(sentence, TextFrame):
self.assertEqual(sentence.text, expected_sentences.pop(0)) self.assertEqual(sentence.text, expected_sentences.pop(0))
async for sentence in aggregator.process_frame(EndStreamQueueFrame()): async for sentence in aggregator.process_frame(EndFrame()):
if len(expected_sentences): if len(expected_sentences):
self.assertIsInstance(sentence, TextQueueFrame) self.assertIsInstance(sentence, TextFrame)
if isinstance(sentence, TextQueueFrame): if isinstance(sentence, TextFrame):
self.assertEqual(sentence.text, expected_sentences.pop(0)) self.assertEqual(sentence.text, expected_sentences.pop(0))
else: else:
self.assertIsInstance(sentence, EndStreamQueueFrame) self.assertIsInstance(sentence, EndFrame)
self.assertEqual(expected_sentences, []) self.assertEqual(expected_sentences, [])
async def test_gated_accumulator(self): async def test_gated_accumulator(self):
gated_aggregator = GatedAggregator( gated_aggregator = GatedAggregator(
gate_open_fn=lambda frame: isinstance(frame, ImageQueueFrame), gate_open_fn=lambda frame: isinstance(frame, ImageFrame),
gate_close_fn=lambda frame: isinstance(frame, LLMResponseStartQueueFrame), gate_close_fn=lambda frame: isinstance(frame, LLMResponseStartFrame),
start_open=False, start_open=False,
) )
frames = [ frames = [
LLMResponseStartQueueFrame(), LLMResponseStartFrame(),
TextQueueFrame("Hello, "), TextFrame("Hello, "),
TextQueueFrame("world."), TextFrame("world."),
AudioQueueFrame(b"hello"), AudioFrame(b"hello"),
ImageQueueFrame("image", b"image"), ImageFrame("image", b"image"),
AudioQueueFrame(b"world"), AudioFrame(b"world"),
LLMResponseEndQueueFrame(), LLMResponseEndFrame(),
] ]
expected_output_frames = [ expected_output_frames = [
ImageQueueFrame("image", b"image"), ImageFrame("image", b"image"),
LLMResponseStartQueueFrame(), LLMResponseStartFrame(),
TextQueueFrame("Hello, "), TextFrame("Hello, "),
TextQueueFrame("world."), TextFrame("world."),
AudioQueueFrame(b"hello"), AudioFrame(b"hello"),
AudioQueueFrame(b"world"), AudioFrame(b"world"),
LLMResponseEndQueueFrame(), LLMResponseEndFrame(),
] ]
for frame in frames: for frame in frames:
async for out_frame in gated_aggregator.process_frame(frame): async for out_frame in gated_aggregator.process_frame(frame):
@@ -98,16 +98,16 @@ class TestDailyFrameAggregators(unittest.IsolatedAsyncioTestCase):
) )
frames = [ frames = [
TextQueueFrame("Hello, "), TextFrame("Hello, "),
TextQueueFrame("world."), TextFrame("world."),
EndStreamQueueFrame() EndFrame()
] ]
expected_output_frames: list[QueueFrame] = [ expected_output_frames: list[Frame] = [
TextQueueFrame(text='Hello, :pipe1.'), TextFrame(text='Hello, :pipe1.'),
TextQueueFrame(text='world.:pipe1.'), TextFrame(text='world.:pipe1.'),
TextQueueFrame(text='Hello, world.:pipe2.'), TextFrame(text='Hello, world.:pipe2.'),
EndStreamQueueFrame() EndFrame()
] ]
for frame in frames: for frame in frames:

14
src/dailyai/tests/test_ai_services.py
View File

@@ -3,11 +3,11 @@ import unittest
from typing import AsyncGenerator, Generator from typing import AsyncGenerator, Generator
from dailyai.services.ai_services import AIService from dailyai.services.ai_services import AIService
from dailyai.pipeline.frames import EndStreamQueueFrame, QueueFrame, TextQueueFrame from dailyai.pipeline.frames import EndFrame, Frame, TextFrame
class SimpleAIService(AIService): class SimpleAIService(AIService):
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
yield frame yield frame
@@ -16,11 +16,11 @@ class TestBaseAIService(unittest.IsolatedAsyncioTestCase):
service = SimpleAIService() service = SimpleAIService()
input_frames = [ input_frames = [
TextQueueFrame("hello"), TextFrame("hello"),
EndStreamQueueFrame() EndFrame()
] ]
async def iterate_frames() -> AsyncGenerator[QueueFrame, None]: async def iterate_frames() -> AsyncGenerator[Frame, None]:
for frame in input_frames: for frame in input_frames:
yield frame yield frame
@@ -33,9 +33,9 @@ class TestBaseAIService(unittest.IsolatedAsyncioTestCase):
async def test_nonasync_input(self): async def test_nonasync_input(self):
service = SimpleAIService() service = SimpleAIService()
input_frames = [TextQueueFrame("hello"), EndStreamQueueFrame()] input_frames = [TextFrame("hello"), EndFrame()]
def iterate_frames() -> Generator[QueueFrame, None, None]: def iterate_frames() -> Generator[Frame, None, None]:
for frame in input_frames: for frame in input_frames:
yield frame yield frame

2
src/dailyai/tests/test_daily_transport_service.py
View File

@@ -3,7 +3,7 @@ import unittest
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
from dailyai.pipeline.frames import AudioQueueFrame, ImageQueueFrame from dailyai.pipeline.frames import AudioFrame, ImageFrame
class TestDailyTransport(unittest.IsolatedAsyncioTestCase): class TestDailyTransport(unittest.IsolatedAsyncioTestCase):

24
src/dailyai/tests/test_pipeline.py
View File

@@ -2,7 +2,7 @@ import asyncio
from doctest import OutputChecker from doctest import OutputChecker
import unittest import unittest
from dailyai.pipeline.aggregators import SentenceAggregator, StatelessTextTransformer from dailyai.pipeline.aggregators import SentenceAggregator, StatelessTextTransformer
from dailyai.pipeline.frames import EndStreamQueueFrame, TextQueueFrame from dailyai.pipeline.frames import EndFrame, TextFrame
from dailyai.pipeline.pipeline import Pipeline from dailyai.pipeline.pipeline import Pipeline
@@ -16,14 +16,14 @@ class TestDailyPipeline(unittest.IsolatedAsyncioTestCase):
incoming_queue = asyncio.Queue() incoming_queue = asyncio.Queue()
pipeline = Pipeline([aggregator], incoming_queue, outgoing_queue) pipeline = Pipeline([aggregator], incoming_queue, outgoing_queue)
await incoming_queue.put(TextQueueFrame("Hello, ")) await incoming_queue.put(TextFrame("Hello, "))
await incoming_queue.put(TextQueueFrame("world.")) await incoming_queue.put(TextFrame("world."))
await incoming_queue.put(EndStreamQueueFrame()) await incoming_queue.put(EndFrame())
await pipeline.run_pipeline() await pipeline.run_pipeline()
self.assertEqual(await outgoing_queue.get(), TextQueueFrame("Hello, world.")) self.assertEqual(await outgoing_queue.get(), TextFrame("Hello, world."))
self.assertIsInstance(await outgoing_queue.get(), EndStreamQueueFrame) self.assertIsInstance(await outgoing_queue.get(), EndFrame)
async def test_pipeline_multiple_stages(self): async def test_pipeline_multiple_stages(self):
sentence_aggregator = SentenceAggregator() sentence_aggregator = SentenceAggregator()
@@ -40,21 +40,21 @@ class TestDailyPipeline(unittest.IsolatedAsyncioTestCase):
sentence = "Hello, world. It's me, a pipeline." sentence = "Hello, world. It's me, a pipeline."
for c in sentence: for c in sentence:
await incoming_queue.put(TextQueueFrame(c)) await incoming_queue.put(TextFrame(c))
await incoming_queue.put(EndStreamQueueFrame()) await incoming_queue.put(EndFrame())
await pipeline.run_pipeline() await pipeline.run_pipeline()
self.assertEqual( self.assertEqual(
await outgoing_queue.get(), TextQueueFrame("H E L L O , W O R L D .") await outgoing_queue.get(), TextFrame("H E L L O , W O R L D .")
) )
self.assertEqual( self.assertEqual(
await outgoing_queue.get(), await outgoing_queue.get(),
TextQueueFrame(" I T ' S M E , A P I P E L I N E ."), TextFrame(" I T ' S M E , A P I P E L I N E ."),
) )
# leftover little bit because of the spacing # leftover little bit because of the spacing
self.assertEqual( self.assertEqual(
await outgoing_queue.get(), await outgoing_queue.get(),
TextQueueFrame(" "), TextFrame(" "),
) )
self.assertIsInstance(await outgoing_queue.get(), EndStreamQueueFrame) self.assertIsInstance(await outgoing_queue.get(), EndFrame)

2
src/examples/foundational/01-say-one-thing.py
View File

@@ -28,7 +28,6 @@ async def main(room_url):
mic_enabled=True mic_enabled=True
) )
"""
tts = ElevenLabsTTSService( tts = ElevenLabsTTSService(
aiohttp_session=session, aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"), api_key=os.getenv("ELEVENLABS_API_KEY"),
@@ -39,6 +38,7 @@ async def main(room_url):
user_id=os.getenv("PLAY_HT_USER_ID"), user_id=os.getenv("PLAY_HT_USER_ID"),
voice_url=os.getenv("PLAY_HT_VOICE_URL"), voice_url=os.getenv("PLAY_HT_VOICE_URL"),
) )
"""
# Register an event handler so we can play the audio when the participant joins. # Register an event handler so we can play the audio when the participant joins.
@transport.event_handler("on_participant_joined") @transport.event_handler("on_participant_joined")

4
src/examples/foundational/03-still-frame.py
View File

@@ -2,7 +2,7 @@ import asyncio
import aiohttp import aiohttp
import os import os
from dailyai.pipeline.frames import TextQueueFrame from dailyai.pipeline.frames import TextFrame
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.services.open_ai_services import OpenAIImageGenService from dailyai.services.open_ai_services import OpenAIImageGenService
@@ -39,7 +39,7 @@ async def main(room_url):
image_task = asyncio.create_task( image_task = asyncio.create_task(
imagegen.run_to_queue( imagegen.run_to_queue(
transport.send_queue, [ transport.send_queue, [
TextQueueFrame("a cat in the style of picasso")])) TextFrame("a cat in the style of picasso")]))
@transport.event_handler("on_first_other_participant_joined") @transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport): async def on_first_other_participant_joined(transport):

4
src/examples/foundational/03a-image-local.py
View File

@@ -4,7 +4,7 @@ import os
import tkinter as tk import tkinter as tk
from dailyai.pipeline.frames import TextQueueFrame from dailyai.pipeline.frames import TextFrame
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.services.local_transport_service import LocalTransportService from dailyai.services.local_transport_service import LocalTransportService
@@ -34,7 +34,7 @@ async def main():
) )
image_task = asyncio.create_task( image_task = asyncio.create_task(
imagegen.run_to_queue( imagegen.run_to_queue(
transport.send_queue, [TextQueueFrame("a cat in the style of picasso")] transport.send_queue, [TextFrame("a cat in the style of picasso")]
) )
) )

4
src/examples/foundational/04-utterance-and-speech.py
View File

@@ -6,7 +6,7 @@ from dailyai.pipeline.pipeline import Pipeline
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.pipeline.frames import EndStreamQueueFrame, LLMMessagesQueueFrame from dailyai.pipeline.frames import EndFrame, LLMMessagesQueueFrame
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from examples.foundational.support.runner import configure from examples.foundational.support.runner import configure
@@ -56,7 +56,7 @@ async def main(room_url: str):
frame = await buffer_queue.get() frame = await buffer_queue.get()
await transport.send_queue.put(frame) await transport.send_queue.put(frame)
buffer_queue.task_done() buffer_queue.task_done()
if isinstance(frame, EndStreamQueueFrame): if isinstance(frame, EndFrame):
break break
await asyncio.gather(pipeline_run_task, buffer_to_send_queue()) await asyncio.gather(pipeline_run_task, buffer_to_send_queue())

8
src/examples/foundational/05-sync-speech-and-image.py
View File

@@ -4,7 +4,7 @@ import aiohttp
import os import os
from dailyai.pipeline.aggregators import GatedAggregator, LLMFullResponseAggregator, ParallelPipeline, SentenceAggregator from dailyai.pipeline.aggregators import GatedAggregator, LLMFullResponseAggregator, ParallelPipeline, SentenceAggregator
from dailyai.pipeline.frames import AudioQueueFrame, EndStreamQueueFrame, ImageQueueFrame, LLMMessagesQueueFrame, LLMResponseStartQueueFrame from dailyai.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesQueueFrame, LLMResponseStartFrame
from dailyai.pipeline.pipeline import Pipeline from dailyai.pipeline.pipeline import Pipeline
from dailyai.services.azure_ai_services import AzureLLMService, AzureImageGenServiceREST, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureImageGenServiceREST, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
@@ -56,11 +56,11 @@ async def main(room_url):
] ]
await source_queue.put(LLMMessagesQueueFrame(messages)) await source_queue.put(LLMMessagesQueueFrame(messages))
await source_queue.put(EndStreamQueueFrame()) await source_queue.put(EndFrame())
gated_aggregator = GatedAggregator( gated_aggregator = GatedAggregator(
gate_open_fn=lambda frame: isinstance(frame, ImageQueueFrame), gate_open_fn=lambda frame: isinstance(frame, ImageFrame),
gate_close_fn=lambda frame: isinstance(frame, LLMResponseStartQueueFrame), gate_close_fn=lambda frame: isinstance(frame, LLMResponseStartFrame),
start_open=False, start_open=False,
) )

6
src/examples/foundational/05a-local-sync-speech-and-text.py
View File

@@ -4,7 +4,7 @@ import asyncio
import tkinter as tk import tkinter as tk
import os import os
from dailyai.pipeline.frames import AudioQueueFrame, ImageQueueFrame from dailyai.pipeline.frames import AudioFrame, ImageFrame
from dailyai.services.azure_ai_services import AzureLLMService from dailyai.services.azure_ai_services import AzureLLMService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
@@ -103,8 +103,8 @@ async def main(room_url):
if data: if data:
await transport.send_queue.put( await transport.send_queue.put(
[ [
ImageQueueFrame(data["image_url"], data["image"]), ImageFrame(data["image_url"], data["image"]),
AudioQueueFrame(data["audio"]), AudioFrame(data["audio"]),
] ]
) )

2
src/examples/foundational/06-listen-and-respond.py
View File

@@ -55,7 +55,7 @@ async def main(room_url: str, token):
tts tts
], ],
) )
await transport.run_pipeline(pipeline) await transport.run_uninterruptible_pipeline(pipeline)
transport.transcription_settings["extra"]["endpointing"] = True transport.transcription_settings["extra"]["endpointing"] = True
transport.transcription_settings["extra"]["punctuate"] = True transport.transcription_settings["extra"]["punctuate"] = True

8
src/examples/foundational/06a-image-sync.py
View File

@@ -8,7 +8,7 @@ import time
import urllib.parse import urllib.parse
from PIL import Image from PIL import Image
from dailyai.pipeline.frames import ImageQueueFrame, QueueFrame from dailyai.pipeline.frames import ImageFrame, Frame
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
@@ -27,10 +27,10 @@ class ImageSyncAggregator(AIService):
self._waiting_image = Image.open(waiting_path) self._waiting_image = Image.open(waiting_path)
self._waiting_image_bytes = self._waiting_image.tobytes() self._waiting_image_bytes = self._waiting_image.tobytes()
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
yield ImageQueueFrame(None, self._speaking_image_bytes) yield ImageFrame(None, self._speaking_image_bytes)
yield frame yield frame
yield ImageQueueFrame(None, self._waiting_image_bytes) yield ImageFrame(None, self._waiting_image_bytes)
async def main(room_url: str, token): async def main(room_url: str, token):

10
src/examples/foundational/08-bots-arguing.py
View File

@@ -6,7 +6,7 @@ from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.pipeline.frames import AudioQueueFrame, ImageQueueFrame from dailyai.pipeline.frames import AudioFrame, ImageFrame
from examples.foundational.support.runner import configure from examples.foundational.support.runner import configure
@@ -90,8 +90,8 @@ async def main(room_url: str):
) )
await transport.send_queue.put( await transport.send_queue.put(
[ [
ImageQueueFrame(None, image_data1[1]), ImageFrame(None, image_data1[1]),
AudioQueueFrame(audio1), AudioFrame(audio1),
] ]
) )
@@ -102,8 +102,8 @@ async def main(room_url: str):
) )
await transport.send_queue.put( await transport.send_queue.put(
[ [
ImageQueueFrame(None, image_data2[1]), ImageFrame(None, image_data2[1]),
AudioQueueFrame(audio2), AudioFrame(audio2),
] ]
) )

24
src/examples/foundational/10-wake-word.py
View File

@@ -11,10 +11,10 @@ from dailyai.services.azure_ai_services import AzureLLMService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.pipeline.aggregators import LLMUserContextAggregator, LLMAssistantContextAggregator from dailyai.pipeline.aggregators import LLMUserContextAggregator, LLMAssistantContextAggregator
from dailyai.pipeline.frames import ( from dailyai.pipeline.frames import (
QueueFrame, Frame,
TextQueueFrame, TextFrame,
ImageQueueFrame, ImageFrame,
SpriteQueueFrame, SpriteFrame,
TranscriptionQueueFrame, TranscriptionQueueFrame,
) )
from dailyai.services.ai_services import AIService from dailyai.services.ai_services import AIService
@@ -45,11 +45,11 @@ for file in image_files:
sprites[file] = img.tobytes() sprites[file] = img.tobytes()
# When the bot isn't talking, show a static image of the cat listening # When the bot isn't talking, show a static image of the cat listening
quiet_frame = ImageQueueFrame("", sprites["sc-listen-1.png"]) quiet_frame = ImageFrame("", sprites["sc-listen-1.png"])
# When the bot is talking, build an animation from two sprites # When the bot is talking, build an animation from two sprites
talking_list = [sprites['sc-default.png'], sprites['sc-talk.png']] talking_list = [sprites['sc-default.png'], sprites['sc-talk.png']]
talking = [random.choice(talking_list) for x in range(30)] talking = [random.choice(talking_list) for x in range(30)]
talking_frame = SpriteQueueFrame(images=talking) talking_frame = SpriteFrame(images=talking)
# TODO: Support "thinking" as soon as we get a valid transcript, while LLM is processing # TODO: Support "thinking" as soon as we get a valid transcript, while LLM is processing
thinking_list = [ thinking_list = [
@@ -57,14 +57,14 @@ thinking_list = [
sprites['sc-think-2.png'], sprites['sc-think-2.png'],
sprites['sc-think-3.png'], sprites['sc-think-3.png'],
sprites['sc-think-4.png']] sprites['sc-think-4.png']]
thinking_frame = SpriteQueueFrame(images=thinking_list) thinking_frame = SpriteFrame(images=thinking_list)
class TranscriptFilter(AIService): class TranscriptFilter(AIService):
def __init__(self, bot_participant_id=None): def __init__(self, bot_participant_id=None):
self.bot_participant_id = bot_participant_id self.bot_participant_id = bot_participant_id
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, TranscriptionQueueFrame): if isinstance(frame, TranscriptionQueueFrame):
if frame.participantId != self.bot_participant_id: if frame.participantId != self.bot_participant_id:
yield frame yield frame
@@ -75,11 +75,11 @@ class NameCheckFilter(AIService):
self.names = names self.names = names
self.sentence = "" self.sentence = ""
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
content: str = "" content: str = ""
# TODO: split up transcription by participant # TODO: split up transcription by participant
if isinstance(frame, TextQueueFrame): if isinstance(frame, TextFrame):
content = frame.text content = frame.text
self.sentence += content self.sentence += content
@@ -87,7 +87,7 @@ class NameCheckFilter(AIService):
if any(name in self.sentence for name in self.names): if any(name in self.sentence for name in self.names):
out = self.sentence out = self.sentence
self.sentence = "" self.sentence = ""
yield TextQueueFrame(out) yield TextFrame(out)
else: else:
out = self.sentence out = self.sentence
self.sentence = "" self.sentence = ""
@@ -97,7 +97,7 @@ class ImageSyncAggregator(AIService):
def __init__(self): def __init__(self):
pass pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
yield talking_frame yield talking_frame
yield frame yield frame
yield quiet_frame yield quiet_frame

14
src/examples/foundational/11-sound-effects.py
View File

@@ -9,7 +9,7 @@ from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.pipeline.aggregators import LLMContextAggregator, LLMUserContextAggregator, LLMAssistantContextAggregator from dailyai.pipeline.aggregators import LLMContextAggregator, LLMUserContextAggregator, LLMAssistantContextAggregator
from dailyai.services.ai_services import AIService, FrameLogger from dailyai.services.ai_services import AIService, FrameLogger
from dailyai.pipeline.frames import QueueFrame, AudioQueueFrame, LLMResponseEndQueueFrame, LLMMessagesQueueFrame from dailyai.pipeline.frames import Frame, AudioFrame, LLMResponseEndFrame, LLMMessagesQueueFrame
from typing import AsyncGenerator from typing import AsyncGenerator
from examples.foundational.support.runner import configure from examples.foundational.support.runner import configure
@@ -40,9 +40,9 @@ class OutboundSoundEffectWrapper(AIService):
def __init__(self): def __init__(self):
pass pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, LLMResponseEndQueueFrame): if isinstance(frame, LLMResponseEndFrame):
yield AudioQueueFrame(sounds["ding1.wav"]) yield AudioFrame(sounds["ding1.wav"])
# In case anything else up the stack needs it # In case anything else up the stack needs it
yield frame yield frame
else: else:
@@ -53,9 +53,9 @@ class InboundSoundEffectWrapper(AIService):
def __init__(self): def __init__(self):
pass pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, LLMMessagesQueueFrame): if isinstance(frame, LLMMessagesQueueFrame):
yield AudioQueueFrame(sounds["ding2.wav"]) yield AudioFrame(sounds["ding2.wav"])
# In case anything else up the stack needs it # In case anything else up the stack needs it
yield frame yield frame
else: else:
@@ -86,7 +86,7 @@ async def main(room_url: str, token):
@transport.event_handler("on_first_other_participant_joined") @transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport): async def on_first_other_participant_joined(transport):
await tts.say("Hi, I'm listening!", transport.send_queue) await tts.say("Hi, I'm listening!", transport.send_queue)
await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"])) await transport.send_queue.put(AudioFrame(sounds["ding1.wav"]))
async def handle_transcriptions(): async def handle_transcriptions():
messages = [ messages = [

6
src/examples/foundational/13a-whisper-local.py
View File

@@ -1,7 +1,7 @@
import argparse import argparse
import asyncio import asyncio
import wave import wave
from dailyai.pipeline.frames import EndStreamQueueFrame, TranscriptionQueueFrame from dailyai.pipeline.frames import EndFrame, TranscriptionQueueFrame
from dailyai.services.local_transport_service import LocalTransportService from dailyai.services.local_transport_service import LocalTransportService
from dailyai.services.whisper_ai_services import WhisperSTTService from dailyai.services.whisper_ai_services import WhisperSTTService
@@ -30,7 +30,7 @@ async def main(room_url: str):
print("got item from queue", item) print("got item from queue", item)
if isinstance(item, TranscriptionQueueFrame): if isinstance(item, TranscriptionQueueFrame):
print(item.text) print(item.text)
elif isinstance(item, EndStreamQueueFrame): elif isinstance(item, EndFrame):
break break
print("handle_transcription done") print("handle_transcription done")
@@ -38,7 +38,7 @@ async def main(room_url: str):
await stt.run_to_queue( await stt.run_to_queue(
transcription_output_queue, transport.get_receive_frames() transcription_output_queue, transport.get_receive_frames()
) )
await transcription_output_queue.put(EndStreamQueueFrame()) await transcription_output_queue.put(EndFrame())
print("handle speaker done.") print("handle speaker done.")
async def run_until_done(): async def run_until_done():

10
src/examples/image-gen.py
View File

@@ -7,7 +7,7 @@ import random
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.pipeline.frames import QueueFrame, FrameType from dailyai.pipeline.frames import Frame, FrameType
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
@@ -45,7 +45,7 @@ async def main(room_url: str, token):
print(f"finder: {finder}") print(f"finder: {finder}")
if finder >= 0: if finder >= 0:
async for audio in tts.run_tts(f"Resetting."): async for audio in tts.run_tts(f"Resetting."):
transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio)) transport.output_queue.put(Frame(FrameType.AUDIO_FRAME, audio))
sentence = "" sentence = ""
continue continue
# todo: we could differentiate between transcriptions from different participants # todo: we could differentiate between transcriptions from different participants
@@ -54,12 +54,12 @@ async def main(room_url: str, token):
# TODO: Cache this audio # TODO: Cache this audio
phrase = random.choice(["OK.", "Got it.", "Sure.", "You bet.", "Sure thing."]) phrase = random.choice(["OK.", "Got it.", "Sure.", "You bet.", "Sure thing."])
async for audio in tts.run_tts(phrase): async for audio in tts.run_tts(phrase):
transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio)) transport.output_queue.put(Frame(FrameType.AUDIO_FRAME, audio))
img_result = img.run_image_gen(sentence, "1024x1024") img_result = img.run_image_gen(sentence, "1024x1024")
awaited_img = await asyncio.gather(img_result) awaited_img = await asyncio.gather(img_result)
transport.output_queue.put( transport.output_queue.put(
[ [
QueueFrame(FrameType.IMAGE_FRAME, awaited_img[0][1]), Frame(FrameType.IMAGE_FRAME, awaited_img[0][1]),
] ]
) )
@@ -72,7 +72,7 @@ async def main(room_url: str, token):
audio_generator = tts.run_tts( audio_generator = tts.run_tts(
f"Hello, {participant['info']['userName']}! Describe an image and I'll create it. To start over, just say 'start over'.") f"Hello, {participant['info']['userName']}! Describe an image and I'll create it. To start over, just say 'start over'.")
async for audio in audio_generator: async for audio in audio_generator:
transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio)) transport.output_queue.put(Frame(FrameType.AUDIO_FRAME, audio))
transport.transcription_settings["extra"]["punctuate"] = False transport.transcription_settings["extra"]["punctuate"] = False
transport.transcription_settings["extra"]["endpointing"] = False transport.transcription_settings["extra"]["endpointing"] = False

14
src/examples/internal/11a-dial-out.py
View File

@@ -7,7 +7,7 @@ from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.pipeline.aggregators import LLMContextAggregator from dailyai.pipeline.aggregators import LLMContextAggregator
from dailyai.services.ai_services import AIService, FrameLogger from dailyai.services.ai_services import AIService, FrameLogger
from dailyai.pipeline.frames import QueueFrame, AudioQueueFrame, LLMResponseEndQueueFrame, LLMMessagesQueueFrame from dailyai.pipeline.frames import Frame, AudioFrame, LLMResponseEndFrame, LLMMessagesQueueFrame
from typing import AsyncGenerator from typing import AsyncGenerator
from examples.foundational.support.runner import configure from examples.foundational.support.runner import configure
@@ -34,9 +34,9 @@ class OutboundSoundEffectWrapper(AIService):
def __init__(self): def __init__(self):
pass pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, LLMResponseEndQueueFrame): if isinstance(frame, LLMResponseEndFrame):
yield AudioQueueFrame(sounds["ding1.wav"]) yield AudioFrame(sounds["ding1.wav"])
# In case anything else up the stack needs it # In case anything else up the stack needs it
yield frame yield frame
else: else:
@@ -47,9 +47,9 @@ class InboundSoundEffectWrapper(AIService):
def __init__(self): def __init__(self):
pass pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, LLMMessagesQueueFrame): if isinstance(frame, LLMMessagesQueueFrame):
yield AudioQueueFrame(sounds["ding2.wav"]) yield AudioFrame(sounds["ding2.wav"])
# In case anything else up the stack needs it # In case anything else up the stack needs it
yield frame yield frame
else: else:
@@ -79,7 +79,7 @@ async def main(room_url: str, token, phone):
@transport.event_handler("on_first_other_participant_joined") @transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport): async def on_first_other_participant_joined(transport):
await tts.say("Hi, I'm listening!", transport.send_queue) await tts.say("Hi, I'm listening!", transport.send_queue)
await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"])) await transport.send_queue.put(AudioFrame(sounds["ding1.wav"]))
async def handle_transcriptions(): async def handle_transcriptions():
messages = [ messages = [