Merge pull request #93 from daily-co/frame-name-cleanup

Cleanup the last few badly-named Frame types
This commit is contained in:
Moishe Lettvin
2024-03-28 14:25:59 -04:00
committed by GitHub
26 changed files with 64 additions and 64 deletions

View File

@@ -4,7 +4,7 @@ import logging
import aiohttp import aiohttp
from dailyai.pipeline.frames import EndFrame, LLMMessagesQueueFrame from dailyai.pipeline.frames import EndFrame, LLMMessagesFrame
from dailyai.pipeline.pipeline import Pipeline from dailyai.pipeline.pipeline import Pipeline
from dailyai.transports.daily_transport import DailyTransport from dailyai.transports.daily_transport import DailyTransport
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
@@ -49,7 +49,7 @@ async def main(room_url):
@transport.event_handler("on_first_other_participant_joined") @transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport): async def on_first_other_participant_joined(transport):
await pipeline.queue_frames([LLMMessagesQueueFrame(messages), EndFrame()]) await pipeline.queue_frames([LLMMessagesFrame(messages), EndFrame()])
await transport.run(pipeline) await transport.run(pipeline)

View File

@@ -9,7 +9,7 @@ from dailyai.pipeline.pipeline import Pipeline
from dailyai.transports.daily_transport import DailyTransport from dailyai.transports.daily_transport import DailyTransport
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.deepgram_ai_services import DeepgramTTSService from dailyai.services.deepgram_ai_services import DeepgramTTSService
from dailyai.pipeline.frames import EndPipeFrame, LLMMessagesQueueFrame, TextFrame from dailyai.pipeline.frames import EndPipeFrame, LLMMessagesFrame, TextFrame
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from runner import configure from runner import configure
@@ -60,7 +60,7 @@ async def main(room_url: str):
# will run in parallel with generating and speaking the audio for static text, so there's no delay to # will run in parallel with generating and speaking the audio for static text, so there's no delay to
# speak the LLM response. # speak the LLM response.
llm_pipeline = Pipeline([llm, elevenlabs_tts]) llm_pipeline = Pipeline([llm, elevenlabs_tts])
await llm_pipeline.queue_frames([LLMMessagesQueueFrame(messages), EndPipeFrame()]) await llm_pipeline.queue_frames([LLMMessagesFrame(messages), EndPipeFrame()])
simple_tts_pipeline = Pipeline([azure_tts]) simple_tts_pipeline = Pipeline([azure_tts])
await simple_tts_pipeline.queue_frames( await simple_tts_pipeline.queue_frames(

View File

@@ -17,7 +17,7 @@ from dailyai.pipeline.frames import (
TextFrame, TextFrame,
EndFrame, EndFrame,
ImageFrame, ImageFrame,
LLMMessagesQueueFrame, LLMMessagesFrame,
LLMResponseStartFrame, LLMResponseStartFrame,
) )
from dailyai.pipeline.frame_processor import FrameProcessor from dailyai.pipeline.frame_processor import FrameProcessor
@@ -133,7 +133,7 @@ async def main(room_url):
} }
] ]
frames.append(MonthFrame(month)) frames.append(MonthFrame(month))
frames.append(LLMMessagesQueueFrame(messages)) frames.append(LLMMessagesFrame(messages))
frames.append(EndFrame()) frames.append(EndFrame())
await pipeline.queue_frames(frames) await pipeline.queue_frames(frames)

View File

@@ -2,7 +2,7 @@ import asyncio
import aiohttp import aiohttp
import logging import logging
import os import os
from dailyai.pipeline.frames import LLMMessagesQueueFrame from dailyai.pipeline.frames import LLMMessagesFrame
from dailyai.pipeline.pipeline import Pipeline from dailyai.pipeline.pipeline import Pipeline
from dailyai.transports.daily_transport import DailyTransport from dailyai.transports.daily_transport import DailyTransport
@@ -76,7 +76,7 @@ async def main(room_url: str, token):
# Kick off the conversation. # Kick off the conversation.
messages.append( messages.append(
{"role": "system", "content": "Please introduce yourself to the user."}) {"role": "system", "content": "Please introduce yourself to the user."})
await pipeline.queue_frames([LLMMessagesQueueFrame(messages)]) await pipeline.queue_frames([LLMMessagesFrame(messages)])
transport.transcription_settings["extra"]["endpointing"] = True transport.transcription_settings["extra"]["endpointing"] = True
transport.transcription_settings["extra"]["punctuate"] = True transport.transcription_settings["extra"]["punctuate"] = True

View File

@@ -10,7 +10,7 @@ from dailyai.transports.daily_transport import DailyTransport
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesQueueFrame, TextFrame from dailyai.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame
from runner import configure from runner import configure
@@ -80,7 +80,7 @@ async def main(room_url: str):
[llm, sentence_aggregator, tts1], source_queue, sink_queue [llm, sentence_aggregator, tts1], source_queue, sink_queue
) )
await source_queue.put(LLMMessagesQueueFrame(messages)) await source_queue.put(LLMMessagesFrame(messages))
await source_queue.put(EndFrame()) await source_queue.put(EndFrame())
await pipeline.run_pipeline() await pipeline.run_pipeline()

View File

@@ -18,7 +18,7 @@ from dailyai.pipeline.frames import (
TextFrame, TextFrame,
ImageFrame, ImageFrame,
SpriteFrame, SpriteFrame,
TranscriptionQueueFrame, TranscriptionFrame,
) )
from dailyai.services.ai_services import AIService from dailyai.services.ai_services import AIService
@@ -76,7 +76,7 @@ class TranscriptFilter(AIService):
self.bot_participant_id = bot_participant_id self.bot_participant_id = bot_participant_id
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, TranscriptionQueueFrame): if isinstance(frame, TranscriptionFrame):
if frame.participantId != self.bot_participant_id: if frame.participantId != self.bot_participant_id:
yield frame yield frame

View File

@@ -16,7 +16,7 @@ from dailyai.pipeline.frames import (
Frame, Frame,
AudioFrame, AudioFrame,
LLMResponseEndFrame, LLMResponseEndFrame,
LLMMessagesQueueFrame, LLMMessagesFrame,
) )
from typing import AsyncGenerator from typing import AsyncGenerator
@@ -62,7 +62,7 @@ class InboundSoundEffectWrapper(AIService):
pass pass
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, LLMMessagesQueueFrame): if isinstance(frame, LLMMessagesFrame):
yield AudioFrame(sounds["ding2.wav"]) yield AudioFrame(sounds["ding2.wav"])
# In case anything else up the stack needs it # In case anything else up the stack needs it
yield frame yield frame

View File

@@ -1,7 +1,7 @@
import argparse import argparse
import asyncio import asyncio
import logging import logging
from dailyai.pipeline.frames import EndFrame, TranscriptionQueueFrame from dailyai.pipeline.frames import EndFrame, TranscriptionFrame
from dailyai.transports.local_transport import LocalTransport from dailyai.transports.local_transport import LocalTransport
from dailyai.services.whisper_ai_services import WhisperSTTService from dailyai.services.whisper_ai_services import WhisperSTTService
@@ -32,7 +32,7 @@ async def main(room_url: str):
while not transport_done.is_set(): while not transport_done.is_set():
item = await transcription_output_queue.get() item = await transcription_output_queue.get()
print("got item from queue", item) print("got item from queue", item)
if isinstance(item, TranscriptionQueueFrame): if isinstance(item, TranscriptionFrame):
print(item.text) print(item.text)
elif isinstance(item, EndFrame): elif isinstance(item, EndFrame):
break break

View File

@@ -3,7 +3,7 @@ import aiohttp
import logging import logging
import os import os
from dailyai.pipeline.frame_processor import FrameProcessor from dailyai.pipeline.frame_processor import FrameProcessor
from dailyai.pipeline.frames import TextFrame, TranscriptionQueueFrame from dailyai.pipeline.frames import TextFrame, TranscriptionFrame
from dailyai.pipeline.pipeline import Pipeline from dailyai.pipeline.pipeline import Pipeline
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.transports.websocket_transport import WebsocketTransport from dailyai.transports.websocket_transport import WebsocketTransport
@@ -16,7 +16,7 @@ logger.setLevel(logging.DEBUG)
class WhisperTranscriber(FrameProcessor): class WhisperTranscriber(FrameProcessor):
async def process_frame(self, frame): async def process_frame(self, frame):
if isinstance(frame, TranscriptionQueueFrame): if isinstance(frame, TranscriptionFrame):
print(f"Transcribed: {frame.text}") print(f"Transcribed: {frame.text}")
else: else:
yield frame yield frame

View File

@@ -7,7 +7,7 @@ from dailyai.transports.daily_transport import DailyTransport
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.pipeline.aggregators import LLMContextAggregator from dailyai.pipeline.aggregators import LLMContextAggregator
from dailyai.services.ai_services import AIService, FrameLogger from dailyai.services.ai_services import AIService, FrameLogger
from dailyai.pipeline.frames import Frame, AudioFrame, LLMResponseEndFrame, LLMMessagesQueueFrame from dailyai.pipeline.frames import Frame, AudioFrame, LLMResponseEndFrame, LLMMessagesFrame
from typing import AsyncGenerator from typing import AsyncGenerator
from runner import configure from runner import configure
@@ -51,7 +51,7 @@ class InboundSoundEffectWrapper(AIService):
pass pass
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, LLMMessagesQueueFrame): if isinstance(frame, LLMMessagesFrame):
yield AudioFrame(sounds["ding2.wav"]) yield AudioFrame(sounds["ding2.wav"])
# In case anything else up the stack needs it # In case anything else up the stack needs it
yield frame yield frame

View File

@@ -14,7 +14,7 @@ from dailyai.pipeline.frames import (
SpriteFrame, SpriteFrame,
Frame, Frame,
LLMResponseEndFrame, LLMResponseEndFrame,
LLMMessagesQueueFrame, LLMMessagesFrame,
AudioFrame, AudioFrame,
PipelineStartedFrame, PipelineStartedFrame,
) )
@@ -129,7 +129,7 @@ async def main(room_url: str, token):
@transport.event_handler("on_first_other_participant_joined") @transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport): async def on_first_other_participant_joined(transport):
print(f"!!! in here, pipeline.source is {pipeline.source}") print(f"!!! in here, pipeline.source is {pipeline.source}")
await pipeline.queue_frames([LLMMessagesQueueFrame(messages)]) await pipeline.queue_frames([LLMMessagesFrame(messages)])
async def run_conversation(): async def run_conversation():

View File

@@ -24,7 +24,7 @@ from dailyai.pipeline.aggregators import (
) )
from dailyai.pipeline.frames import ( from dailyai.pipeline.frames import (
EndPipeFrame, EndPipeFrame,
LLMMessagesQueueFrame, LLMMessagesFrame,
Frame, Frame,
TextFrame, TextFrame,
LLMResponseEndFrame, LLMResponseEndFrame,
@@ -172,7 +172,7 @@ class StoryImageGenerator(FrameProcessor):
prompt = f"You are an illustrator for a children's story book. Here is the story so far:\n\n\"{' '.join(self._story[:-1])}\"\n\nGenerate a prompt for DALL-E to create an illustration for the next page. Here's the sentence for the next page:\n\n\"{self._story[-1:][0]}\"\n\n Your response should start with the phrase \"Children's book illustration of\"." prompt = f"You are an illustrator for a children's story book. Here is the story so far:\n\n\"{' '.join(self._story[:-1])}\"\n\nGenerate a prompt for DALL-E to create an illustration for the next page. Here's the sentence for the next page:\n\n\"{self._story[-1:][0]}\"\n\n Your response should start with the phrase \"Children's book illustration of\"."
msgs = [{"role": "system", "content": prompt}] msgs = [{"role": "system", "content": prompt}]
image_prompt = "" image_prompt = ""
async for f in self._llm.process_frame(LLMMessagesQueueFrame(msgs)): async for f in self._llm.process_frame(LLMMessagesFrame(msgs)):
if isinstance(f, TextFrame): if isinstance(f, TextFrame):
image_prompt += f.text image_prompt += f.text
async for f in self._img.process_frame(TextFrame(image_prompt)): async for f in self._img.process_frame(TextFrame(image_prompt)):
@@ -253,7 +253,7 @@ async def main(room_url: str, token):
await local_pipeline.queue_frames( await local_pipeline.queue_frames(
[ [
ImageFrame(None, images["grandma-listening.png"]), ImageFrame(None, images["grandma-listening.png"]),
LLMMessagesQueueFrame(intro_messages), LLMMessagesFrame(intro_messages),
AudioFrame(sounds["listening.wav"]), AudioFrame(sounds["listening.wav"]),
EndPipeFrame(), EndPipeFrame(),
] ]

View File

@@ -7,7 +7,7 @@ from typing import AsyncGenerator
from dailyai.pipeline.aggregators import ( from dailyai.pipeline.aggregators import (
SentenceAggregator, SentenceAggregator,
) )
from dailyai.pipeline.frames import Frame, LLMMessagesQueueFrame, TextFrame from dailyai.pipeline.frames import Frame, LLMMessagesFrame, TextFrame
from dailyai.pipeline.frame_processor import FrameProcessor from dailyai.pipeline.frame_processor import FrameProcessor
from dailyai.pipeline.pipeline import Pipeline from dailyai.pipeline.pipeline import Pipeline
from dailyai.transports.daily_transport import DailyTransport from dailyai.transports.daily_transport import DailyTransport
@@ -44,7 +44,7 @@ class TranslationProcessor(FrameProcessor):
}, },
{"role": "user", "content": frame.text}, {"role": "user", "content": frame.text},
] ]
yield LLMMessagesQueueFrame(context) yield LLMMessagesFrame(context)
else: else:
yield frame yield frame

View File

@@ -7,11 +7,11 @@ from dailyai.pipeline.frames import (
EndFrame, EndFrame,
EndPipeFrame, EndPipeFrame,
Frame, Frame,
LLMMessagesQueueFrame, LLMMessagesFrame,
LLMResponseEndFrame, LLMResponseEndFrame,
LLMResponseStartFrame, LLMResponseStartFrame,
TextFrame, TextFrame,
TranscriptionQueueFrame, TranscriptionFrame,
UserStartedSpeakingFrame, UserStartedSpeakingFrame,
UserStoppedSpeakingFrame, UserStoppedSpeakingFrame,
) )
@@ -57,7 +57,7 @@ class ResponseAggregator(FrameProcessor):
{"role": self._role, "content": self.aggregation}) {"role": self._role, "content": self.aggregation})
self.aggregation = "" self.aggregation = ""
yield self._end_frame() yield self._end_frame()
yield LLMMessagesQueueFrame(self.messages) yield LLMMessagesFrame(self.messages)
elif isinstance(frame, self._accumulator_frame) and self.aggregating: elif isinstance(frame, self._accumulator_frame) and self.aggregating:
self.aggregation += f" {frame.text}" self.aggregation += f" {frame.text}"
if self._pass_through: if self._pass_through:
@@ -84,7 +84,7 @@ class UserResponseAggregator(ResponseAggregator):
role="user", role="user",
start_frame=UserStartedSpeakingFrame, start_frame=UserStartedSpeakingFrame,
end_frame=UserStoppedSpeakingFrame, end_frame=UserStoppedSpeakingFrame,
accumulator_frame=TranscriptionQueueFrame, accumulator_frame=TranscriptionFrame,
pass_through=False, pass_through=False,
) )
@@ -114,7 +114,7 @@ class LLMContextAggregator(AIService):
return return
# Ignore transcription frames from the bot # Ignore transcription frames from the bot
if isinstance(frame, TranscriptionQueueFrame): if isinstance(frame, TranscriptionFrame):
if frame.participantId == self.bot_participant_id: if frame.participantId == self.bot_participant_id:
return return
@@ -126,19 +126,19 @@ class LLMContextAggregator(AIService):
# TODO: split up transcription by participant # TODO: split up transcription by participant
if self.complete_sentences: if self.complete_sentences:
# type: ignore -- the linter thinks this isn't a TextQueueFrame, even # type: ignore -- the linter thinks this isn't a TextFrame, even
# though we check it above # though we check it above
self.sentence += frame.text self.sentence += frame.text
if self.sentence.endswith((".", "?", "!")): if self.sentence.endswith((".", "?", "!")):
self.messages.append( self.messages.append(
{"role": self.role, "content": self.sentence}) {"role": self.role, "content": self.sentence})
self.sentence = "" self.sentence = ""
yield LLMMessagesQueueFrame(self.messages) yield LLMMessagesFrame(self.messages)
else: else:
# type: ignore -- the linter thinks this isn't a TextQueueFrame, even # type: ignore -- the linter thinks this isn't a TextFrame, even
# though we check it above # though we check it above
self.messages.append({"role": self.role, "content": frame.text}) self.messages.append({"role": self.role, "content": frame.text})
yield LLMMessagesQueueFrame(self.messages) yield LLMMessagesFrame(self.messages)
class LLMUserContextAggregator(LLMContextAggregator): class LLMUserContextAggregator(LLMContextAggregator):
@@ -334,7 +334,7 @@ class ParallelPipeline(FrameProcessor):
continue continue
seen_ids.add(id(frame)) seen_ids.add(id(frame))
# Skip passing along EndParallelPipeQueueFrame, because we use them # Skip passing along EndPipeFrame, because we use them
# for our own flow control. # for our own flow control.
if not isinstance(frame, EndPipeFrame): if not isinstance(frame, EndPipeFrame):
yield frame yield frame

View File

@@ -12,9 +12,9 @@ class FrameProcessor:
By convention, FrameProcessors should immediately yield any frames they don't process. By convention, FrameProcessors should immediately yield any frames they don't process.
Stateful FrameProcessors should watch for the EndStreamQueueFrame and finalize their Stateful FrameProcessors should watch for the EndFrame and finalize their
output, eg. yielding an unfinished sentence if they're aggregating LLM output to full output, eg. yielding an unfinished sentence if they're aggregating LLM output to full
sentences. EndStreamQueueFrame is also a chance to clean up any services that need to sentences. EndFrame is also a chance to clean up any services that need to
be closed, del'd, etc. be closed, del'd, etc.
""" """

View File

@@ -102,7 +102,7 @@ class TextFrame(Frame):
@dataclass() @dataclass()
class TranscriptionQueueFrame(TextFrame): class TranscriptionFrame(TextFrame):
"""A text frame with transcription-specific data. Will be placed in the """A text frame with transcription-specific data. Will be placed in the
transport's receive queue when a participant speaks.""" transport's receive queue when a participant speaks."""
participantId: str participantId: str
@@ -126,7 +126,7 @@ class TTSEndFrame(ControlFrame):
@dataclass() @dataclass()
class LLMMessagesQueueFrame(Frame): class LLMMessagesFrame(Frame):
"""A frame containing a list of LLM messages. Used to signal that an LLM """A frame containing a list of LLM messages. Used to signal that an LLM
service should run a chat completion and emit an LLMStartFrames, TextFrames service should run a chat completion and emit an LLMStartFrames, TextFrames
and an LLMEndFrame. and an LLMEndFrame.
@@ -137,7 +137,7 @@ class LLMMessagesQueueFrame(Frame):
@dataclass() @dataclass()
class OpenAILLMContextFrame(Frame): class OpenAILLMContextFrame(Frame):
"""Like an LLMMessagesQueueFrame, but with extra context specific to the """Like an LLMMessagesFrame, but with extra context specific to the
OpenAI API. The context in this message is also mutable, and will be OpenAI API. The context in this message is also mutable, and will be
changed by the OpenAIContextAggregator frame processor.""" changed by the OpenAIContextAggregator frame processor."""
context: OpenAILLMContext context: OpenAILLMContext

View File

@@ -6,7 +6,7 @@ from dailyai.pipeline.frames import (
LLMResponseStartFrame, LLMResponseStartFrame,
OpenAILLMContextFrame, OpenAILLMContextFrame,
TextFrame, TextFrame,
TranscriptionQueueFrame, TranscriptionFrame,
UserStartedSpeakingFrame, UserStartedSpeakingFrame,
UserStoppedSpeakingFrame, UserStoppedSpeakingFrame,
) )
@@ -90,7 +90,7 @@ class OpenAIUserContextAggregator(OpenAIContextAggregator):
role="user", role="user",
start_frame=UserStartedSpeakingFrame, start_frame=UserStartedSpeakingFrame,
end_frame=UserStoppedSpeakingFrame, end_frame=UserStoppedSpeakingFrame,
accumulator_frame=TranscriptionQueueFrame, accumulator_frame=TranscriptionFrame,
pass_through=False, pass_through=False,
) )

View File

@@ -81,8 +81,8 @@ class Pipeline:
The source and sink queues must be set before calling this method. The source and sink queues must be set before calling this method.
This method will exit when an EndStreamQueueFrame is placed on the sink queue. This method will exit when an EndFrame is placed on the sink queue.
No more frames will be placed on the sink queue after an EndStreamQueueFrame, even No more frames will be placed on the sink queue after an EndFrame, even
if it's not the last frame yielded by the last frame_processor in the pipeline.. if it's not the last frame yielded by the last frame_processor in the pipeline..
""" """

View File

@@ -1,6 +1,6 @@
import dataclasses import dataclasses
from typing import Text from typing import Text
from dailyai.pipeline.frames import AudioFrame, Frame, TextFrame, TranscriptionQueueFrame from dailyai.pipeline.frames import AudioFrame, Frame, TextFrame, TranscriptionFrame
import dailyai.pipeline.protobufs.frames_pb2 as frame_protos import dailyai.pipeline.protobufs.frames_pb2 as frame_protos
from dailyai.serializers.abstract_frame_serializer import FrameSerializer from dailyai.serializers.abstract_frame_serializer import FrameSerializer
@@ -9,7 +9,7 @@ class ProtobufFrameSerializer(FrameSerializer):
SERIALIZABLE_TYPES = { SERIALIZABLE_TYPES = {
TextFrame: "text", TextFrame: "text",
AudioFrame: "audio", AudioFrame: "audio",
TranscriptionQueueFrame: "transcription" TranscriptionFrame: "transcription"
} }
SERIALIZABLE_FIELDS = {v: k for k, v in SERIALIZABLE_TYPES.items()} SERIALIZABLE_FIELDS = {v: k for k, v in SERIALIZABLE_TYPES.items()}
@@ -45,9 +45,9 @@ class ProtobufFrameSerializer(FrameSerializer):
... serializer.serialize(TextFrame(text='hello world'))) ... serializer.serialize(TextFrame(text='hello world')))
TextFrame(text='hello world') TextFrame(text='hello world')
>>> serializer.deserialize(serializer.serialize(TranscriptionQueueFrame( >>> serializer.deserialize(serializer.serialize(TranscriptionFrame(
... text="Hello there!", participantId="123", timestamp="2021-01-01"))) ... text="Hello there!", participantId="123", timestamp="2021-01-01")))
TranscriptionQueueFrame(text='Hello there!', participantId='123', timestamp='2021-01-01') TranscriptionFrame(text='Hello there!', participantId='123', timestamp='2021-01-01')
""" """
proto = frame_protos.Frame.FromString(data) proto = frame_protos.Frame.FromString(data)

View File

@@ -13,7 +13,7 @@ from dailyai.pipeline.frames import (
TTSEndFrame, TTSEndFrame,
TTSStartFrame, TTSStartFrame,
TextFrame, TextFrame,
TranscriptionQueueFrame, TranscriptionFrame,
) )
from abc import abstractmethod from abc import abstractmethod
@@ -128,7 +128,7 @@ class STTService(AIService):
ww.close() ww.close()
content.seek(0) content.seek(0)
text = await self.run_stt(content) text = await self.run_stt(content)
yield TranscriptionQueueFrame(text, "", str(time.time())) yield TranscriptionFrame(text, "", str(time.time()))
class FrameLogger(AIService): class FrameLogger(AIService):

View File

@@ -1,6 +1,6 @@
from typing import AsyncGenerator from typing import AsyncGenerator
from anthropic import AsyncAnthropic from anthropic import AsyncAnthropic
from dailyai.pipeline.frames import Frame, LLMMessagesQueueFrame, TextFrame from dailyai.pipeline.frames import Frame, LLMMessagesFrame, TextFrame
from dailyai.services.ai_services import LLMService from dailyai.services.ai_services import LLMService
@@ -18,7 +18,7 @@ class AnthropicLLMService(LLMService):
self.max_tokens = max_tokens self.max_tokens = max_tokens
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if not isinstance(frame, LLMMessagesQueueFrame): if not isinstance(frame, LLMMessagesFrame):
yield frame yield frame
stream = await self.client.messages.create( stream = await self.client.messages.create(

View File

@@ -4,7 +4,7 @@ import math
import time import time
from typing import AsyncGenerator from typing import AsyncGenerator
import wave import wave
from dailyai.pipeline.frames import AudioFrame, Frame, TranscriptionQueueFrame from dailyai.pipeline.frames import AudioFrame, Frame, TranscriptionFrame
from dailyai.services.ai_services import STTService from dailyai.services.ai_services import STTService
@@ -61,7 +61,7 @@ class LocalSTTService(STTService):
self._content.seek(0) self._content.seek(0)
text = await self.run_stt(self._content) text = await self.run_stt(self._content)
self._new_wave() self._new_wave()
yield TranscriptionQueueFrame(text, '', str(time.time())) yield TranscriptionFrame(text, '', str(time.time()))
# If we get this far, this is a frame of silence # If we get this far, this is a frame of silence
self._current_silence_frames += 1 self._current_silence_frames += 1

View File

@@ -6,7 +6,7 @@ from dailyai.pipeline.frames import (
Frame, Frame,
LLMFunctionCallFrame, LLMFunctionCallFrame,
LLMFunctionStartFrame, LLMFunctionStartFrame,
LLMMessagesQueueFrame, LLMMessagesFrame,
LLMResponseEndFrame, LLMResponseEndFrame,
LLMResponseStartFrame, LLMResponseStartFrame,
OpenAILLMContextFrame, OpenAILLMContextFrame,
@@ -75,7 +75,7 @@ class BaseOpenAILLMService(LLMService):
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, OpenAILLMContextFrame): if isinstance(frame, OpenAILLMContextFrame):
context: OpenAILLMContext = frame.context context: OpenAILLMContext = frame.context
elif isinstance(frame, LLMMessagesQueueFrame): elif isinstance(frame, LLMMessagesFrame):
context = OpenAILLMContext.from_messages(frame.messages) context = OpenAILLMContext.from_messages(frame.messages)
else: else:
yield frame yield frame

View File

@@ -10,7 +10,7 @@ from typing import Any
from dailyai.pipeline.frames import ( from dailyai.pipeline.frames import (
ReceivedAppMessageFrame, ReceivedAppMessageFrame,
TranscriptionQueueFrame, TranscriptionFrame,
) )
from threading import Event from threading import Event
@@ -269,7 +269,7 @@ class DailyTransport(ThreadedTransport, EventHandler):
elif "session_id" in message: elif "session_id" in message:
participantId = message["session_id"] participantId = message["session_id"]
if self._my_participant_id and participantId != self._my_participant_id: if self._my_participant_id and participantId != self._my_participant_id:
frame = TranscriptionQueueFrame( frame = TranscriptionFrame(
message["text"], participantId, message["timestamp"]) message["text"], participantId, message["timestamp"])
asyncio.run_coroutine_threadsafe( asyncio.run_coroutine_threadsafe(
self.receive_queue.put(frame), self._loop) self.receive_queue.put(frame), self._loop)

View File

@@ -65,10 +65,10 @@ class TestDailyTransport(unittest.IsolatedAsyncioTestCase):
daily_mock.create_camera_device.return_value = camera daily_mock.create_camera_device.return_value = camera
async def send_audio_frame(): async def send_audio_frame():
await transport.send_queue.put(AudioQueueFrame(bytes([0] * 3300))) await transport.send_queue.put(AudioFrame(bytes([0] * 3300)))
async def send_video_frame(): async def send_video_frame():
await transport.send_queue.put(ImageQueueFrame(None, b"test")) await transport.send_queue.put(ImageFrame(None, b"test"))
await asyncio.gather(transport.run(), send_audio_frame(), send_video_frame()) await asyncio.gather(transport.run(), send_audio_frame(), send_video_frame())

View File

@@ -1,6 +1,6 @@
import unittest import unittest
from dailyai.pipeline.frames import AudioFrame, TextFrame, TranscriptionQueueFrame from dailyai.pipeline.frames import AudioFrame, TextFrame, TranscriptionFrame
from dailyai.serializers.protobuf_serializer import ProtobufFrameSerializer from dailyai.serializers.protobuf_serializer import ProtobufFrameSerializer
@@ -14,7 +14,7 @@ class TestProtobufFrameSerializer(unittest.IsolatedAsyncioTestCase):
self.serializer.serialize(text_frame)) self.serializer.serialize(text_frame))
self.assertEqual(frame, TextFrame(text='hello world')) self.assertEqual(frame, TextFrame(text='hello world'))
transcription_frame = TranscriptionQueueFrame( transcription_frame = TranscriptionFrame(
text="Hello there!", participantId="123", timestamp="2021-01-01") text="Hello there!", participantId="123", timestamp="2021-01-01")
frame = self.serializer.deserialize( frame = self.serializer.deserialize(
self.serializer.serialize(transcription_frame)) self.serializer.serialize(transcription_frame))