Merge pull request #93 from daily-co/frame-name-cleanup
Cleanup the last few badly-named Frame types
This commit is contained in:
@@ -4,7 +4,7 @@ import logging
|
|||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
|
||||||
from dailyai.pipeline.frames import EndFrame, LLMMessagesQueueFrame
|
from dailyai.pipeline.frames import EndFrame, LLMMessagesFrame
|
||||||
from dailyai.pipeline.pipeline import Pipeline
|
from dailyai.pipeline.pipeline import Pipeline
|
||||||
from dailyai.transports.daily_transport import DailyTransport
|
from dailyai.transports.daily_transport import DailyTransport
|
||||||
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
||||||
@@ -49,7 +49,7 @@ async def main(room_url):
|
|||||||
|
|
||||||
@transport.event_handler("on_first_other_participant_joined")
|
@transport.event_handler("on_first_other_participant_joined")
|
||||||
async def on_first_other_participant_joined(transport):
|
async def on_first_other_participant_joined(transport):
|
||||||
await pipeline.queue_frames([LLMMessagesQueueFrame(messages), EndFrame()])
|
await pipeline.queue_frames([LLMMessagesFrame(messages), EndFrame()])
|
||||||
|
|
||||||
await transport.run(pipeline)
|
await transport.run(pipeline)
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from dailyai.pipeline.pipeline import Pipeline
|
|||||||
from dailyai.transports.daily_transport import DailyTransport
|
from dailyai.transports.daily_transport import DailyTransport
|
||||||
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
||||||
from dailyai.services.deepgram_ai_services import DeepgramTTSService
|
from dailyai.services.deepgram_ai_services import DeepgramTTSService
|
||||||
from dailyai.pipeline.frames import EndPipeFrame, LLMMessagesQueueFrame, TextFrame
|
from dailyai.pipeline.frames import EndPipeFrame, LLMMessagesFrame, TextFrame
|
||||||
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
||||||
|
|
||||||
from runner import configure
|
from runner import configure
|
||||||
@@ -60,7 +60,7 @@ async def main(room_url: str):
|
|||||||
# will run in parallel with generating and speaking the audio for static text, so there's no delay to
|
# will run in parallel with generating and speaking the audio for static text, so there's no delay to
|
||||||
# speak the LLM response.
|
# speak the LLM response.
|
||||||
llm_pipeline = Pipeline([llm, elevenlabs_tts])
|
llm_pipeline = Pipeline([llm, elevenlabs_tts])
|
||||||
await llm_pipeline.queue_frames([LLMMessagesQueueFrame(messages), EndPipeFrame()])
|
await llm_pipeline.queue_frames([LLMMessagesFrame(messages), EndPipeFrame()])
|
||||||
|
|
||||||
simple_tts_pipeline = Pipeline([azure_tts])
|
simple_tts_pipeline = Pipeline([azure_tts])
|
||||||
await simple_tts_pipeline.queue_frames(
|
await simple_tts_pipeline.queue_frames(
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ from dailyai.pipeline.frames import (
|
|||||||
TextFrame,
|
TextFrame,
|
||||||
EndFrame,
|
EndFrame,
|
||||||
ImageFrame,
|
ImageFrame,
|
||||||
LLMMessagesQueueFrame,
|
LLMMessagesFrame,
|
||||||
LLMResponseStartFrame,
|
LLMResponseStartFrame,
|
||||||
)
|
)
|
||||||
from dailyai.pipeline.frame_processor import FrameProcessor
|
from dailyai.pipeline.frame_processor import FrameProcessor
|
||||||
@@ -133,7 +133,7 @@ async def main(room_url):
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
frames.append(MonthFrame(month))
|
frames.append(MonthFrame(month))
|
||||||
frames.append(LLMMessagesQueueFrame(messages))
|
frames.append(LLMMessagesFrame(messages))
|
||||||
|
|
||||||
frames.append(EndFrame())
|
frames.append(EndFrame())
|
||||||
await pipeline.queue_frames(frames)
|
await pipeline.queue_frames(frames)
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import asyncio
|
|||||||
import aiohttp
|
import aiohttp
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
from dailyai.pipeline.frames import LLMMessagesQueueFrame
|
from dailyai.pipeline.frames import LLMMessagesFrame
|
||||||
from dailyai.pipeline.pipeline import Pipeline
|
from dailyai.pipeline.pipeline import Pipeline
|
||||||
|
|
||||||
from dailyai.transports.daily_transport import DailyTransport
|
from dailyai.transports.daily_transport import DailyTransport
|
||||||
@@ -76,7 +76,7 @@ async def main(room_url: str, token):
|
|||||||
# Kick off the conversation.
|
# Kick off the conversation.
|
||||||
messages.append(
|
messages.append(
|
||||||
{"role": "system", "content": "Please introduce yourself to the user."})
|
{"role": "system", "content": "Please introduce yourself to the user."})
|
||||||
await pipeline.queue_frames([LLMMessagesQueueFrame(messages)])
|
await pipeline.queue_frames([LLMMessagesFrame(messages)])
|
||||||
|
|
||||||
transport.transcription_settings["extra"]["endpointing"] = True
|
transport.transcription_settings["extra"]["endpointing"] = True
|
||||||
transport.transcription_settings["extra"]["punctuate"] = True
|
transport.transcription_settings["extra"]["punctuate"] = True
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from dailyai.transports.daily_transport import DailyTransport
|
|||||||
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
||||||
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
||||||
from dailyai.services.fal_ai_services import FalImageGenService
|
from dailyai.services.fal_ai_services import FalImageGenService
|
||||||
from dailyai.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesQueueFrame, TextFrame
|
from dailyai.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame
|
||||||
|
|
||||||
from runner import configure
|
from runner import configure
|
||||||
|
|
||||||
@@ -80,7 +80,7 @@ async def main(room_url: str):
|
|||||||
[llm, sentence_aggregator, tts1], source_queue, sink_queue
|
[llm, sentence_aggregator, tts1], source_queue, sink_queue
|
||||||
)
|
)
|
||||||
|
|
||||||
await source_queue.put(LLMMessagesQueueFrame(messages))
|
await source_queue.put(LLMMessagesFrame(messages))
|
||||||
await source_queue.put(EndFrame())
|
await source_queue.put(EndFrame())
|
||||||
await pipeline.run_pipeline()
|
await pipeline.run_pipeline()
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ from dailyai.pipeline.frames import (
|
|||||||
TextFrame,
|
TextFrame,
|
||||||
ImageFrame,
|
ImageFrame,
|
||||||
SpriteFrame,
|
SpriteFrame,
|
||||||
TranscriptionQueueFrame,
|
TranscriptionFrame,
|
||||||
)
|
)
|
||||||
from dailyai.services.ai_services import AIService
|
from dailyai.services.ai_services import AIService
|
||||||
|
|
||||||
@@ -76,7 +76,7 @@ class TranscriptFilter(AIService):
|
|||||||
self.bot_participant_id = bot_participant_id
|
self.bot_participant_id = bot_participant_id
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
|
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
|
||||||
if isinstance(frame, TranscriptionQueueFrame):
|
if isinstance(frame, TranscriptionFrame):
|
||||||
if frame.participantId != self.bot_participant_id:
|
if frame.participantId != self.bot_participant_id:
|
||||||
yield frame
|
yield frame
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ from dailyai.pipeline.frames import (
|
|||||||
Frame,
|
Frame,
|
||||||
AudioFrame,
|
AudioFrame,
|
||||||
LLMResponseEndFrame,
|
LLMResponseEndFrame,
|
||||||
LLMMessagesQueueFrame,
|
LLMMessagesFrame,
|
||||||
)
|
)
|
||||||
from typing import AsyncGenerator
|
from typing import AsyncGenerator
|
||||||
|
|
||||||
@@ -62,7 +62,7 @@ class InboundSoundEffectWrapper(AIService):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
|
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
|
||||||
if isinstance(frame, LLMMessagesQueueFrame):
|
if isinstance(frame, LLMMessagesFrame):
|
||||||
yield AudioFrame(sounds["ding2.wav"])
|
yield AudioFrame(sounds["ding2.wav"])
|
||||||
# In case anything else up the stack needs it
|
# In case anything else up the stack needs it
|
||||||
yield frame
|
yield frame
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
from dailyai.pipeline.frames import EndFrame, TranscriptionQueueFrame
|
from dailyai.pipeline.frames import EndFrame, TranscriptionFrame
|
||||||
|
|
||||||
from dailyai.transports.local_transport import LocalTransport
|
from dailyai.transports.local_transport import LocalTransport
|
||||||
from dailyai.services.whisper_ai_services import WhisperSTTService
|
from dailyai.services.whisper_ai_services import WhisperSTTService
|
||||||
@@ -32,7 +32,7 @@ async def main(room_url: str):
|
|||||||
while not transport_done.is_set():
|
while not transport_done.is_set():
|
||||||
item = await transcription_output_queue.get()
|
item = await transcription_output_queue.get()
|
||||||
print("got item from queue", item)
|
print("got item from queue", item)
|
||||||
if isinstance(item, TranscriptionQueueFrame):
|
if isinstance(item, TranscriptionFrame):
|
||||||
print(item.text)
|
print(item.text)
|
||||||
elif isinstance(item, EndFrame):
|
elif isinstance(item, EndFrame):
|
||||||
break
|
break
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import aiohttp
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
from dailyai.pipeline.frame_processor import FrameProcessor
|
from dailyai.pipeline.frame_processor import FrameProcessor
|
||||||
from dailyai.pipeline.frames import TextFrame, TranscriptionQueueFrame
|
from dailyai.pipeline.frames import TextFrame, TranscriptionFrame
|
||||||
from dailyai.pipeline.pipeline import Pipeline
|
from dailyai.pipeline.pipeline import Pipeline
|
||||||
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
||||||
from dailyai.transports.websocket_transport import WebsocketTransport
|
from dailyai.transports.websocket_transport import WebsocketTransport
|
||||||
@@ -16,7 +16,7 @@ logger.setLevel(logging.DEBUG)
|
|||||||
|
|
||||||
class WhisperTranscriber(FrameProcessor):
|
class WhisperTranscriber(FrameProcessor):
|
||||||
async def process_frame(self, frame):
|
async def process_frame(self, frame):
|
||||||
if isinstance(frame, TranscriptionQueueFrame):
|
if isinstance(frame, TranscriptionFrame):
|
||||||
print(f"Transcribed: {frame.text}")
|
print(f"Transcribed: {frame.text}")
|
||||||
else:
|
else:
|
||||||
yield frame
|
yield frame
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from dailyai.transports.daily_transport import DailyTransport
|
|||||||
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
||||||
from dailyai.pipeline.aggregators import LLMContextAggregator
|
from dailyai.pipeline.aggregators import LLMContextAggregator
|
||||||
from dailyai.services.ai_services import AIService, FrameLogger
|
from dailyai.services.ai_services import AIService, FrameLogger
|
||||||
from dailyai.pipeline.frames import Frame, AudioFrame, LLMResponseEndFrame, LLMMessagesQueueFrame
|
from dailyai.pipeline.frames import Frame, AudioFrame, LLMResponseEndFrame, LLMMessagesFrame
|
||||||
from typing import AsyncGenerator
|
from typing import AsyncGenerator
|
||||||
|
|
||||||
from runner import configure
|
from runner import configure
|
||||||
@@ -51,7 +51,7 @@ class InboundSoundEffectWrapper(AIService):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
|
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
|
||||||
if isinstance(frame, LLMMessagesQueueFrame):
|
if isinstance(frame, LLMMessagesFrame):
|
||||||
yield AudioFrame(sounds["ding2.wav"])
|
yield AudioFrame(sounds["ding2.wav"])
|
||||||
# In case anything else up the stack needs it
|
# In case anything else up the stack needs it
|
||||||
yield frame
|
yield frame
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from dailyai.pipeline.frames import (
|
|||||||
SpriteFrame,
|
SpriteFrame,
|
||||||
Frame,
|
Frame,
|
||||||
LLMResponseEndFrame,
|
LLMResponseEndFrame,
|
||||||
LLMMessagesQueueFrame,
|
LLMMessagesFrame,
|
||||||
AudioFrame,
|
AudioFrame,
|
||||||
PipelineStartedFrame,
|
PipelineStartedFrame,
|
||||||
)
|
)
|
||||||
@@ -129,7 +129,7 @@ async def main(room_url: str, token):
|
|||||||
@transport.event_handler("on_first_other_participant_joined")
|
@transport.event_handler("on_first_other_participant_joined")
|
||||||
async def on_first_other_participant_joined(transport):
|
async def on_first_other_participant_joined(transport):
|
||||||
print(f"!!! in here, pipeline.source is {pipeline.source}")
|
print(f"!!! in here, pipeline.source is {pipeline.source}")
|
||||||
await pipeline.queue_frames([LLMMessagesQueueFrame(messages)])
|
await pipeline.queue_frames([LLMMessagesFrame(messages)])
|
||||||
|
|
||||||
async def run_conversation():
|
async def run_conversation():
|
||||||
|
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ from dailyai.pipeline.aggregators import (
|
|||||||
)
|
)
|
||||||
from dailyai.pipeline.frames import (
|
from dailyai.pipeline.frames import (
|
||||||
EndPipeFrame,
|
EndPipeFrame,
|
||||||
LLMMessagesQueueFrame,
|
LLMMessagesFrame,
|
||||||
Frame,
|
Frame,
|
||||||
TextFrame,
|
TextFrame,
|
||||||
LLMResponseEndFrame,
|
LLMResponseEndFrame,
|
||||||
@@ -172,7 +172,7 @@ class StoryImageGenerator(FrameProcessor):
|
|||||||
prompt = f"You are an illustrator for a children's story book. Here is the story so far:\n\n\"{' '.join(self._story[:-1])}\"\n\nGenerate a prompt for DALL-E to create an illustration for the next page. Here's the sentence for the next page:\n\n\"{self._story[-1:][0]}\"\n\n Your response should start with the phrase \"Children's book illustration of\"."
|
prompt = f"You are an illustrator for a children's story book. Here is the story so far:\n\n\"{' '.join(self._story[:-1])}\"\n\nGenerate a prompt for DALL-E to create an illustration for the next page. Here's the sentence for the next page:\n\n\"{self._story[-1:][0]}\"\n\n Your response should start with the phrase \"Children's book illustration of\"."
|
||||||
msgs = [{"role": "system", "content": prompt}]
|
msgs = [{"role": "system", "content": prompt}]
|
||||||
image_prompt = ""
|
image_prompt = ""
|
||||||
async for f in self._llm.process_frame(LLMMessagesQueueFrame(msgs)):
|
async for f in self._llm.process_frame(LLMMessagesFrame(msgs)):
|
||||||
if isinstance(f, TextFrame):
|
if isinstance(f, TextFrame):
|
||||||
image_prompt += f.text
|
image_prompt += f.text
|
||||||
async for f in self._img.process_frame(TextFrame(image_prompt)):
|
async for f in self._img.process_frame(TextFrame(image_prompt)):
|
||||||
@@ -253,7 +253,7 @@ async def main(room_url: str, token):
|
|||||||
await local_pipeline.queue_frames(
|
await local_pipeline.queue_frames(
|
||||||
[
|
[
|
||||||
ImageFrame(None, images["grandma-listening.png"]),
|
ImageFrame(None, images["grandma-listening.png"]),
|
||||||
LLMMessagesQueueFrame(intro_messages),
|
LLMMessagesFrame(intro_messages),
|
||||||
AudioFrame(sounds["listening.wav"]),
|
AudioFrame(sounds["listening.wav"]),
|
||||||
EndPipeFrame(),
|
EndPipeFrame(),
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from typing import AsyncGenerator
|
|||||||
from dailyai.pipeline.aggregators import (
|
from dailyai.pipeline.aggregators import (
|
||||||
SentenceAggregator,
|
SentenceAggregator,
|
||||||
)
|
)
|
||||||
from dailyai.pipeline.frames import Frame, LLMMessagesQueueFrame, TextFrame
|
from dailyai.pipeline.frames import Frame, LLMMessagesFrame, TextFrame
|
||||||
from dailyai.pipeline.frame_processor import FrameProcessor
|
from dailyai.pipeline.frame_processor import FrameProcessor
|
||||||
from dailyai.pipeline.pipeline import Pipeline
|
from dailyai.pipeline.pipeline import Pipeline
|
||||||
from dailyai.transports.daily_transport import DailyTransport
|
from dailyai.transports.daily_transport import DailyTransport
|
||||||
@@ -44,7 +44,7 @@ class TranslationProcessor(FrameProcessor):
|
|||||||
},
|
},
|
||||||
{"role": "user", "content": frame.text},
|
{"role": "user", "content": frame.text},
|
||||||
]
|
]
|
||||||
yield LLMMessagesQueueFrame(context)
|
yield LLMMessagesFrame(context)
|
||||||
else:
|
else:
|
||||||
yield frame
|
yield frame
|
||||||
|
|
||||||
|
|||||||
@@ -7,11 +7,11 @@ from dailyai.pipeline.frames import (
|
|||||||
EndFrame,
|
EndFrame,
|
||||||
EndPipeFrame,
|
EndPipeFrame,
|
||||||
Frame,
|
Frame,
|
||||||
LLMMessagesQueueFrame,
|
LLMMessagesFrame,
|
||||||
LLMResponseEndFrame,
|
LLMResponseEndFrame,
|
||||||
LLMResponseStartFrame,
|
LLMResponseStartFrame,
|
||||||
TextFrame,
|
TextFrame,
|
||||||
TranscriptionQueueFrame,
|
TranscriptionFrame,
|
||||||
UserStartedSpeakingFrame,
|
UserStartedSpeakingFrame,
|
||||||
UserStoppedSpeakingFrame,
|
UserStoppedSpeakingFrame,
|
||||||
)
|
)
|
||||||
@@ -57,7 +57,7 @@ class ResponseAggregator(FrameProcessor):
|
|||||||
{"role": self._role, "content": self.aggregation})
|
{"role": self._role, "content": self.aggregation})
|
||||||
self.aggregation = ""
|
self.aggregation = ""
|
||||||
yield self._end_frame()
|
yield self._end_frame()
|
||||||
yield LLMMessagesQueueFrame(self.messages)
|
yield LLMMessagesFrame(self.messages)
|
||||||
elif isinstance(frame, self._accumulator_frame) and self.aggregating:
|
elif isinstance(frame, self._accumulator_frame) and self.aggregating:
|
||||||
self.aggregation += f" {frame.text}"
|
self.aggregation += f" {frame.text}"
|
||||||
if self._pass_through:
|
if self._pass_through:
|
||||||
@@ -84,7 +84,7 @@ class UserResponseAggregator(ResponseAggregator):
|
|||||||
role="user",
|
role="user",
|
||||||
start_frame=UserStartedSpeakingFrame,
|
start_frame=UserStartedSpeakingFrame,
|
||||||
end_frame=UserStoppedSpeakingFrame,
|
end_frame=UserStoppedSpeakingFrame,
|
||||||
accumulator_frame=TranscriptionQueueFrame,
|
accumulator_frame=TranscriptionFrame,
|
||||||
pass_through=False,
|
pass_through=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -114,7 +114,7 @@ class LLMContextAggregator(AIService):
|
|||||||
return
|
return
|
||||||
|
|
||||||
# Ignore transcription frames from the bot
|
# Ignore transcription frames from the bot
|
||||||
if isinstance(frame, TranscriptionQueueFrame):
|
if isinstance(frame, TranscriptionFrame):
|
||||||
if frame.participantId == self.bot_participant_id:
|
if frame.participantId == self.bot_participant_id:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -126,19 +126,19 @@ class LLMContextAggregator(AIService):
|
|||||||
|
|
||||||
# TODO: split up transcription by participant
|
# TODO: split up transcription by participant
|
||||||
if self.complete_sentences:
|
if self.complete_sentences:
|
||||||
# type: ignore -- the linter thinks this isn't a TextQueueFrame, even
|
# type: ignore -- the linter thinks this isn't a TextFrame, even
|
||||||
# though we check it above
|
# though we check it above
|
||||||
self.sentence += frame.text
|
self.sentence += frame.text
|
||||||
if self.sentence.endswith((".", "?", "!")):
|
if self.sentence.endswith((".", "?", "!")):
|
||||||
self.messages.append(
|
self.messages.append(
|
||||||
{"role": self.role, "content": self.sentence})
|
{"role": self.role, "content": self.sentence})
|
||||||
self.sentence = ""
|
self.sentence = ""
|
||||||
yield LLMMessagesQueueFrame(self.messages)
|
yield LLMMessagesFrame(self.messages)
|
||||||
else:
|
else:
|
||||||
# type: ignore -- the linter thinks this isn't a TextQueueFrame, even
|
# type: ignore -- the linter thinks this isn't a TextFrame, even
|
||||||
# though we check it above
|
# though we check it above
|
||||||
self.messages.append({"role": self.role, "content": frame.text})
|
self.messages.append({"role": self.role, "content": frame.text})
|
||||||
yield LLMMessagesQueueFrame(self.messages)
|
yield LLMMessagesFrame(self.messages)
|
||||||
|
|
||||||
|
|
||||||
class LLMUserContextAggregator(LLMContextAggregator):
|
class LLMUserContextAggregator(LLMContextAggregator):
|
||||||
@@ -334,7 +334,7 @@ class ParallelPipeline(FrameProcessor):
|
|||||||
continue
|
continue
|
||||||
seen_ids.add(id(frame))
|
seen_ids.add(id(frame))
|
||||||
|
|
||||||
# Skip passing along EndParallelPipeQueueFrame, because we use them
|
# Skip passing along EndPipeFrame, because we use them
|
||||||
# for our own flow control.
|
# for our own flow control.
|
||||||
if not isinstance(frame, EndPipeFrame):
|
if not isinstance(frame, EndPipeFrame):
|
||||||
yield frame
|
yield frame
|
||||||
|
|||||||
@@ -12,9 +12,9 @@ class FrameProcessor:
|
|||||||
|
|
||||||
By convention, FrameProcessors should immediately yield any frames they don't process.
|
By convention, FrameProcessors should immediately yield any frames they don't process.
|
||||||
|
|
||||||
Stateful FrameProcessors should watch for the EndStreamQueueFrame and finalize their
|
Stateful FrameProcessors should watch for the EndFrame and finalize their
|
||||||
output, eg. yielding an unfinished sentence if they're aggregating LLM output to full
|
output, eg. yielding an unfinished sentence if they're aggregating LLM output to full
|
||||||
sentences. EndStreamQueueFrame is also a chance to clean up any services that need to
|
sentences. EndFrame is also a chance to clean up any services that need to
|
||||||
be closed, del'd, etc.
|
be closed, del'd, etc.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|||||||
@@ -102,7 +102,7 @@ class TextFrame(Frame):
|
|||||||
|
|
||||||
|
|
||||||
@dataclass()
|
@dataclass()
|
||||||
class TranscriptionQueueFrame(TextFrame):
|
class TranscriptionFrame(TextFrame):
|
||||||
"""A text frame with transcription-specific data. Will be placed in the
|
"""A text frame with transcription-specific data. Will be placed in the
|
||||||
transport's receive queue when a participant speaks."""
|
transport's receive queue when a participant speaks."""
|
||||||
participantId: str
|
participantId: str
|
||||||
@@ -126,7 +126,7 @@ class TTSEndFrame(ControlFrame):
|
|||||||
|
|
||||||
|
|
||||||
@dataclass()
|
@dataclass()
|
||||||
class LLMMessagesQueueFrame(Frame):
|
class LLMMessagesFrame(Frame):
|
||||||
"""A frame containing a list of LLM messages. Used to signal that an LLM
|
"""A frame containing a list of LLM messages. Used to signal that an LLM
|
||||||
service should run a chat completion and emit an LLMStartFrames, TextFrames
|
service should run a chat completion and emit an LLMStartFrames, TextFrames
|
||||||
and an LLMEndFrame.
|
and an LLMEndFrame.
|
||||||
@@ -137,7 +137,7 @@ class LLMMessagesQueueFrame(Frame):
|
|||||||
|
|
||||||
@dataclass()
|
@dataclass()
|
||||||
class OpenAILLMContextFrame(Frame):
|
class OpenAILLMContextFrame(Frame):
|
||||||
"""Like an LLMMessagesQueueFrame, but with extra context specific to the
|
"""Like an LLMMessagesFrame, but with extra context specific to the
|
||||||
OpenAI API. The context in this message is also mutable, and will be
|
OpenAI API. The context in this message is also mutable, and will be
|
||||||
changed by the OpenAIContextAggregator frame processor."""
|
changed by the OpenAIContextAggregator frame processor."""
|
||||||
context: OpenAILLMContext
|
context: OpenAILLMContext
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from dailyai.pipeline.frames import (
|
|||||||
LLMResponseStartFrame,
|
LLMResponseStartFrame,
|
||||||
OpenAILLMContextFrame,
|
OpenAILLMContextFrame,
|
||||||
TextFrame,
|
TextFrame,
|
||||||
TranscriptionQueueFrame,
|
TranscriptionFrame,
|
||||||
UserStartedSpeakingFrame,
|
UserStartedSpeakingFrame,
|
||||||
UserStoppedSpeakingFrame,
|
UserStoppedSpeakingFrame,
|
||||||
)
|
)
|
||||||
@@ -90,7 +90,7 @@ class OpenAIUserContextAggregator(OpenAIContextAggregator):
|
|||||||
role="user",
|
role="user",
|
||||||
start_frame=UserStartedSpeakingFrame,
|
start_frame=UserStartedSpeakingFrame,
|
||||||
end_frame=UserStoppedSpeakingFrame,
|
end_frame=UserStoppedSpeakingFrame,
|
||||||
accumulator_frame=TranscriptionQueueFrame,
|
accumulator_frame=TranscriptionFrame,
|
||||||
pass_through=False,
|
pass_through=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -81,8 +81,8 @@ class Pipeline:
|
|||||||
|
|
||||||
The source and sink queues must be set before calling this method.
|
The source and sink queues must be set before calling this method.
|
||||||
|
|
||||||
This method will exit when an EndStreamQueueFrame is placed on the sink queue.
|
This method will exit when an EndFrame is placed on the sink queue.
|
||||||
No more frames will be placed on the sink queue after an EndStreamQueueFrame, even
|
No more frames will be placed on the sink queue after an EndFrame, even
|
||||||
if it's not the last frame yielded by the last frame_processor in the pipeline..
|
if it's not the last frame yielded by the last frame_processor in the pipeline..
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import dataclasses
|
import dataclasses
|
||||||
from typing import Text
|
from typing import Text
|
||||||
from dailyai.pipeline.frames import AudioFrame, Frame, TextFrame, TranscriptionQueueFrame
|
from dailyai.pipeline.frames import AudioFrame, Frame, TextFrame, TranscriptionFrame
|
||||||
import dailyai.pipeline.protobufs.frames_pb2 as frame_protos
|
import dailyai.pipeline.protobufs.frames_pb2 as frame_protos
|
||||||
from dailyai.serializers.abstract_frame_serializer import FrameSerializer
|
from dailyai.serializers.abstract_frame_serializer import FrameSerializer
|
||||||
|
|
||||||
@@ -9,7 +9,7 @@ class ProtobufFrameSerializer(FrameSerializer):
|
|||||||
SERIALIZABLE_TYPES = {
|
SERIALIZABLE_TYPES = {
|
||||||
TextFrame: "text",
|
TextFrame: "text",
|
||||||
AudioFrame: "audio",
|
AudioFrame: "audio",
|
||||||
TranscriptionQueueFrame: "transcription"
|
TranscriptionFrame: "transcription"
|
||||||
}
|
}
|
||||||
|
|
||||||
SERIALIZABLE_FIELDS = {v: k for k, v in SERIALIZABLE_TYPES.items()}
|
SERIALIZABLE_FIELDS = {v: k for k, v in SERIALIZABLE_TYPES.items()}
|
||||||
@@ -45,9 +45,9 @@ class ProtobufFrameSerializer(FrameSerializer):
|
|||||||
... serializer.serialize(TextFrame(text='hello world')))
|
... serializer.serialize(TextFrame(text='hello world')))
|
||||||
TextFrame(text='hello world')
|
TextFrame(text='hello world')
|
||||||
|
|
||||||
>>> serializer.deserialize(serializer.serialize(TranscriptionQueueFrame(
|
>>> serializer.deserialize(serializer.serialize(TranscriptionFrame(
|
||||||
... text="Hello there!", participantId="123", timestamp="2021-01-01")))
|
... text="Hello there!", participantId="123", timestamp="2021-01-01")))
|
||||||
TranscriptionQueueFrame(text='Hello there!', participantId='123', timestamp='2021-01-01')
|
TranscriptionFrame(text='Hello there!', participantId='123', timestamp='2021-01-01')
|
||||||
"""
|
"""
|
||||||
|
|
||||||
proto = frame_protos.Frame.FromString(data)
|
proto = frame_protos.Frame.FromString(data)
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from dailyai.pipeline.frames import (
|
|||||||
TTSEndFrame,
|
TTSEndFrame,
|
||||||
TTSStartFrame,
|
TTSStartFrame,
|
||||||
TextFrame,
|
TextFrame,
|
||||||
TranscriptionQueueFrame,
|
TranscriptionFrame,
|
||||||
)
|
)
|
||||||
|
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
@@ -128,7 +128,7 @@ class STTService(AIService):
|
|||||||
ww.close()
|
ww.close()
|
||||||
content.seek(0)
|
content.seek(0)
|
||||||
text = await self.run_stt(content)
|
text = await self.run_stt(content)
|
||||||
yield TranscriptionQueueFrame(text, "", str(time.time()))
|
yield TranscriptionFrame(text, "", str(time.time()))
|
||||||
|
|
||||||
|
|
||||||
class FrameLogger(AIService):
|
class FrameLogger(AIService):
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
from typing import AsyncGenerator
|
from typing import AsyncGenerator
|
||||||
from anthropic import AsyncAnthropic
|
from anthropic import AsyncAnthropic
|
||||||
from dailyai.pipeline.frames import Frame, LLMMessagesQueueFrame, TextFrame
|
from dailyai.pipeline.frames import Frame, LLMMessagesFrame, TextFrame
|
||||||
|
|
||||||
from dailyai.services.ai_services import LLMService
|
from dailyai.services.ai_services import LLMService
|
||||||
|
|
||||||
@@ -18,7 +18,7 @@ class AnthropicLLMService(LLMService):
|
|||||||
self.max_tokens = max_tokens
|
self.max_tokens = max_tokens
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
|
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
|
||||||
if not isinstance(frame, LLMMessagesQueueFrame):
|
if not isinstance(frame, LLMMessagesFrame):
|
||||||
yield frame
|
yield frame
|
||||||
|
|
||||||
stream = await self.client.messages.create(
|
stream = await self.client.messages.create(
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import math
|
|||||||
import time
|
import time
|
||||||
from typing import AsyncGenerator
|
from typing import AsyncGenerator
|
||||||
import wave
|
import wave
|
||||||
from dailyai.pipeline.frames import AudioFrame, Frame, TranscriptionQueueFrame
|
from dailyai.pipeline.frames import AudioFrame, Frame, TranscriptionFrame
|
||||||
from dailyai.services.ai_services import STTService
|
from dailyai.services.ai_services import STTService
|
||||||
|
|
||||||
|
|
||||||
@@ -61,7 +61,7 @@ class LocalSTTService(STTService):
|
|||||||
self._content.seek(0)
|
self._content.seek(0)
|
||||||
text = await self.run_stt(self._content)
|
text = await self.run_stt(self._content)
|
||||||
self._new_wave()
|
self._new_wave()
|
||||||
yield TranscriptionQueueFrame(text, '', str(time.time()))
|
yield TranscriptionFrame(text, '', str(time.time()))
|
||||||
# If we get this far, this is a frame of silence
|
# If we get this far, this is a frame of silence
|
||||||
self._current_silence_frames += 1
|
self._current_silence_frames += 1
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from dailyai.pipeline.frames import (
|
|||||||
Frame,
|
Frame,
|
||||||
LLMFunctionCallFrame,
|
LLMFunctionCallFrame,
|
||||||
LLMFunctionStartFrame,
|
LLMFunctionStartFrame,
|
||||||
LLMMessagesQueueFrame,
|
LLMMessagesFrame,
|
||||||
LLMResponseEndFrame,
|
LLMResponseEndFrame,
|
||||||
LLMResponseStartFrame,
|
LLMResponseStartFrame,
|
||||||
OpenAILLMContextFrame,
|
OpenAILLMContextFrame,
|
||||||
@@ -75,7 +75,7 @@ class BaseOpenAILLMService(LLMService):
|
|||||||
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
|
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
|
||||||
if isinstance(frame, OpenAILLMContextFrame):
|
if isinstance(frame, OpenAILLMContextFrame):
|
||||||
context: OpenAILLMContext = frame.context
|
context: OpenAILLMContext = frame.context
|
||||||
elif isinstance(frame, LLMMessagesQueueFrame):
|
elif isinstance(frame, LLMMessagesFrame):
|
||||||
context = OpenAILLMContext.from_messages(frame.messages)
|
context = OpenAILLMContext.from_messages(frame.messages)
|
||||||
else:
|
else:
|
||||||
yield frame
|
yield frame
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from typing import Any
|
|||||||
|
|
||||||
from dailyai.pipeline.frames import (
|
from dailyai.pipeline.frames import (
|
||||||
ReceivedAppMessageFrame,
|
ReceivedAppMessageFrame,
|
||||||
TranscriptionQueueFrame,
|
TranscriptionFrame,
|
||||||
)
|
)
|
||||||
|
|
||||||
from threading import Event
|
from threading import Event
|
||||||
@@ -269,7 +269,7 @@ class DailyTransport(ThreadedTransport, EventHandler):
|
|||||||
elif "session_id" in message:
|
elif "session_id" in message:
|
||||||
participantId = message["session_id"]
|
participantId = message["session_id"]
|
||||||
if self._my_participant_id and participantId != self._my_participant_id:
|
if self._my_participant_id and participantId != self._my_participant_id:
|
||||||
frame = TranscriptionQueueFrame(
|
frame = TranscriptionFrame(
|
||||||
message["text"], participantId, message["timestamp"])
|
message["text"], participantId, message["timestamp"])
|
||||||
asyncio.run_coroutine_threadsafe(
|
asyncio.run_coroutine_threadsafe(
|
||||||
self.receive_queue.put(frame), self._loop)
|
self.receive_queue.put(frame), self._loop)
|
||||||
|
|||||||
@@ -65,10 +65,10 @@ class TestDailyTransport(unittest.IsolatedAsyncioTestCase):
|
|||||||
daily_mock.create_camera_device.return_value = camera
|
daily_mock.create_camera_device.return_value = camera
|
||||||
|
|
||||||
async def send_audio_frame():
|
async def send_audio_frame():
|
||||||
await transport.send_queue.put(AudioQueueFrame(bytes([0] * 3300)))
|
await transport.send_queue.put(AudioFrame(bytes([0] * 3300)))
|
||||||
|
|
||||||
async def send_video_frame():
|
async def send_video_frame():
|
||||||
await transport.send_queue.put(ImageQueueFrame(None, b"test"))
|
await transport.send_queue.put(ImageFrame(None, b"test"))
|
||||||
|
|
||||||
await asyncio.gather(transport.run(), send_audio_frame(), send_video_frame())
|
await asyncio.gather(transport.run(), send_audio_frame(), send_video_frame())
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from dailyai.pipeline.frames import AudioFrame, TextFrame, TranscriptionQueueFrame
|
from dailyai.pipeline.frames import AudioFrame, TextFrame, TranscriptionFrame
|
||||||
from dailyai.serializers.protobuf_serializer import ProtobufFrameSerializer
|
from dailyai.serializers.protobuf_serializer import ProtobufFrameSerializer
|
||||||
|
|
||||||
|
|
||||||
@@ -14,7 +14,7 @@ class TestProtobufFrameSerializer(unittest.IsolatedAsyncioTestCase):
|
|||||||
self.serializer.serialize(text_frame))
|
self.serializer.serialize(text_frame))
|
||||||
self.assertEqual(frame, TextFrame(text='hello world'))
|
self.assertEqual(frame, TextFrame(text='hello world'))
|
||||||
|
|
||||||
transcription_frame = TranscriptionQueueFrame(
|
transcription_frame = TranscriptionFrame(
|
||||||
text="Hello there!", participantId="123", timestamp="2021-01-01")
|
text="Hello there!", participantId="123", timestamp="2021-01-01")
|
||||||
frame = self.serializer.deserialize(
|
frame = self.serializer.deserialize(
|
||||||
self.serializer.serialize(transcription_frame))
|
self.serializer.serialize(transcription_frame))
|
||||||
|
|||||||
Reference in New Issue
Block a user