wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #93 from daily-co/frame-name-cleanup

Browse Source 
Cleanup the last few badly-named Frame types
This commit is contained in:
Moishe Lettvin
2024-03-28 14:25:59 -04:00
committed by GitHub
parent 22bbedec93 27322108b7
commit e7f9296b5a
26 changed files with 64 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/foundational/02-llm-say-one-thing.py
View File

@@ -4,7 +4,7 @@ import logging
import aiohttp
from dailyai.pipeline.frames import EndFrame, LLMMessagesQueueFrame
from dailyai.pipeline.frames import EndFrame, LLMMessagesFrame
from dailyai.pipeline.pipeline import Pipeline
from dailyai.transports.daily_transport import DailyTransport
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
@@ -49,7 +49,7 @@ async def main(room_url):
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await pipeline.queue_frames([LLMMessagesQueueFrame(messages), EndFrame()])
await pipeline.queue_frames([LLMMessagesFrame(messages), EndFrame()])
await transport.run(pipeline)

4
examples/foundational/04-utterance-and-speech.py
View File

@@ -9,7 +9,7 @@ from dailyai.pipeline.pipeline import Pipeline
from dailyai.transports.daily_transport import DailyTransport
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.deepgram_ai_services import DeepgramTTSService
from dailyai.pipeline.frames import EndPipeFrame, LLMMessagesQueueFrame, TextFrame
from dailyai.pipeline.frames import EndPipeFrame, LLMMessagesFrame, TextFrame
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from runner import configure
@@ -60,7 +60,7 @@ async def main(room_url: str):
# will run in parallel with generating and speaking the audio for static text, so there's no delay to
# speak the LLM response.
llm_pipeline = Pipeline([llm, elevenlabs_tts])
await llm_pipeline.queue_frames([LLMMessagesQueueFrame(messages), EndPipeFrame()])
await llm_pipeline.queue_frames([LLMMessagesFrame(messages), EndPipeFrame()])
simple_tts_pipeline = Pipeline([azure_tts])
await simple_tts_pipeline.queue_frames(

4
examples/foundational/05-sync-speech-and-image.py
View File

@@ -17,7 +17,7 @@ from dailyai.pipeline.frames import (
TextFrame,
EndFrame,
ImageFrame,
LLMMessagesQueueFrame,
LLMMessagesFrame,
LLMResponseStartFrame,
)
from dailyai.pipeline.frame_processor import FrameProcessor
@@ -133,7 +133,7 @@ async def main(room_url):
}
]
frames.append(MonthFrame(month))
frames.append(LLMMessagesQueueFrame(messages))
frames.append(LLMMessagesFrame(messages))
frames.append(EndFrame())
await pipeline.queue_frames(frames)

4
examples/foundational/06-listen-and-respond.py
View File

@@ -2,7 +2,7 @@ import asyncio
import aiohttp
import logging
import os
from dailyai.pipeline.frames import LLMMessagesQueueFrame
from dailyai.pipeline.frames import LLMMessagesFrame
from dailyai.pipeline.pipeline import Pipeline
from dailyai.transports.daily_transport import DailyTransport
@@ -76,7 +76,7 @@ async def main(room_url: str, token):
# Kick off the conversation.
messages.append(
{"role": "system", "content": "Please introduce yourself to the user."})
await pipeline.queue_frames([LLMMessagesQueueFrame(messages)])
await pipeline.queue_frames([LLMMessagesFrame(messages)])
transport.transcription_settings["extra"]["endpointing"] = True
transport.transcription_settings["extra"]["punctuate"] = True

4
examples/foundational/08-bots-arguing.py
View File

@@ -10,7 +10,7 @@ from dailyai.transports.daily_transport import DailyTransport
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesQueueFrame, TextFrame
from dailyai.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame
from runner import configure
@@ -80,7 +80,7 @@ async def main(room_url: str):
[llm, sentence_aggregator, tts1], source_queue, sink_queue
)
await source_queue.put(LLMMessagesQueueFrame(messages))
await source_queue.put(LLMMessagesFrame(messages))
await source_queue.put(EndFrame())
await pipeline.run_pipeline()

4
examples/foundational/10-wake-word.py
View File

@@ -18,7 +18,7 @@ from dailyai.pipeline.frames import (
TextFrame,
ImageFrame,
SpriteFrame,
TranscriptionQueueFrame,
TranscriptionFrame,
)
from dailyai.services.ai_services import AIService
@@ -76,7 +76,7 @@ class TranscriptFilter(AIService):
self.bot_participant_id = bot_participant_id
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, TranscriptionQueueFrame):
if isinstance(frame, TranscriptionFrame):
if frame.participantId != self.bot_participant_id:
yield frame

4
examples/foundational/11-sound-effects.py
View File

@@ -16,7 +16,7 @@ from dailyai.pipeline.frames import (
Frame,
AudioFrame,
LLMResponseEndFrame,
LLMMessagesQueueFrame,
LLMMessagesFrame,
)
from typing import AsyncGenerator
@@ -62,7 +62,7 @@ class InboundSoundEffectWrapper(AIService):
pass
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, LLMMessagesQueueFrame):
if isinstance(frame, LLMMessagesFrame):
yield AudioFrame(sounds["ding2.wav"])
# In case anything else up the stack needs it
yield frame

4
examples/foundational/13a-whisper-local.py
View File

@@ -1,7 +1,7 @@
import argparse
import asyncio
import logging
from dailyai.pipeline.frames import EndFrame, TranscriptionQueueFrame
from dailyai.pipeline.frames import EndFrame, TranscriptionFrame
from dailyai.transports.local_transport import LocalTransport
from dailyai.services.whisper_ai_services import WhisperSTTService
@@ -32,7 +32,7 @@ async def main(room_url: str):
while not transport_done.is_set():
item = await transcription_output_queue.get()
print("got item from queue", item)
if isinstance(item, TranscriptionQueueFrame):
if isinstance(item, TranscriptionFrame):
print(item.text)
elif isinstance(item, EndFrame):
break

4
examples/foundational/websocket-server/sample.py
View File

@@ -3,7 +3,7 @@ import aiohttp
import logging
import os
from dailyai.pipeline.frame_processor import FrameProcessor
from dailyai.pipeline.frames import TextFrame, TranscriptionQueueFrame
from dailyai.pipeline.frames import TextFrame, TranscriptionFrame
from dailyai.pipeline.pipeline import Pipeline
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.transports.websocket_transport import WebsocketTransport
@@ -16,7 +16,7 @@ logger.setLevel(logging.DEBUG)
class WhisperTranscriber(FrameProcessor):
async def process_frame(self, frame):
if isinstance(frame, TranscriptionQueueFrame):
if isinstance(frame, TranscriptionFrame):
print(f"Transcribed: {frame.text}")
else:
yield frame

4
examples/internal/11a-dial-out.py
View File

@@ -7,7 +7,7 @@ from dailyai.transports.daily_transport import DailyTransport
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.pipeline.aggregators import LLMContextAggregator
from dailyai.services.ai_services import AIService, FrameLogger
from dailyai.pipeline.frames import Frame, AudioFrame, LLMResponseEndFrame, LLMMessagesQueueFrame
from dailyai.pipeline.frames import Frame, AudioFrame, LLMResponseEndFrame, LLMMessagesFrame
from typing import AsyncGenerator
from runner import configure
@@ -51,7 +51,7 @@ class InboundSoundEffectWrapper(AIService):
pass
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, LLMMessagesQueueFrame):
if isinstance(frame, LLMMessagesFrame):
yield AudioFrame(sounds["ding2.wav"])
# In case anything else up the stack needs it
yield frame

4
examples/starter-apps/chatbot.py
View File

@@ -14,7 +14,7 @@ from dailyai.pipeline.frames import (
SpriteFrame,
Frame,
LLMResponseEndFrame,
LLMMessagesQueueFrame,
LLMMessagesFrame,
AudioFrame,
PipelineStartedFrame,
)
@@ -129,7 +129,7 @@ async def main(room_url: str, token):
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
print(f"!!! in here, pipeline.source is {pipeline.source}")
await pipeline.queue_frames([LLMMessagesQueueFrame(messages)])
await pipeline.queue_frames([LLMMessagesFrame(messages)])
async def run_conversation():

6
examples/starter-apps/storybot.py
View File

@@ -24,7 +24,7 @@ from dailyai.pipeline.aggregators import (
)
from dailyai.pipeline.frames import (
EndPipeFrame,
LLMMessagesQueueFrame,
LLMMessagesFrame,
Frame,
TextFrame,
LLMResponseEndFrame,
@@ -172,7 +172,7 @@ class StoryImageGenerator(FrameProcessor):
prompt = f"You are an illustrator for a children's story book. Here is the story so far:\n\n\"{' '.join(self._story[:-1])}\"\n\nGenerate a prompt for DALL-E to create an illustration for the next page. Here's the sentence for the next page:\n\n\"{self._story[-1:][0]}\"\n\n Your response should start with the phrase \"Children's book illustration of\"."
msgs = [{"role": "system", "content": prompt}]
image_prompt = ""
async for f in self._llm.process_frame(LLMMessagesQueueFrame(msgs)):
async for f in self._llm.process_frame(LLMMessagesFrame(msgs)):
if isinstance(f, TextFrame):
image_prompt += f.text
async for f in self._img.process_frame(TextFrame(image_prompt)):
@@ -253,7 +253,7 @@ async def main(room_url: str, token):
await local_pipeline.queue_frames(
[
ImageFrame(None, images["grandma-listening.png"]),
LLMMessagesQueueFrame(intro_messages),
LLMMessagesFrame(intro_messages),
AudioFrame(sounds["listening.wav"]),
EndPipeFrame(),
]

4
examples/starter-apps/translator.py
View File

@@ -7,7 +7,7 @@ from typing import AsyncGenerator
from dailyai.pipeline.aggregators import (
SentenceAggregator,
)
from dailyai.pipeline.frames import Frame, LLMMessagesQueueFrame, TextFrame
from dailyai.pipeline.frames import Frame, LLMMessagesFrame, TextFrame
from dailyai.pipeline.frame_processor import FrameProcessor
from dailyai.pipeline.pipeline import Pipeline
from dailyai.transports.daily_transport import DailyTransport
@@ -44,7 +44,7 @@ class TranslationProcessor(FrameProcessor):
},
{"role": "user", "content": frame.text},
]
yield LLMMessagesQueueFrame(context)
yield LLMMessagesFrame(context)
else:
yield frame

20
src/dailyai/pipeline/aggregators.py
View File

@@ -7,11 +7,11 @@ from dailyai.pipeline.frames import (
EndFrame,
EndPipeFrame,
Frame,
LLMMessagesQueueFrame,
LLMMessagesFrame,
LLMResponseEndFrame,
LLMResponseStartFrame,
TextFrame,
TranscriptionQueueFrame,
TranscriptionFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
@@ -57,7 +57,7 @@ class ResponseAggregator(FrameProcessor):
{"role": self._role, "content": self.aggregation})
self.aggregation = ""
yield self._end_frame()
yield LLMMessagesQueueFrame(self.messages)
yield LLMMessagesFrame(self.messages)
elif isinstance(frame, self._accumulator_frame) and self.aggregating:
self.aggregation += f" {frame.text}"
if self._pass_through:
@@ -84,7 +84,7 @@ class UserResponseAggregator(ResponseAggregator):
role="user",
start_frame=UserStartedSpeakingFrame,
end_frame=UserStoppedSpeakingFrame,
accumulator_frame=TranscriptionQueueFrame,
accumulator_frame=TranscriptionFrame,
pass_through=False,
)
@@ -114,7 +114,7 @@ class LLMContextAggregator(AIService):
return
# Ignore transcription frames from the bot
if isinstance(frame, TranscriptionQueueFrame):
if isinstance(frame, TranscriptionFrame):
if frame.participantId == self.bot_participant_id:
return
@@ -126,19 +126,19 @@ class LLMContextAggregator(AIService):
# TODO: split up transcription by participant
if self.complete_sentences:
# type: ignore -- the linter thinks this isn't a TextQueueFrame, even
# type: ignore -- the linter thinks this isn't a TextFrame, even
# though we check it above
self.sentence += frame.text
if self.sentence.endswith((".", "?", "!")):
self.messages.append(
{"role": self.role, "content": self.sentence})
self.sentence = ""
yield LLMMessagesQueueFrame(self.messages)
yield LLMMessagesFrame(self.messages)
else:
# type: ignore -- the linter thinks this isn't a TextQueueFrame, even
# type: ignore -- the linter thinks this isn't a TextFrame, even
# though we check it above
self.messages.append({"role": self.role, "content": frame.text})
yield LLMMessagesQueueFrame(self.messages)
yield LLMMessagesFrame(self.messages)
class LLMUserContextAggregator(LLMContextAggregator):
@@ -334,7 +334,7 @@ class ParallelPipeline(FrameProcessor):
continue
seen_ids.add(id(frame))
# Skip passing along EndParallelPipeQueueFrame, because we use them
# Skip passing along EndPipeFrame, because we use them
# for our own flow control.
if not isinstance(frame, EndPipeFrame):
yield frame

4
src/dailyai/pipeline/frame_processor.py
View File

@@ -12,9 +12,9 @@ class FrameProcessor:
By convention, FrameProcessors should immediately yield any frames they don't process.
Stateful FrameProcessors should watch for the EndStreamQueueFrame and finalize their
Stateful FrameProcessors should watch for the EndFrame and finalize their
output, eg. yielding an unfinished sentence if they're aggregating LLM output to full
sentences. EndStreamQueueFrame is also a chance to clean up any services that need to
sentences. EndFrame is also a chance to clean up any services that need to
be closed, del'd, etc.
"""

6
src/dailyai/pipeline/frames.py
View File

@@ -102,7 +102,7 @@ class TextFrame(Frame):
@dataclass()
class TranscriptionQueueFrame(TextFrame):
class TranscriptionFrame(TextFrame):
"""A text frame with transcription-specific data. Will be placed in the
transport's receive queue when a participant speaks."""
participantId: str
@@ -126,7 +126,7 @@ class TTSEndFrame(ControlFrame):
@dataclass()
class LLMMessagesQueueFrame(Frame):
class LLMMessagesFrame(Frame):
"""A frame containing a list of LLM messages. Used to signal that an LLM
service should run a chat completion and emit an LLMStartFrames, TextFrames
and an LLMEndFrame.
@@ -137,7 +137,7 @@ class LLMMessagesQueueFrame(Frame):
@dataclass()
class OpenAILLMContextFrame(Frame):
"""Like an LLMMessagesQueueFrame, but with extra context specific to the
"""Like an LLMMessagesFrame, but with extra context specific to the
OpenAI API. The context in this message is also mutable, and will be
changed by the OpenAIContextAggregator frame processor."""
context: OpenAILLMContext

4
src/dailyai/pipeline/opeanai_llm_aggregator.py
View File

@@ -6,7 +6,7 @@ from dailyai.pipeline.frames import (
LLMResponseStartFrame,
OpenAILLMContextFrame,
TextFrame,
TranscriptionQueueFrame,
TranscriptionFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
@@ -90,7 +90,7 @@ class OpenAIUserContextAggregator(OpenAIContextAggregator):
role="user",
start_frame=UserStartedSpeakingFrame,
end_frame=UserStoppedSpeakingFrame,
accumulator_frame=TranscriptionQueueFrame,
accumulator_frame=TranscriptionFrame,
pass_through=False,
)

4
src/dailyai/pipeline/pipeline.py
View File

@@ -81,8 +81,8 @@ class Pipeline:
The source and sink queues must be set before calling this method.
This method will exit when an EndStreamQueueFrame is placed on the sink queue.
No more frames will be placed on the sink queue after an EndStreamQueueFrame, even
This method will exit when an EndFrame is placed on the sink queue.
No more frames will be placed on the sink queue after an EndFrame, even
if it's not the last frame yielded by the last frame_processor in the pipeline..
"""

8
src/dailyai/serializers/protobuf_serializer.py
View File

@@ -1,6 +1,6 @@
import dataclasses
from typing import Text
from dailyai.pipeline.frames import AudioFrame, Frame, TextFrame, TranscriptionQueueFrame
from dailyai.pipeline.frames import AudioFrame, Frame, TextFrame, TranscriptionFrame
import dailyai.pipeline.protobufs.frames_pb2 as frame_protos
from dailyai.serializers.abstract_frame_serializer import FrameSerializer
@@ -9,7 +9,7 @@ class ProtobufFrameSerializer(FrameSerializer):
SERIALIZABLE_TYPES = {
TextFrame: "text",
AudioFrame: "audio",
TranscriptionQueueFrame: "transcription"
TranscriptionFrame: "transcription"
}
SERIALIZABLE_FIELDS = {v: k for k, v in SERIALIZABLE_TYPES.items()}
@@ -45,9 +45,9 @@ class ProtobufFrameSerializer(FrameSerializer):
... serializer.serialize(TextFrame(text='hello world')))
TextFrame(text='hello world')
>>> serializer.deserialize(serializer.serialize(TranscriptionQueueFrame(
>>> serializer.deserialize(serializer.serialize(TranscriptionFrame(
... text="Hello there!", participantId="123", timestamp="2021-01-01")))
TranscriptionQueueFrame(text='Hello there!', participantId='123', timestamp='2021-01-01')
TranscriptionFrame(text='Hello there!', participantId='123', timestamp='2021-01-01')
"""
proto = frame_protos.Frame.FromString(data)

4
src/dailyai/services/ai_services.py
View File

@@ -13,7 +13,7 @@ from dailyai.pipeline.frames import (
TTSEndFrame,
TTSStartFrame,
TextFrame,
TranscriptionQueueFrame,
TranscriptionFrame,
)
from abc import abstractmethod
@@ -128,7 +128,7 @@ class STTService(AIService):
ww.close()
content.seek(0)
text = await self.run_stt(content)
yield TranscriptionQueueFrame(text, "", str(time.time()))
yield TranscriptionFrame(text, "", str(time.time()))
class FrameLogger(AIService):

4
src/dailyai/services/anthropic_llm_service.py
View File

@@ -1,6 +1,6 @@
from typing import AsyncGenerator
from anthropic import AsyncAnthropic
from dailyai.pipeline.frames import Frame, LLMMessagesQueueFrame, TextFrame
from dailyai.pipeline.frames import Frame, LLMMessagesFrame, TextFrame
from dailyai.services.ai_services import LLMService
@@ -18,7 +18,7 @@ class AnthropicLLMService(LLMService):
self.max_tokens = max_tokens
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if not isinstance(frame, LLMMessagesQueueFrame):
if not isinstance(frame, LLMMessagesFrame):
yield frame
stream = await self.client.messages.create(

4
src/dailyai/services/local_stt_service.py
View File

@@ -4,7 +4,7 @@ import math
import time
from typing import AsyncGenerator
import wave
from dailyai.pipeline.frames import AudioFrame, Frame, TranscriptionQueueFrame
from dailyai.pipeline.frames import AudioFrame, Frame, TranscriptionFrame
from dailyai.services.ai_services import STTService
@@ -61,7 +61,7 @@ class LocalSTTService(STTService):
self._content.seek(0)
text = await self.run_stt(self._content)
self._new_wave()
yield TranscriptionQueueFrame(text, '', str(time.time()))
yield TranscriptionFrame(text, '', str(time.time()))
# If we get this far, this is a frame of silence
self._current_silence_frames += 1

4
src/dailyai/services/openai_api_llm_service.py
View File

@@ -6,7 +6,7 @@ from dailyai.pipeline.frames import (
Frame,
LLMFunctionCallFrame,
LLMFunctionStartFrame,
LLMMessagesQueueFrame,
LLMMessagesFrame,
LLMResponseEndFrame,
LLMResponseStartFrame,
OpenAILLMContextFrame,
@@ -75,7 +75,7 @@ class BaseOpenAILLMService(LLMService):
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, OpenAILLMContextFrame):
context: OpenAILLMContext = frame.context
elif isinstance(frame, LLMMessagesQueueFrame):
elif isinstance(frame, LLMMessagesFrame):
context = OpenAILLMContext.from_messages(frame.messages)
else:
yield frame

4
src/dailyai/transports/daily_transport.py
View File

@@ -10,7 +10,7 @@ from typing import Any
from dailyai.pipeline.frames import (
ReceivedAppMessageFrame,
TranscriptionQueueFrame,
TranscriptionFrame,
)
from threading import Event
@@ -269,7 +269,7 @@ class DailyTransport(ThreadedTransport, EventHandler):
elif "session_id" in message:
participantId = message["session_id"]
if self._my_participant_id and participantId != self._my_participant_id:
frame = TranscriptionQueueFrame(
frame = TranscriptionFrame(
message["text"], participantId, message["timestamp"])
asyncio.run_coroutine_threadsafe(
self.receive_queue.put(frame), self._loop)

4
tests/test_daily_transport_service.py
View File

@@ -65,10 +65,10 @@ class TestDailyTransport(unittest.IsolatedAsyncioTestCase):
daily_mock.create_camera_device.return_value = camera
async def send_audio_frame():
await transport.send_queue.put(AudioQueueFrame(bytes([0] * 3300)))
await transport.send_queue.put(AudioFrame(bytes([0] * 3300)))
async def send_video_frame():
await transport.send_queue.put(ImageQueueFrame(None, b"test"))
await transport.send_queue.put(ImageFrame(None, b"test"))
await asyncio.gather(transport.run(), send_audio_frame(), send_video_frame())

4
tests/test_protobuf_serializer.py
View File

@@ -1,6 +1,6 @@
import unittest
from dailyai.pipeline.frames import AudioFrame, TextFrame, TranscriptionQueueFrame
from dailyai.pipeline.frames import AudioFrame, TextFrame, TranscriptionFrame
from dailyai.serializers.protobuf_serializer import ProtobufFrameSerializer
@@ -14,7 +14,7 @@ class TestProtobufFrameSerializer(unittest.IsolatedAsyncioTestCase):
self.serializer.serialize(text_frame))
self.assertEqual(frame, TextFrame(text='hello world'))
transcription_frame = TranscriptionQueueFrame(
transcription_frame = TranscriptionFrame(
text="Hello there!", participantId="123", timestamp="2021-01-01")
frame = self.serializer.deserialize(
self.serializer.serialize(transcription_frame))