@@ -17,14 +17,23 @@ class ControlFrame(Frame):
|
||||
|
||||
|
||||
class StartFrame(ControlFrame):
|
||||
"""Used (but not required) to start a pipeline, and is also used to
|
||||
indicate that an interruption has ended and the transport should start
|
||||
processing frames again."""
|
||||
pass
|
||||
|
||||
|
||||
class EndFrame(ControlFrame):
|
||||
"""Indicates that a pipeline has ended and frame processors and pipelines
|
||||
should be shut down. If the transport receives this frame, it will stop
|
||||
sending frames to its output channel(s) and close all its threads."""
|
||||
pass
|
||||
|
||||
|
||||
class EndPipeFrame(ControlFrame):
|
||||
"""Indicates that a pipeline has ended but that the transport should
|
||||
continue processing. This frame is used in parallel pipelines and other
|
||||
sub-pipelines."""
|
||||
pass
|
||||
|
||||
|
||||
@@ -39,15 +48,20 @@ class PipelineStartedFrame(ControlFrame):
|
||||
|
||||
|
||||
class LLMResponseStartFrame(ControlFrame):
|
||||
"""Used to indicate the beginning of an LLM response. Following TextFrames
|
||||
are part of the LLM response until an LLMResponseEndFrame"""
|
||||
pass
|
||||
|
||||
|
||||
class LLMResponseEndFrame(ControlFrame):
|
||||
"""Indicates the end of an LLM response."""
|
||||
pass
|
||||
|
||||
|
||||
@dataclass()
|
||||
class AudioFrame(Frame):
|
||||
"""A chunk of audio. Will be played by the transport if the transport's mic
|
||||
has been enabled."""
|
||||
data: bytes
|
||||
|
||||
def __str__(self):
|
||||
@@ -56,6 +70,8 @@ class AudioFrame(Frame):
|
||||
|
||||
@dataclass()
|
||||
class ImageFrame(Frame):
|
||||
"""An image. Will be shown by the transport if the transport's camera is
|
||||
enabled."""
|
||||
url: str | None
|
||||
image: bytes
|
||||
|
||||
@@ -65,14 +81,19 @@ class ImageFrame(Frame):
|
||||
|
||||
@dataclass()
|
||||
class SpriteFrame(Frame):
|
||||
"""An animated sprite. Will be shown by the transport if the transport's
|
||||
camera is enabled. Will play at the framerate specified in the transport's
|
||||
`fps` constructor parameter."""
|
||||
images: list[bytes]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.__class__.name__}, list size: {len(self.images)}"
|
||||
return f"{self.__class__.__name__}, list size: {len(self.images)}"
|
||||
|
||||
|
||||
@dataclass()
|
||||
class TextFrame(Frame):
|
||||
"""A chunk of text. Emitted by LLM services, consumed by TTS services, can
|
||||
be used to send text through pipelines."""
|
||||
text: str
|
||||
|
||||
def __str__(self):
|
||||
@@ -81,17 +102,27 @@ class TextFrame(Frame):
|
||||
|
||||
@dataclass()
|
||||
class TranscriptionQueueFrame(TextFrame):
|
||||
"""A text frame with transcription-specific data. Will be placed in the
|
||||
transport's receive queue when a participant speaks."""
|
||||
participantId: str
|
||||
timestamp: str
|
||||
|
||||
|
||||
@dataclass()
|
||||
class LLMMessagesQueueFrame(Frame):
|
||||
"""A frame containing a list of LLM messages. Used to signal that an LLM
|
||||
service should run a chat completion and emit an LLMStartFrames, TextFrames
|
||||
and an LLMEndFrame.
|
||||
Note that the messages property on this class is mutable, and will be
|
||||
be updated by various ResponseAggregator frame processors."""
|
||||
messages: List[dict]
|
||||
|
||||
|
||||
@dataclass()
|
||||
class OpenAILLMContextFrame(Frame):
|
||||
"""Like an LLMMessagesQueueFrame, but with extra context specific to the
|
||||
OpenAI API. The context in this message is also mutable, and will be
|
||||
changed by the OpenAIContextAggregator frame processor."""
|
||||
context: OpenAILLMContext
|
||||
|
||||
|
||||
@@ -114,10 +145,15 @@ class SendAppMessageFrame(Frame):
|
||||
|
||||
|
||||
class UserStartedSpeakingFrame(Frame):
|
||||
"""Emitted by VAD to indicate that a participant has started speaking.
|
||||
This can be used for interruptions or other times when detecting that
|
||||
someone is speaking is more important than knowing what they're saying
|
||||
(as you will with a TranscriptionFrame)"""
|
||||
pass
|
||||
|
||||
|
||||
class UserStoppedSpeakingFrame(Frame):
|
||||
"""Emitted by the VAD to indicate that a user stopped speaking."""
|
||||
pass
|
||||
|
||||
|
||||
@@ -131,10 +167,15 @@ class BotStoppedSpeakingFrame(Frame):
|
||||
|
||||
@dataclass()
|
||||
class LLMFunctionStartFrame(Frame):
|
||||
"""Emitted when the LLM receives the beginning of a function call
|
||||
completion. A frame processor can use this frame to indicate that it should
|
||||
start preparing to make a function call, if it can do so in the absence of
|
||||
any arguments."""
|
||||
function_name: str
|
||||
|
||||
|
||||
@dataclass()
|
||||
class LLMFunctionCallFrame(Frame):
|
||||
"""Emitted when the LLM has received an entire function call completion."""
|
||||
function_name: str
|
||||
arguments: str
|
||||
|
||||
Reference in New Issue
Block a user