wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge from main

Browse Source
This commit is contained in:
Adrian Cowham
2024-10-03 09:42:06 -07:00
parent 387a36dd8a 65eeb0f1f6
commit 2d82702e04
191 changed files with 8988 additions and 8902 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
src/pipecat/clocks/__init__.py Normal file
View File

17
src/pipecat/clocks/base_clock.py Normal file
View File

@@ -0,0 +1,17 @@
#
# Copyright (c) 2024, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from abc import ABC, abstractmethod
class BaseClock(ABC):
@abstractmethod
def get_time(self) -> int:
pass
@abstractmethod
def start(self):
pass

20
src/pipecat/clocks/system_clock.py Normal file
View File

@@ -0,0 +1,20 @@
#
# Copyright (c) 2024, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import time
from pipecat.clocks.base_clock import BaseClock
class SystemClock(BaseClock):
def __init__(self):
self._time = 0
def get_time(self) -> int:
return time.monotonic_ns() - self._time if self._time > 0 else 0
def start(self):
self._time = time.monotonic_ns()

1
src/pipecat/frames/frames.proto
View File

@@ -24,6 +24,7 @@ message AudioRawFrame {
bytes audio = 3;
uint32 sample_rate = 4;
uint32 num_channels = 5;
optional uint64 pts = 6;
}
message TranscriptionFrame {

262
src/pipecat/frames/frames.py
View File

@@ -4,23 +4,31 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import time
from dataclasses import dataclass, field
from typing import Any, List, Mapping, Optional, Tuple
from typing import Any, Dict, List, Optional, Tuple
from pipecat.clocks.base_clock import BaseClock
from pipecat.metrics.metrics import MetricsData
from pipecat.transcriptions.language import Language
from pipecat.utils.time import nanoseconds_to_str
from pipecat.utils.utils import obj_count, obj_id
from pipecat.vad.vad_analyzer import VADParams
def format_pts(pts: int | None):
return nanoseconds_to_str(pts) if pts else None
@dataclass
class Frame:
id: int = field(init=False)
name: str = field(init=False)
pts: Optional[int] = field(init=False)
def __post_init__(self):
self.id: int = obj_id()
self.name: str = f"{self.__class__.__name__}#{obj_count(self)}"
self.pts: Optional[int] = None
def __str__(self):
return self.name
@@ -33,10 +41,8 @@ class DataFrame(Frame):
@dataclass
class AudioRawFrame(DataFrame):
"""A chunk of audio. Will be played by the transport if the transport's
microphone has been enabled.
"""A chunk of audio."""
"""
audio: bytes
sample_rate: int
num_channels: int
@@ -46,7 +52,32 @@ class AudioRawFrame(DataFrame):
self.num_frames = int(len(self.audio) / (self.num_channels * 2))
def __str__(self):
return f"{self.name}(size: {len(self.audio)}, frames: {self.num_frames}, sample_rate: {self.sample_rate}, channels: {self.num_channels})"
pts = format_pts(self.pts)
return f"{self.name}(pts: {pts}, size: {len(self.audio)}, frames: {self.num_frames}, sample_rate: {self.sample_rate}, channels: {self.num_channels})"
@dataclass
class InputAudioRawFrame(AudioRawFrame):
"""A chunk of audio usually coming from an input transport."""
pass
@dataclass
class OutputAudioRawFrame(AudioRawFrame):
"""A chunk of audio. Will be played by the output transport if the
transport's microphone has been enabled.
"""
pass
@dataclass
class TTSAudioRawFrame(OutputAudioRawFrame):
"""A chunk of output audio generated by a TTS service."""
pass
@dataclass
@@ -55,48 +86,66 @@ class ImageRawFrame(DataFrame):
enabled.
"""
image: bytes
size: Tuple[int, int]
format: str | None
def __str__(self):
return f"{self.name}(size: {self.size}, format: {self.format})"
pts = format_pts(self.pts)
return f"{self.name}(pts: {pts}, size: {self.size}, format: {self.format})"
@dataclass
class URLImageRawFrame(ImageRawFrame):
"""An image with an associated URL. Will be shown by the transport if the
transport's camera is enabled.
"""
url: str | None
def __str__(self):
return f"{self.name}(url: {self.url}, size: {self.size}, format: {self.format})"
class InputImageRawFrame(ImageRawFrame):
pass
@dataclass
class VisionImageRawFrame(ImageRawFrame):
"""An image with an associated text to ask for a description of it. Will be
shown by the transport if the transport's camera is enabled.
"""
text: str | None
def __str__(self):
return f"{self.name}(text: {self.text}, size: {self.size}, format: {self.format})"
class OutputImageRawFrame(ImageRawFrame):
pass
@dataclass
class UserImageRawFrame(ImageRawFrame):
class UserImageRawFrame(InputImageRawFrame):
"""An image associated to a user. Will be shown by the transport if the
transport's camera is enabled.
"""
user_id: str
def __str__(self):
return f"{self.name}(user: {self.user_id}, size: {self.size}, format: {self.format})"
pts = format_pts(self.pts)
return f"{self.name}(pts: {pts}, user: {self.user_id}, size: {self.size}, format: {self.format})"
@dataclass
class VisionImageRawFrame(InputImageRawFrame):
"""An image with an associated text to ask for a description of it. Will be
shown by the transport if the transport's camera is enabled.
"""
text: str | None
def __str__(self):
pts = format_pts(self.pts)
return f"{self.name}(pts: {pts}, text: [{self.text}], size: {self.size}, format: {self.format})"
@dataclass
class URLImageRawFrame(OutputImageRawFrame):
"""An image with an associated URL. Will be shown by the transport if the
transport's camera is enabled.
"""
url: str | None
def __str__(self):
pts = format_pts(self.pts)
return f"{self.name}(pts: {pts}, url: {self.url}, size: {self.size}, format: {self.format})"
@dataclass
@@ -106,10 +155,12 @@ class SpriteFrame(Frame):
`camera_out_framerate` constructor parameter.
"""
images: List[ImageRawFrame]
def __str__(self):
return f"{self.name}(size: {len(self.images)})"
pts = format_pts(self.pts)
return f"{self.name}(pts: {pts}, size: {len(self.images)})"
@dataclass
@@ -118,10 +169,12 @@ class TextFrame(DataFrame):
be used to send text through pipelines.
"""
text: str
def __str__(self):
return f"{self.name}(text: {self.text})"
pts = format_pts(self.pts)
return f"{self.name}(pts: {pts}, text: [{self.text}])"
@dataclass
@@ -130,24 +183,26 @@ class TranscriptionFrame(TextFrame):
transport's receive queue when a participant speaks.
"""
user_id: str
timestamp: str
language: Language | None = None
def __str__(self):
return f"{self.name}(user: {self.user_id}, text: {self.text}, language: {self.language}, timestamp: {self.timestamp})"
return f"{self.name}(user: {self.user_id}, text: [{self.text}], language: {self.language}, timestamp: {self.timestamp})"
@dataclass
class InterimTranscriptionFrame(TextFrame):
"""A text frame with interim transcription-specific data. Will be placed in
the transport's receive queue when a participant speaks."""
user_id: str
timestamp: str
language: Language | None = None
def __str__(self):
return f"{self.name}(user: {self.user_id}, text: {self.text}, language: {self.language}, timestamp: {self.timestamp})"
return f"{self.name}(user: {self.user_id}, text: [{self.text}], language: {self.language}, timestamp: {self.timestamp})"
@dataclass
@@ -159,6 +214,7 @@ class LLMMessagesFrame(DataFrame):
processors.
"""
messages: List[dict]
@@ -168,6 +224,7 @@ class LLMMessagesAppendFrame(DataFrame):
current context.
"""
messages: List[dict]
@@ -178,6 +235,7 @@ class LLMMessagesUpdateFrame(DataFrame):
LLMMessagesFrame.
"""
messages: List[dict]
@@ -187,13 +245,14 @@ class LLMSetToolsFrame(DataFrame):
The specific format depends on the LLM being used, but it should typically
contain JSON Schema objects.
"""
tools: List[dict]
@dataclass
class LLMEnablePromptCachingFrame(DataFrame):
"""A frame to enable/disable prompt caching in certain LLMs.
"""
"""A frame to enable/disable prompt caching in certain LLMs."""
enable: bool
@@ -203,6 +262,7 @@ class TTSSpeakFrame(DataFrame):
pipeline (if any).
"""
text: str
@@ -214,6 +274,7 @@ class TransportMessageFrame(DataFrame):
def __str__(self):
return f"{self.name}(message: {self.message})"
#
# App frames. Application user-defined frames.
#
@@ -243,9 +304,21 @@ class SystemFrame(Frame):
pass
@dataclass
class StartFrame(SystemFrame):
"""This is the first frame that should be pushed down a pipeline."""
clock: BaseClock
allow_interruptions: bool = False
enable_metrics: bool = False
enable_usage_metrics: bool = False
report_only_initial_ttfb: bool = False
@dataclass
class CancelFrame(SystemFrame):
"""Indicates that a pipeline needs to stop right away."""
pass
@@ -256,6 +329,7 @@ class ErrorFrame(SystemFrame):
bot should exit.
"""
error: str
fatal: bool = False
@@ -269,9 +343,31 @@ class FatalErrorFrame(ErrorFrame):
that the bot should exit.
"""
fatal: bool = field(default=True, init=False)
@dataclass
class EndTaskFrame(SystemFrame):
"""This is used to notify the pipeline task that the pipeline should be
closed nicely (flushing all the queued frames) by pushing an EndFrame
downstream.
"""
pass
@dataclass
class CancelTaskFrame(SystemFrame):
"""This is used to notify the pipeline task that the pipeline should be
stopped immediately by pushing a CancelFrame downstream.
"""
pass
@dataclass
class StopTaskFrame(SystemFrame):
"""Indicates that a pipeline task should be stopped but that the pipeline
@@ -279,6 +375,7 @@ class StopTaskFrame(SystemFrame):
the pipeline task.
"""
pass
@@ -290,6 +387,7 @@ class StartInterruptionFrame(SystemFrame):
guaranteed).
"""
pass
@@ -301,6 +399,7 @@ class StopInterruptionFrame(SystemFrame):
guaranteed).
"""
pass
@@ -311,17 +410,16 @@ class BotInterruptionFrame(SystemFrame):
UserStartedSpeakingFrame and UserStoppedSpeakingFrame won't be generated.
"""
pass
@dataclass
class MetricsFrame(SystemFrame):
"""Emitted by processor that can compute metrics like latencies.
"""
ttfb: List[Mapping[str, Any]] | None = None
processing: List[Mapping[str, Any]] | None = None
tokens: List[Mapping[str, Any]] | None = None
characters: List[Mapping[str, Any]] | None = None
"""Emitted by processor that can compute metrics like latencies."""
data: List[MetricsData]
#
# Control frames
@@ -333,15 +431,6 @@ class ControlFrame(Frame):
pass
@dataclass
class StartFrame(ControlFrame):
"""This is the first frame that should be pushed down a pipeline."""
allow_interruptions: bool = False
enable_metrics: bool = False
enable_usage_metrics: bool = False
report_only_initial_ttfb: bool = False
@dataclass
class EndFrame(ControlFrame):
"""Indicates that a pipeline has ended and frame processors and pipelines
@@ -351,6 +440,7 @@ class EndFrame(ControlFrame):
was sent (unline system frames).
"""
pass
@@ -358,12 +448,14 @@ class EndFrame(ControlFrame):
class LLMFullResponseStartFrame(ControlFrame):
"""Used to indicate the beginning of an LLM response. Following by one or
more TextFrame and a final LLMFullResponseEndFrame."""
pass
@dataclass
class LLMFullResponseEndFrame(ControlFrame):
"""Indicates the end of an LLM response."""
pass
@@ -375,28 +467,28 @@ class UserStartedSpeakingFrame(ControlFrame):
with a TranscriptionFrame)
"""
pass
@dataclass
class UserStoppedSpeakingFrame(ControlFrame):
"""Emitted by the VAD to indicate that a user stopped speaking."""
pass
@dataclass
class BotStartedSpeakingFrame(ControlFrame):
"""Emitted upstream by transport outputs to indicate the bot started speaking.
"""Emitted upstream by transport outputs to indicate the bot started speaking."""
"""
pass
@dataclass
class BotStoppedSpeakingFrame(ControlFrame):
"""Emitted upstream by transport outputs to indicate the bot stopped speaking.
"""Emitted upstream by transport outputs to indicate the bot stopped speaking."""
"""
pass
@@ -408,30 +500,34 @@ class BotSpeakingFrame(ControlFrame):
since the user might be listening.
"""
pass
@dataclass
class TTSStartedFrame(ControlFrame):
"""Used to indicate the beginning of a TTS response. Following
AudioRawFrames are part of the TTS response until an TTSEndFrame. These
frames can be used for aggregating audio frames in a transport to optimize
the size of frames sent to the session, without needing to control this in
the TTS service.
TTSAudioRawFrames are part of the TTS response until an
TTSStoppedFrame. These frames can be used for aggregating audio frames in a
transport to optimize the size of frames sent to the session, without
needing to control this in the TTS service.
"""
pass
@dataclass
class TTSStoppedFrame(ControlFrame):
"""Indicates the end of a TTS response."""
pass
@dataclass
class UserImageRequestFrame(ControlFrame):
"""A frame user to request an image from the given user."""
user_id: str
context: Optional[Any] = None
@@ -440,55 +536,31 @@ class UserImageRequestFrame(ControlFrame):
@dataclass
class LLMModelUpdateFrame(ControlFrame):
"""A control frame containing a request to update to a new LLM model.
"""
model: str
class ServiceUpdateSettingsFrame(ControlFrame):
"""A control frame containing a request to update service settings."""
settings: Dict[str, Any]
@dataclass
class TTSModelUpdateFrame(ControlFrame):
"""A control frame containing a request to update the TTS model.
"""
model: str
class LLMUpdateSettingsFrame(ServiceUpdateSettingsFrame):
pass
@dataclass
class TTSVoiceUpdateFrame(ControlFrame):
"""A control frame containing a request to update to a new TTS voice.
"""
voice: str
class TTSUpdateSettingsFrame(ServiceUpdateSettingsFrame):
pass
@dataclass
class TTSLanguageUpdateFrame(ControlFrame):
"""A control frame containing a request to update to a new TTS language and
optional voice.
"""
language: Language
@dataclass
class STTModelUpdateFrame(ControlFrame):
"""A control frame containing a request to update the STT model and optional
language.
"""
model: str
@dataclass
class STTLanguageUpdateFrame(ControlFrame):
"""A control frame containing a request to update to STT language.
"""
language: Language
class STTUpdateSettingsFrame(ServiceUpdateSettingsFrame):
pass
@dataclass
class FunctionCallInProgressFrame(SystemFrame):
"""A frame signaling that a function call is in progress.
"""
"""A frame signaling that a function call is in progress."""
function_name: str
tool_call_id: str
arguments: str
@@ -496,12 +568,13 @@ class FunctionCallInProgressFrame(SystemFrame):
@dataclass
class FunctionCallResultFrame(DataFrame):
"""A frame containing the result of an LLM function (tool) call.
"""
"""A frame containing the result of an LLM function (tool) call."""
function_name: str
tool_call_id: str
arguments: str
result: Any
run_llm: bool = True
@dataclass
@@ -509,4 +582,5 @@ class VADParamsUpdateFrame(ControlFrame):
"""A control frame containing a request to update VAD params. Intended
to be pushed upstream from RTVI processor.
"""
params: VADParams

12
src/pipecat/frames/protobufs/frames_pb2.py
View File

@@ -14,7 +14,7 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x66rames.proto\x12\x07pipecat\"3\n\tTextFrame\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04text\x18\x03 \x01(\t\"c\n\rAudioRawFrame\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\r\n\x05\x61udio\x18\x03 \x01(\x0c\x12\x13\n\x0bsample_rate\x18\x04 \x01(\r\x12\x14\n\x0cnum_channels\x18\x05 \x01(\r\"`\n\x12TranscriptionFrame\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04text\x18\x03 \x01(\t\x12\x0f\n\x07user_id\x18\x04 \x01(\t\x12\x11\n\ttimestamp\x18\x05 \x01(\t\"\x93\x01\n\x05\x46rame\x12\"\n\x04text\x18\x01 \x01(\x0b\x32\x12.pipecat.TextFrameH\x00\x12\'\n\x05\x61udio\x18\x02 \x01(\x0b\x32\x16.pipecat.AudioRawFrameH\x00\x12\x34\n\rtranscription\x18\x03 \x01(\x0b\x32\x1b.pipecat.TranscriptionFrameH\x00\x42\x07\n\x05\x66rameb\x06proto3')
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x66rames.proto\x12\x07pipecat\"3\n\tTextFrame\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04text\x18\x03 \x01(\t\"}\n\rAudioRawFrame\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\r\n\x05\x61udio\x18\x03 \x01(\x0c\x12\x13\n\x0bsample_rate\x18\x04 \x01(\r\x12\x14\n\x0cnum_channels\x18\x05 \x01(\r\x12\x10\n\x03pts\x18\x06 \x01(\x04H\x00\x88\x01\x01\x42\x06\n\x04_pts\"`\n\x12TranscriptionFrame\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04text\x18\x03 \x01(\t\x12\x0f\n\x07user_id\x18\x04 \x01(\t\x12\x11\n\ttimestamp\x18\x05 \x01(\t\"\x93\x01\n\x05\x46rame\x12\"\n\x04text\x18\x01 \x01(\x0b\x32\x12.pipecat.TextFrameH\x00\x12\'\n\x05\x61udio\x18\x02 \x01(\x0b\x32\x16.pipecat.AudioRawFrameH\x00\x12\x34\n\rtranscription\x18\x03 \x01(\x0b\x32\x1b.pipecat.TranscriptionFrameH\x00\x42\x07\n\x05\x66rameb\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -24,9 +24,9 @@ if _descriptor._USE_C_DESCRIPTORS == False:
_globals['_TEXTFRAME']._serialized_start=25
_globals['_TEXTFRAME']._serialized_end=76
_globals['_AUDIORAWFRAME']._serialized_start=78
_globals['_AUDIORAWFRAME']._serialized_end=177
_globals['_TRANSCRIPTIONFRAME']._serialized_start=179
_globals['_TRANSCRIPTIONFRAME']._serialized_end=275
_globals['_FRAME']._serialized_start=278
_globals['_FRAME']._serialized_end=425
_globals['_AUDIORAWFRAME']._serialized_end=203
_globals['_TRANSCRIPTIONFRAME']._serialized_start=205
_globals['_TRANSCRIPTIONFRAME']._serialized_end=301
_globals['_FRAME']._serialized_start=304
_globals['_FRAME']._serialized_end=451
# @@protoc_insertion_point(module_scope)

0
src/pipecat/metrics/__init__.py Normal file
View File

31
src/pipecat/metrics/metrics.py Normal file
View File

@@ -0,0 +1,31 @@
from typing import Optional
from pydantic import BaseModel
class MetricsData(BaseModel):
processor: str
model: Optional[str] = None
class TTFBMetricsData(MetricsData):
value: float
class ProcessingMetricsData(MetricsData):
value: float
class LLMTokenUsage(BaseModel):
prompt_tokens: int
completion_tokens: int
total_tokens: int
cache_read_input_tokens: Optional[int] = None
cache_creation_input_tokens: Optional[int] = None
class LLMUsageMetricsData(MetricsData):
value: LLMTokenUsage
class TTSUsageMetricsData(MetricsData):
value: int

1
src/pipecat/pipeline/base_pipeline.py
View File

@@ -12,7 +12,6 @@ from pipecat.processors.frame_processor import FrameProcessor
class BasePipeline(FrameProcessor):
def __init__(self):
super().__init__()

2
src/pipecat/pipeline/parallel_pipeline.py
View File

@@ -18,7 +18,6 @@ from loguru import logger
class Source(FrameProcessor):
def __init__(self, upstream_queue: asyncio.Queue):
super().__init__()
self._up_queue = upstream_queue
@@ -34,7 +33,6 @@ class Source(FrameProcessor):
class Sink(FrameProcessor):
def __init__(self, downstream_queue: asyncio.Queue):
super().__init__()
self._down_queue = downstream_queue

3
src/pipecat/pipeline/pipeline.py
View File

@@ -12,7 +12,6 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class PipelineSource(FrameProcessor):
def __init__(self, upstream_push_frame: Callable[[Frame, FrameDirection], Coroutine]):
super().__init__()
self._upstream_push_frame = upstream_push_frame
@@ -28,7 +27,6 @@ class PipelineSource(FrameProcessor):
class PipelineSink(FrameProcessor):
def __init__(self, downstream_push_frame: Callable[[Frame, FrameDirection], Coroutine]):
super().__init__()
self._downstream_push_frame = downstream_push_frame
@@ -44,7 +42,6 @@ class PipelineSink(FrameProcessor):
class Pipeline(BasePipeline):
def __init__(self, processors: List[FrameProcessor]):
super().__init__()

7
src/pipecat/pipeline/runner.py
View File

@@ -14,7 +14,6 @@ from loguru import logger
class PipelineRunner:
def __init__(self, *, name: str | None = None, handle_sigint: bool = True):
self.id: int = obj_id()
self.name: str = name or f"{self.__class__.__name__}#{obj_count(self)}"
@@ -42,12 +41,10 @@ class PipelineRunner:
def _setup_sigint(self):
loop = asyncio.get_running_loop()
loop.add_signal_handler(
signal.SIGINT,
lambda *args: asyncio.create_task(self._sig_handler())
signal.SIGINT, lambda *args: asyncio.create_task(self._sig_handler())
)
loop.add_signal_handler(
signal.SIGTERM,
lambda *args: asyncio.create_task(self._sig_handler())
signal.SIGTERM, lambda *args: asyncio.create_task(self._sig_handler())
)
async def _sig_handler(self):

74
src/pipecat/pipeline/parallel_task.py → src/pipecat/pipeline/sync_parallel_pipeline.py
View File

@@ -6,19 +6,26 @@
import asyncio
from dataclasses import dataclass
from itertools import chain
from typing import List
from pipecat.frames.frames import ControlFrame, EndFrame, Frame, SystemFrame
from pipecat.pipeline.base_pipeline import BasePipeline
from pipecat.pipeline.pipeline import Pipeline
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.frames.frames import Frame
from loguru import logger
class Source(FrameProcessor):
@dataclass
class SyncFrame(ControlFrame):
"""This frame is used to know when the internal pipelines have finished."""
pass
class Source(FrameProcessor):
def __init__(self, upstream_queue: asyncio.Queue):
super().__init__()
self._up_queue = upstream_queue
@@ -34,7 +41,6 @@ class Source(FrameProcessor):
class Sink(FrameProcessor):
def __init__(self, downstream_queue: asyncio.Queue):
super().__init__()
self._down_queue = downstream_queue
@@ -49,12 +55,12 @@ class Sink(FrameProcessor):
await self._down_queue.put(frame)
class ParallelTask(BasePipeline):
class SyncParallelPipeline(BasePipeline):
def __init__(self, *args):
super().__init__()
if len(args) == 0:
raise Exception(f"ParallelTask needs at least one argument")
raise Exception(f"SyncParallelPipeline needs at least one argument")
self._sinks = []
self._sources = []
@@ -66,16 +72,19 @@ class ParallelTask(BasePipeline):
logger.debug(f"Creating {self} pipelines")
for processors in args:
if not isinstance(processors, list):
raise TypeError(f"ParallelTask argument {processors} is not a list")
raise TypeError(f"SyncParallelPipeline argument {processors} is not a list")
# We add a source at the beginning of the pipeline and a sink at the end.
source = Source(self._up_queue)
sink = Sink(self._down_queue)
up_queue = asyncio.Queue()
down_queue = asyncio.Queue()
source = Source(up_queue)
sink = Sink(down_queue)
processors: List[FrameProcessor] = [source] + processors + [sink]
# Keep track of sources and sinks.
self._sources.append(source)
self._sinks.append(sink)
# Keep track of sources and sinks. We also keep the output queue of
# the source and the sinks so we can use it later.
self._sources.append({"processor": source, "queue": down_queue})
self._sinks.append({"processor": sink, "queue": up_queue})
# Create pipeline
pipeline = Pipeline(processors)
@@ -96,17 +105,52 @@ class ParallelTask(BasePipeline):
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# The last processor of each pipeline needs to be synchronous otherwise
# this element won't work. Since, we know it should be synchronous we
# push a SyncFrame. Since frames are ordered we know this frame will be
# pushed after the synchronous processor has pushed its data allowing us
# to synchrnonize all the internal pipelines by waiting for the
# SyncFrame in all of them.
async def wait_for_sync(
obj, main_queue: asyncio.Queue, frame: Frame, direction: FrameDirection
):
processor = obj["processor"]
queue = obj["queue"]
await processor.process_frame(frame, direction)
if isinstance(frame, (SystemFrame, EndFrame)):
new_frame = await queue.get()
if isinstance(new_frame, (SystemFrame, EndFrame)):
await main_queue.put(new_frame)
else:
while not isinstance(new_frame, (SystemFrame, EndFrame)):
await main_queue.put(new_frame)
queue.task_done()
new_frame = await queue.get()
else:
await processor.process_frame(SyncFrame(), direction)
new_frame = await queue.get()
while not isinstance(new_frame, SyncFrame):
await main_queue.put(new_frame)
queue.task_done()
new_frame = await queue.get()
if direction == FrameDirection.UPSTREAM:
# If we get an upstream frame we process it in each sink.
await asyncio.gather(*[s.process_frame(frame, direction) for s in self._sinks])
await asyncio.gather(
*[wait_for_sync(s, self._up_queue, frame, direction) for s in self._sinks]
)
elif direction == FrameDirection.DOWNSTREAM:
# If we get a downstream frame we process it in each source.
await asyncio.gather(*[s.process_frame(frame, direction) for s in self._sources])
await asyncio.gather(
*[wait_for_sync(s, self._down_queue, frame, direction) for s in self._sources]
)
seen_ids = set()
while not self._up_queue.empty():
frame = await self._up_queue.get()
if frame and frame.id not in seen_ids:
if frame.id not in seen_ids:
await self.push_frame(frame, FrameDirection.UPSTREAM)
seen_ids.add(frame.id)
self._up_queue.task_done()
@@ -114,7 +158,7 @@ class ParallelTask(BasePipeline):
seen_ids = set()
while not self._down_queue.empty():
frame = await self._down_queue.get()
if frame and frame.id not in seen_ids:
if frame.id not in seen_ids:
await self.push_frame(frame, FrameDirection.DOWNSTREAM)
seen_ids.add(frame.id)
self._down_queue.task_done()

92
src/pipecat/pipeline/task.py
View File

@@ -10,14 +10,20 @@ from typing import AsyncIterable, Iterable
from pydantic import BaseModel
from pipecat.clocks.base_clock import BaseClock
from pipecat.clocks.system_clock import SystemClock
from pipecat.frames.frames import (
CancelFrame,
CancelTaskFrame,
EndFrame,
EndTaskFrame,
ErrorFrame,
Frame,
MetricsFrame,
StartFrame,
StopTaskFrame)
StopTaskFrame,
)
from pipecat.metrics.metrics import TTFBMetricsData, ProcessingMetricsData
from pipecat.pipeline.base_pipeline import BasePipeline
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.utils import obj_count, obj_id
@@ -34,7 +40,6 @@ class PipelineParams(BaseModel):
class Source(FrameProcessor):
def __init__(self, up_queue: asyncio.Queue):
super().__init__()
self._up_queue = up_queue
@@ -49,7 +54,13 @@ class Source(FrameProcessor):
await self.push_frame(frame, direction)
async def _handle_upstream_frame(self, frame: Frame):
if isinstance(frame, ErrorFrame):
if isinstance(frame, EndTaskFrame):
# Tell the task we should end nicely.
await self._up_queue.put(EndTaskFrame())
elif isinstance(frame, CancelTaskFrame):
# Tell the task we should end right away.
await self._up_queue.put(CancelTaskFrame())
elif isinstance(frame, ErrorFrame):
logger.error(f"Error running app: {frame}")
if frame.fatal:
# Cancel all tasks downstream.
@@ -58,22 +69,44 @@ class Source(FrameProcessor):
await self._up_queue.put(StopTaskFrame())
class PipelineTask:
class Sink(FrameProcessor):
def __init__(self, down_queue: asyncio.Queue):
super().__init__()
self._down_queue = down_queue
def __init__(self, pipeline: BasePipeline, params: PipelineParams = PipelineParams()):
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# We really just want to know when the EndFrame reached the sink.
if isinstance(frame, EndFrame):
await self._down_queue.put(frame)
class PipelineTask:
def __init__(
self,
pipeline: BasePipeline,
params: PipelineParams = PipelineParams(),
clock: BaseClock = SystemClock(),
):
self.id: int = obj_id()
self.name: str = f"{self.__class__.__name__}#{obj_count(self)}"
self._pipeline = pipeline
self._clock = clock
self._params = params
self._finished = False
self._down_queue = asyncio.Queue()
self._up_queue = asyncio.Queue()
self._down_queue = asyncio.Queue()
self._push_queue = asyncio.Queue()
self._source = Source(self._up_queue)
self._source.link(pipeline)
self._sink = Sink(self._down_queue)
pipeline.link(self._sink)
def has_finished(self):
return self._finished
@@ -87,19 +120,19 @@ class PipelineTask:
# out-of-band from the main streaming task which is what we want since
# we want to cancel right away.
await self._source.push_frame(CancelFrame())
self._process_down_task.cancel()
self._process_push_task.cancel()
self._process_up_task.cancel()
await self._process_down_task
await self._process_push_task
await self._process_up_task
async def run(self):
self._process_up_task = asyncio.create_task(self._process_up_queue())
self._process_down_task = asyncio.create_task(self._process_down_queue())
await asyncio.gather(self._process_up_task, self._process_down_task)
self._process_push_task = asyncio.create_task(self._process_push_queue())
await asyncio.gather(self._process_up_task, self._process_push_task)
self._finished = True
async def queue_frame(self, frame: Frame):
await self._down_queue.put(frame)
await self._push_queue.put(frame)
async def queue_frames(self, frames: Iterable[Frame] | AsyncIterable[Frame]):
if isinstance(frames, AsyncIterable):
@@ -111,31 +144,40 @@ class PipelineTask:
def _initial_metrics_frame(self) -> MetricsFrame:
processors = self._pipeline.processors_with_metrics()
ttfb = [{"processor": p.name, "value": 0.0} for p in processors]
processing = [{"processor": p.name, "value": 0.0} for p in processors]
return MetricsFrame(ttfb=ttfb, processing=processing)
data = []
for p in processors:
data.append(TTFBMetricsData(processor=p.name, value=0.0))
data.append(ProcessingMetricsData(processor=p.name, value=0.0))
return MetricsFrame(data=data)
async def _process_push_queue(self):
self._clock.start()
async def _process_down_queue(self):
start_frame = StartFrame(
allow_interruptions=self._params.allow_interruptions,
enable_metrics=self._params.enable_metrics,
enable_usage_metrics=self._params.enable_metrics,
report_only_initial_ttfb=self._params.report_only_initial_ttfb
report_only_initial_ttfb=self._params.report_only_initial_ttfb,
clock=self._clock,
)
await self._source.process_frame(start_frame, FrameDirection.DOWNSTREAM)
if self._params.enable_metrics and self._params.send_initial_empty_metrics:
await self._source.process_frame(self._initial_metrics_frame(), FrameDirection.DOWNSTREAM)
await self._source.process_frame(
self._initial_metrics_frame(), FrameDirection.DOWNSTREAM
)
running = True
should_cleanup = True
while running:
try:
frame = await self._down_queue.get()
frame = await self._push_queue.get()
await self._source.process_frame(frame, FrameDirection.DOWNSTREAM)
if isinstance(frame, EndFrame):
await self._wait_for_endframe()
running = not (isinstance(frame, StopTaskFrame) or isinstance(frame, EndFrame))
should_cleanup = not isinstance(frame, StopTaskFrame)
self._down_queue.task_done()
self._push_queue.task_done()
except asyncio.CancelledError:
break
# Cleanup only if we need to.
@@ -146,11 +188,21 @@ class PipelineTask:
self._process_up_task.cancel()
await self._process_up_task
async def _wait_for_endframe(self):
# NOTE(aleix): the Sink element just pushes EndFrames to the down queue,
# so just wait for it. In the future we might do something else here,
# but for now this is fine.
await self._down_queue.get()
async def _process_up_queue(self):
while True:
try:
frame = await self._up_queue.get()
if isinstance(frame, StopTaskFrame):
if isinstance(frame, EndTaskFrame):
await self.queue_frame(EndFrame())
elif isinstance(frame, CancelTaskFrame):
await self.queue_frame(CancelFrame())
elif isinstance(frame, StopTaskFrame):
await self.queue_frame(StopTaskFrame())
self._up_queue.task_done()
except asyncio.CancelledError:

6
src/pipecat/pipeline/merge_pipeline.py → src/pipecat/pipeline/to_be_updated/merge_pipeline.py
View File

@@ -1,5 +1,5 @@
from typing import List
from pipecat.pipeline.frames import EndFrame, EndPipeFrame
from pipecat.frames.frames import EndFrame, EndPipeFrame
from pipecat.pipeline.pipeline import Pipeline
@@ -15,9 +15,7 @@ class SequentialMergePipeline(Pipeline):
for idx, pipeline in enumerate(self.pipelines):
while True:
frame = await pipeline.sink.get()
if isinstance(
frame, EndFrame) or isinstance(
frame, EndPipeFrame):
if isinstance(frame, EndFrame) or isinstance(frame, EndPipeFrame):
break
await self.sink.put(frame)

20
src/pipecat/processors/aggregators/gated.py
View File

@@ -17,7 +17,8 @@ class GatedAggregator(FrameProcessor):
Yields gate-opening frame before any accumulated frames, then ensuing frames
until and not including the gate-closed frame.
>>> from pipecat.pipeline.frames import ImageFrame
Doctest: FIXME to work with asyncio
>>> from pipecat.frames.frames import ImageRawFrame
>>> async def print_frames(aggregator, frame):
... async for frame in aggregator.process_frame(frame):
@@ -28,20 +29,25 @@ class GatedAggregator(FrameProcessor):
>>> aggregator = GatedAggregator(
... gate_close_fn=lambda x: isinstance(x, LLMResponseStartFrame),
... gate_open_fn=lambda x: isinstance(x, ImageFrame),
... gate_open_fn=lambda x: isinstance(x, ImageRawFrame),
... start_open=False)
>>> asyncio.run(print_frames(aggregator, TextFrame("Hello")))
>>> asyncio.run(print_frames(aggregator, TextFrame("Hello again.")))
>>> asyncio.run(print_frames(aggregator, ImageFrame(image=bytes([]), size=(0, 0))))
ImageFrame
>>> asyncio.run(print_frames(aggregator, ImageRawFrame(image=bytes([]), size=(0, 0))))
ImageRawFrame
Hello
Hello again.
>>> asyncio.run(print_frames(aggregator, TextFrame("Goodbye.")))
Goodbye.
"""
def __init__(self, gate_open_fn, gate_close_fn, start_open,
direction: FrameDirection = FrameDirection.DOWNSTREAM):
def __init__(
self,
gate_open_fn,
gate_close_fn,
start_open,
direction: FrameDirection = FrameDirection.DOWNSTREAM,
):
super().__init__()
self._gate_open_fn = gate_open_fn
self._gate_close_fn = gate_close_fn
@@ -74,7 +80,7 @@ class GatedAggregator(FrameProcessor):
if self._gate_open:
await self.push_frame(frame, direction)
for (f, d) in self._accumulator:
for f, d in self._accumulator:
await self.push_frame(f, d)
self._accumulator = []
else:

41
src/pipecat/processors/aggregators/llm_response.py
View File

@@ -4,12 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import sys
from typing import List
from typing import List, Type
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame, OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.frames.frames import (
Frame,
InterimTranscriptionFrame,
@@ -20,14 +16,19 @@ from pipecat.frames.frames import (
LLMMessagesUpdateFrame,
LLMSetToolsFrame,
StartInterruptionFrame,
TranscriptionFrame,
TextFrame,
TranscriptionFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame)
UserStoppedSpeakingFrame,
)
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class LLMResponseAggregator(FrameProcessor):
def __init__(
self,
*,
@@ -35,9 +36,10 @@ class LLMResponseAggregator(FrameProcessor):
role: str,
start_frame,
end_frame,
accumulator_frame: TextFrame,
interim_accumulator_frame: TextFrame | None = None,
handle_interruptions: bool = False
accumulator_frame: Type[TextFrame],
interim_accumulator_frame: Type[TextFrame] | None = None,
handle_interruptions: bool = False,
expect_stripped_words: bool = True, # if True, need to add spaces between words
):
super().__init__()
@@ -48,6 +50,7 @@ class LLMResponseAggregator(FrameProcessor):
self._accumulator_frame = accumulator_frame
self._interim_accumulator_frame = interim_accumulator_frame
self._handle_interruptions = handle_interruptions
self._expect_stripped_words = expect_stripped_words
# Reset our accumulator state.
self._reset()
@@ -109,7 +112,10 @@ class LLMResponseAggregator(FrameProcessor):
await self.push_frame(frame, direction)
elif isinstance(frame, self._accumulator_frame):
if self._aggregating:
self._aggregation += f" {frame.text}" if self._aggregation else frame.text
if self._expect_stripped_words:
self._aggregation += f" {frame.text}" if self._aggregation else frame.text
else:
self._aggregation += frame.text
# We have recevied a complete sentence, so if we have seen the
# end frame and we were still aggregating, it means we should
# send the aggregation.
@@ -176,7 +182,7 @@ class LLMAssistantResponseAggregator(LLMResponseAggregator):
start_frame=LLMFullResponseStartFrame,
end_frame=LLMFullResponseEndFrame,
accumulator_frame=TextFrame,
handle_interruptions=True
handle_interruptions=True,
)
@@ -188,7 +194,7 @@ class LLMUserResponseAggregator(LLMResponseAggregator):
start_frame=UserStartedSpeakingFrame,
end_frame=UserStoppedSpeakingFrame,
accumulator_frame=TranscriptionFrame,
interim_accumulator_frame=InterimTranscriptionFrame
interim_accumulator_frame=InterimTranscriptionFrame,
)
@@ -288,7 +294,7 @@ class LLMContextAggregator(LLMResponseAggregator):
class LLMAssistantContextAggregator(LLMContextAggregator):
def __init__(self, context: OpenAILLMContext):
def __init__(self, context: OpenAILLMContext, *, expect_stripped_words: bool = True):
super().__init__(
messages=[],
context=context,
@@ -296,7 +302,8 @@ class LLMAssistantContextAggregator(LLMContextAggregator):
start_frame=LLMFullResponseStartFrame,
end_frame=LLMFullResponseEndFrame,
accumulator_frame=TextFrame,
handle_interruptions=True
handle_interruptions=True,
expect_stripped_words=expect_stripped_words,
)
@@ -309,5 +316,5 @@ class LLMUserContextAggregator(LLMContextAggregator):
start_frame=UserStartedSpeakingFrame,
end_frame=UserStoppedSpeakingFrame,
accumulator_frame=TranscriptionFrame,
interim_accumulator_frame=InterimTranscriptionFrame
interim_accumulator_frame=InterimTranscriptionFrame,
)

128
src/pipecat/processors/aggregators/openai_llm_context.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import base64
import copy
import io
import json
@@ -13,7 +15,12 @@ from typing import Any, Awaitable, Callable, List
from PIL import Image
from pipecat.frames.frames import Frame, VisionImageRawFrame, FunctionCallInProgressFrame, FunctionCallResultFrame
from pipecat.frames.frames import (
Frame,
VisionImageRawFrame,
FunctionCallInProgressFrame,
FunctionCallResultFrame,
)
from pipecat.processors.frame_processor import FrameProcessor
from loguru import logger
@@ -24,12 +31,13 @@ try:
from openai.types.chat import (
ChatCompletionToolParam,
ChatCompletionToolChoiceOptionParam,
ChatCompletionMessageParam
ChatCompletionMessageParam,
)
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable.")
"In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable."
)
raise Exception(f"Missing module: {e}")
# JSON custom encoder to handle bytes arrays so that we can log contexts
@@ -40,22 +48,21 @@ class CustomEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, io.BytesIO):
# Convert the first 8 bytes to an ASCII hex string
return (f"{obj.getbuffer()[0:8].hex()}...")
return f"{obj.getbuffer()[0:8].hex()}..."
return super().default(obj)
class OpenAILLMContext:
def __init__(
self,
messages: List[ChatCompletionMessageParam] | None = None,
tools: List[ChatCompletionToolParam] | NotGiven = NOT_GIVEN,
tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN
tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN,
):
self._messages: List[ChatCompletionMessageParam] = messages if messages else [
]
self._messages: List[ChatCompletionMessageParam] = messages if messages else []
self._tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = tool_choice
self._tools: List[ChatCompletionToolParam] | NotGiven = tools
self._user_image_request_context = {}
@staticmethod
def from_messages(messages: List[dict]) -> "OpenAILLMContext":
@@ -77,19 +84,10 @@ class OpenAILLMContext:
"""
context = OpenAILLMContext()
buffer = io.BytesIO()
Image.frombytes(
frame.format,
frame.size,
frame.image
).save(
buffer,
format="JPEG")
context.add_message({
"content": frame.text,
"role": "user",
"data": buffer,
"mime_type": "image/jpeg"
})
Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG")
context.add_message(
{"content": frame.text, "role": "user", "data": buffer, "mime_type": "image/jpeg"}
)
return context
@property
@@ -119,9 +117,22 @@ class OpenAILLMContext:
def get_messages_json(self) -> str:
return json.dumps(self._messages, cls=CustomEncoder)
def set_tool_choice(
self, tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven
):
def get_messages_for_logging(self) -> str:
msgs = []
for message in self.messages:
msg = copy.deepcopy(message)
if "content" in msg:
if isinstance(msg["content"], list):
for item in msg["content"]:
if item["type"] == "image_url":
if item["image_url"]["url"].startswith("data:image/"):
item["image_url"]["url"] = "data:image/..."
if "mime_type" in msg and msg["mime_type"].startswith("image/"):
msg["data"] = "..."
msgs.append(msg)
return json.dumps(msgs)
def set_tool_choice(self, tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven):
self._tool_choice = tool_choice
def set_tools(self, tools: List[ChatCompletionToolParam] | NotGiven = NOT_GIVEN):
@@ -129,37 +140,57 @@ class OpenAILLMContext:
tools = NOT_GIVEN
self._tools = tools
async def call_function(self,
f: Callable[[str,
str,
Any,
FrameProcessor,
'OpenAILLMContext',
Callable[[Any],
Awaitable[None]]],
Awaitable[None]],
*,
function_name: str,
tool_call_id: str,
arguments: str,
llm: FrameProcessor) -> None:
def add_image_frame_message(
self, *, format: str, size: tuple[int, int], image: bytes, text: str = None
):
buffer = io.BytesIO()
Image.frombytes(format, size, image).save(buffer, format="JPEG")
encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
content = [
{"type": "text", "text": text},
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}},
]
if text:
content.append({"type": "text", "text": text})
self.add_message({"role": "user", "content": content})
async def call_function(
self,
f: Callable[
[str, str, Any, FrameProcessor, "OpenAILLMContext", Callable[[Any], Awaitable[None]]],
Awaitable[None],
],
*,
function_name: str,
tool_call_id: str,
arguments: str,
llm: FrameProcessor,
run_llm: bool = True,
) -> None:
# Push a SystemFrame downstream. This frame will let our assistant context aggregator
# know that we are in the middle of a function call. Some contexts/aggregators may
# not need this. But some definitely do (Anthropic, for example).
await llm.push_frame(FunctionCallInProgressFrame(
function_name=function_name,
tool_call_id=tool_call_id,
arguments=arguments,
))
# Define a callback function that pushes a FunctionCallResultFrame downstream.
async def function_call_result_callback(result):
await llm.push_frame(FunctionCallResultFrame(
await llm.push_frame(
FunctionCallInProgressFrame(
function_name=function_name,
tool_call_id=tool_call_id,
arguments=arguments,
result=result))
)
)
# Define a callback function that pushes a FunctionCallResultFrame downstream.
async def function_call_result_callback(result):
await llm.push_frame(
FunctionCallResultFrame(
function_name=function_name,
tool_call_id=tool_call_id,
arguments=arguments,
result=result,
run_llm=run_llm,
)
)
await f(function_name, tool_call_id, arguments, llm, self, function_call_result_callback)
@@ -170,4 +201,5 @@ class OpenAILLMContextFrame(Frame):
OpenAIContextAggregator frame processor.
"""
context: OpenAILLMContext

3
src/pipecat/processors/aggregators/sentence.py
View File

@@ -16,7 +16,8 @@ class SentenceAggregator(FrameProcessor):
TextFrame("Hello,") -> None
TextFrame(" world.") -> TextFrame("Hello world.")
Doctest:
Doctest: FIXME to work with asyncio
>>> import asyncio
>>> async def print_frames(aggregator, frame):
... async for frame in aggregator.process_frame(frame):
... print(frame.text)

7
src/pipecat/processors/aggregators/user_response.py
View File

@@ -12,7 +12,8 @@ from pipecat.frames.frames import (
TextFrame,
TranscriptionFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame)
UserStoppedSpeakingFrame,
)
class ResponseAggregator(FrameProcessor):
@@ -25,7 +26,7 @@ class ResponseAggregator(FrameProcessor):
TranscriptionFrame(" world.") -> None
UserStoppedSpeakingFrame() -> TextFrame("Hello world.")
Doctest:
Doctest: FIXME to work with asyncio
>>> async def print_frames(aggregator, frame):
... async for frame in aggregator.process_frame(frame):
... if isinstance(frame, TextFrame):
@@ -49,7 +50,7 @@ class ResponseAggregator(FrameProcessor):
start_frame,
end_frame,
accumulator_frame: TextFrame,
interim_accumulator_frame: TextFrame | None = None
interim_accumulator_frame: TextFrame | None = None,
):
super().__init__()

12
src/pipecat/processors/aggregators/vision_image_frame.py
View File

@@ -4,15 +4,16 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
from pipecat.frames.frames import Frame, ImageRawFrame, TextFrame, VisionImageRawFrame
from pipecat.frames.frames import Frame, InputImageRawFrame, TextFrame, VisionImageRawFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class VisionImageFrameAggregator(FrameProcessor):
"""This aggregator waits for a consecutive TextFrame and an
ImageFrame. After the ImageFrame arrives it will output a VisionImageFrame.
InputImageRawFrame. After the InputImageRawFrame arrives it will output a
VisionImageRawFrame.
>>> from pipecat.pipeline.frames import ImageFrame
>>> from pipecat.frames.frames import ImageFrame
>>> async def print_frames(aggregator, frame):
... async for frame in aggregator.process_frame(frame):
@@ -34,13 +35,14 @@ class VisionImageFrameAggregator(FrameProcessor):
if isinstance(frame, TextFrame):
self._describe_text = frame.text
elif isinstance(frame, ImageRawFrame):
elif isinstance(frame, InputImageRawFrame):
if self._describe_text:
frame = VisionImageRawFrame(
text=self._describe_text,
image=frame.image,
size=frame.size,
format=frame.format)
format=frame.format,
)
await self.push_frame(frame)
self._describe_text = None
else:

64
src/pipecat/processors/async_frame_processor.py
View File

@@ -1,64 +0,0 @@
#
# Copyright (c) 2024, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
from pipecat.frames.frames import EndFrame, Frame, StartInterruptionFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class AsyncFrameProcessor(FrameProcessor):
def __init__(
self,
*,
name: str | None = None,
loop: asyncio.AbstractEventLoop | None = None,
**kwargs):
super().__init__(name=name, loop=loop, **kwargs)
self._create_push_task()
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, StartInterruptionFrame):
await self._handle_interruptions(frame)
async def queue_frame(
self,
frame: Frame,
direction: FrameDirection = FrameDirection.DOWNSTREAM):
await self._push_queue.put((frame, direction))
async def cleanup(self):
self._push_frame_task.cancel()
await self._push_frame_task
async def _handle_interruptions(self, frame: Frame):
# Cancel the task. This will stop pushing frames downstream.
self._push_frame_task.cancel()
await self._push_frame_task
# Push an out-of-band frame (i.e. not using the ordered push
# frame task).
await self.push_frame(frame)
# Create a new queue and task.
self._create_push_task()
def _create_push_task(self):
self._push_queue = asyncio.Queue()
self._push_frame_task = self.get_event_loop().create_task(self._push_frame_task_handler())
async def _push_frame_task_handler(self):
running = True
while running:
try:
(frame, direction) = await self._push_queue.get()
await self.push_frame(frame, direction)
running = not isinstance(frame, EndFrame)
self._push_queue.task_done()
except asyncio.CancelledError:
break

44
src/pipecat/processors/async_generator.py Normal file
View File

@@ -0,0 +1,44 @@
#
# Copyright (c) 2024, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
from typing import Any, AsyncGenerator
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
Frame,
)
from pipecat.processors.frame_processor import FrameProcessor, FrameDirection
from pipecat.serializers.base_serializer import FrameSerializer
class AsyncGeneratorProcessor(FrameProcessor):
def __init__(self, *, serializer: FrameSerializer, **kwargs):
super().__init__(**kwargs)
self._serializer = serializer
self._data_queue = asyncio.Queue()
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction)
if isinstance(frame, (CancelFrame, EndFrame)):
await self._data_queue.put(None)
else:
data = self._serializer.serialize(frame)
if data:
await self._data_queue.put(data)
async def generator(self) -> AsyncGenerator[Any, None]:
running = True
while running:
data = await self._data_queue.get()
running = data is not None
if data:
yield data

9
src/pipecat/processors/filters/frame_filter.py
View File

@@ -11,7 +11,6 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class FrameFilter(FrameProcessor):
def __init__(self, types: List[type]):
super().__init__()
self._types = types
@@ -25,9 +24,11 @@ class FrameFilter(FrameProcessor):
if isinstance(frame, t):
return True
return (isinstance(frame, AppFrame)
or isinstance(frame, ControlFrame)
or isinstance(frame, SystemFrame))
return (
isinstance(frame, AppFrame)
or isinstance(frame, ControlFrame)
or isinstance(frame, SystemFrame)
)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)

1
src/pipecat/processors/filters/function_filter.py
View File

@@ -11,7 +11,6 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class FunctionFilter(FrameProcessor):
def __init__(self, filter: Callable[[Frame], Awaitable[bool]]):
super().__init__()
self._filter = filter

11
src/pipecat/processors/filters/wake_check_filter.py
View File

@@ -21,6 +21,7 @@ class WakeCheckFilter(FrameProcessor):
after a wake phrase has been detected. It also has a keepalive timeout to allow for a brief
period of continued conversation after a wake phrase has been detected.
"""
class WakeState(Enum):
IDLE = 1
AWAKE = 2
@@ -38,8 +39,9 @@ class WakeCheckFilter(FrameProcessor):
self._keepalive_timeout = keepalive_timeout
self._wake_patterns = []
for name in wake_phrases:
pattern = re.compile(r'\b' + r'\s*'.join(re.escape(word)
for word in name.split()) + r'\b', re.IGNORECASE)
pattern = re.compile(
r"\b" + r"\s*".join(re.escape(word) for word in name.split()) + r"\b", re.IGNORECASE
)
self._wake_patterns.append(pattern)
async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -57,7 +59,8 @@ class WakeCheckFilter(FrameProcessor):
if p.state == WakeCheckFilter.WakeState.AWAKE:
if time.time() - p.wake_timer < self._keepalive_timeout:
logger.debug(
f"Wake phrase keepalive timeout has not expired. Pushing {frame}")
f"Wake phrase keepalive timeout has not expired. Pushing {frame}"
)
p.wake_timer = time.time()
await self.push_frame(frame)
return
@@ -73,7 +76,7 @@ class WakeCheckFilter(FrameProcessor):
# and modify the frame in place.
p.state = WakeCheckFilter.WakeState.AWAKE
p.wake_timer = time.time()
frame.text = p.accumulator[match.start():]
frame.text = p.accumulator[match.start() :]
p.accumulator = ""
await self.push_frame(frame)
else:

168
src/pipecat/processors/frame_processor.py
View File

@@ -5,17 +5,22 @@
#
import asyncio
import time
import inspect
from enum import Enum
from pipecat.clocks.base_clock import BaseClock
from pipecat.frames.frames import (
EndFrame,
ErrorFrame,
Frame,
MetricsFrame,
StartFrame,
StartInterruptionFrame,
UserStoppedSpeakingFrame)
StopInterruptionFrame,
SystemFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage, MetricsData
from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMetrics
from pipecat.utils.utils import obj_count, obj_id
from loguru import logger
@@ -26,69 +31,15 @@ class FrameDirection(Enum):
UPSTREAM = 2
class FrameProcessorMetrics:
def __init__(self, name: str):
self._name = name
self._start_ttfb_time = 0
self._start_processing_time = 0
self._should_report_ttfb = True
async def start_ttfb_metrics(self, report_only_initial_ttfb):
if self._should_report_ttfb:
self._start_ttfb_time = time.time()
self._should_report_ttfb = not report_only_initial_ttfb
async def stop_ttfb_metrics(self):
if self._start_ttfb_time == 0:
return None
value = time.time() - self._start_ttfb_time
logger.debug(f"{self._name} TTFB: {value}")
ttfb = {
"processor": self._name,
"value": value
}
self._start_ttfb_time = 0
return MetricsFrame(ttfb=[ttfb])
async def start_processing_metrics(self):
self._start_processing_time = time.time()
async def stop_processing_metrics(self):
if self._start_processing_time == 0:
return None
value = time.time() - self._start_processing_time
logger.debug(f"{self._name} processing time: {value}")
processing = {
"processor": self._name,
"value": value
}
self._start_processing_time = 0
return MetricsFrame(processing=[processing])
async def start_llm_usage_metrics(self, tokens: dict):
logger.debug(
f"{self._name} prompt tokens: {tokens['prompt_tokens']}, completion tokens: {tokens['completion_tokens']}")
return MetricsFrame(tokens=[tokens])
async def start_tts_usage_metrics(self, text: str):
characters = {
"processor": self._name,
"value": len(text),
}
logger.debug(f"{self._name} usage characters: {characters['value']}")
return MetricsFrame(characters=[characters])
class FrameProcessor:
def __init__(
self,
*,
name: str | None = None,
loop: asyncio.AbstractEventLoop | None = None,
**kwargs):
self,
*,
name: str | None = None,
metrics: FrameProcessorMetrics | None = None,
loop: asyncio.AbstractEventLoop | None = None,
**kwargs,
):
self.id: int = obj_id()
self.name = name or f"{self.__class__.__name__}#{obj_count(self)}"
self._parent: "FrameProcessor" | None = None
@@ -96,6 +47,11 @@ class FrameProcessor:
self._next: "FrameProcessor" | None = None
self._loop: asyncio.AbstractEventLoop = loop or asyncio.get_running_loop()
self._event_handlers: dict = {}
# Clock
self._clock: BaseClock | None = None
# Properties
self._allow_interruptions = False
self._enable_metrics = False
@@ -103,7 +59,13 @@ class FrameProcessor:
self._report_only_initial_ttfb = False
# Metrics
self._metrics = FrameProcessorMetrics(name=self.name)
self._metrics = metrics or FrameProcessorMetrics()
self._metrics.set_processor_name(self.name)
# Every processor in Pipecat should only output frames from a single
# task. This avoid problems like audio overlapping. System frames are
# the exception to this rule. This create this task.
self.__create_push_task()
@property
def interruptions_allowed(self):
@@ -124,6 +86,9 @@ class FrameProcessor:
def can_generate_metrics(self) -> bool:
return False
def set_core_metrics_data(self, data: MetricsData):
self._metrics.set_core_metrics_data(data)
async def start_ttfb_metrics(self):
if self.can_generate_metrics() and self.metrics_enabled:
await self._metrics.start_ttfb_metrics(self._report_only_initial_ttfb)
@@ -144,7 +109,7 @@ class FrameProcessor:
if frame:
await self.push_frame(frame)
async def start_llm_usage_metrics(self, tokens: dict):
async def start_llm_usage_metrics(self, tokens: LLMTokenUsage):
if self.can_generate_metrics() and self.usage_metrics_enabled:
frame = await self._metrics.start_llm_usage_metrics(tokens)
if frame:
@@ -177,21 +142,65 @@ class FrameProcessor:
def get_parent(self) -> "FrameProcessor":
return self._parent
def get_clock(self) -> BaseClock:
return self._clock
async def process_frame(self, frame: Frame, direction: FrameDirection):
if isinstance(frame, StartFrame):
self._clock = frame.clock
self._allow_interruptions = frame.allow_interruptions
self._enable_metrics = frame.enable_metrics
self._enable_usage_metrics = frame.enable_usage_metrics
self._report_only_initial_ttfb = frame.report_only_initial_ttfb
elif isinstance(frame, StartInterruptionFrame):
await self._start_interruption()
await self.stop_all_metrics()
elif isinstance(frame, UserStoppedSpeakingFrame):
elif isinstance(frame, StopInterruptionFrame):
self._should_report_ttfb = True
async def push_error(self, error: ErrorFrame):
await self.push_frame(error, FrameDirection.UPSTREAM)
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
if isinstance(frame, SystemFrame):
await self.__internal_push_frame(frame, direction)
else:
await self.__push_queue.put((frame, direction))
def event_handler(self, event_name: str):
def decorator(handler):
self.add_event_handler(event_name, handler)
return handler
return decorator
def add_event_handler(self, event_name: str, handler):
if event_name not in self._event_handlers:
raise Exception(f"Event handler {event_name} not registered")
self._event_handlers[event_name].append(handler)
def _register_event_handler(self, event_name: str):
if event_name in self._event_handlers:
raise Exception(f"Event handler {event_name} already registered")
self._event_handlers[event_name] = []
#
# Handle interruptions
#
async def _start_interruption(self):
# Cancel the task. This will stop pushing frames downstream.
self.__push_frame_task.cancel()
await self.__push_frame_task
# Create a new queue and task.
self.__create_push_task()
async def _stop_interruption(self):
# Nothing to do right now.
pass
async def __internal_push_frame(self, frame: Frame, direction: FrameDirection):
try:
if direction == FrameDirection.DOWNSTREAM and self._next:
logger.trace(f"Pushing {frame} from {self} to {self._next}")
@@ -202,5 +211,30 @@ class FrameProcessor:
except Exception as e:
logger.exception(f"Uncaught exception in {self}: {e}")
def __create_push_task(self):
self.__push_queue = asyncio.Queue()
self.__push_frame_task = self.get_event_loop().create_task(self.__push_frame_task_handler())
async def __push_frame_task_handler(self):
running = True
while running:
try:
(frame, direction) = await self.__push_queue.get()
await self.__internal_push_frame(frame, direction)
running = not isinstance(frame, EndFrame)
self.__push_queue.task_done()
except asyncio.CancelledError:
break
async def _call_event_handler(self, event_name: str, *args, **kwargs):
try:
for handler in self._event_handlers[event_name]:
if inspect.iscoroutinefunction(handler):
await handler(self, *args, **kwargs)
else:
handler(self, *args, **kwargs)
except Exception as e:
logger.exception(f"Exception in event handler {event_name}: {e}")
def __str__(self):
return self.name

7
src/pipecat/processors/frameworks/langchain.py
View File

@@ -11,7 +11,8 @@ from pipecat.frames.frames import (
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMMessagesFrame,
TextFrame)
TextFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from loguru import logger
@@ -20,9 +21,7 @@ try:
from langchain_core.messages import AIMessageChunk
from langchain_core.runnables import Runnable
except ModuleNotFoundError as e:
logger.exception(
"In order to use Langchain, you need to `pip install pipecat-ai[langchain]`. "
)
logger.exception("In order to use Langchain, you need to `pip install pipecat-ai[langchain]`. ")
raise Exception(f"Missing module: {e}")

593
src/pipecat/processors/frameworks/rtvi.py
View File

@@ -5,33 +5,43 @@
#
import asyncio
import base64
from typing import Any, Awaitable, Callable, Dict, List, Literal, Optional, Union
from pydantic import BaseModel, Field, PrivateAttr, ValidationError
from dataclasses import dataclass
from pipecat.frames.frames import (
BotInterruptionFrame,
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
CancelFrame,
DataFrame,
EndFrame,
ErrorFrame,
Frame,
InterimTranscriptionFrame,
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
OutputAudioRawFrame,
StartFrame,
SystemFrame,
TTSStartedFrame,
TTSStoppedFrame,
TextFrame,
TranscriptionFrame,
TransportMessageFrame,
UserStartedSpeakingFrame,
FunctionCallResultFrame,
UserStoppedSpeakingFrame)
UserStoppedSpeakingFrame,
)
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from loguru import logger
RTVI_PROTOCOL_VERSION = "0.1"
RTVI_PROTOCOL_VERSION = "0.2"
ActionResult = Union[bool, int, float, str, list, dict]
@@ -39,8 +49,9 @@ ActionResult = Union[bool, int, float, str, list, dict]
class RTVIServiceOption(BaseModel):
name: str
type: Literal["bool", "number", "string", "array", "object"]
handler: Callable[["RTVIProcessor", str, "RTVIServiceOptionConfig"],
Awaitable[None]] = Field(exclude=True)
handler: Callable[["RTVIProcessor", str, "RTVIServiceOptionConfig"], Awaitable[None]] = Field(
exclude=True
)
class RTVIService(BaseModel):
@@ -70,8 +81,9 @@ class RTVIAction(BaseModel):
action: str
arguments: List[RTVIActionArgument] = []
result: Literal["bool", "number", "string", "array", "object"]
handler: Callable[["RTVIProcessor", str, Dict[str, Any]],
Awaitable[ActionResult]] = Field(exclude=True)
handler: Callable[["RTVIProcessor", str, Dict[str, Any]], Awaitable[ActionResult]] = Field(
exclude=True
)
_arguments_dict: Dict[str, RTVIActionArgument] = PrivateAttr(default={})
def model_post_init(self, __context: Any) -> None:
@@ -116,12 +128,19 @@ class RTVIActionRun(BaseModel):
arguments: Optional[List[RTVIActionRunArgument]] = None
@dataclass
class RTVIActionFrame(DataFrame):
rtvi_action_run: RTVIActionRun
message_id: Optional[str] = None
class RTVIMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
type: str
id: str
data: Optional[Dict[str, Any]] = None
#
# Pipecat -> Client responses and messages.
#
@@ -230,17 +249,75 @@ class RTVILLMFunctionCallResultData(BaseModel):
result: dict | str
class RTVITranscriptionMessageData(BaseModel):
class RTVIBotLLMStartedMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
type: Literal["bot-llm-started"] = "bot-llm-started"
class RTVIBotLLMStoppedMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
type: Literal["bot-llm-stopped"] = "bot-llm-stopped"
class RTVIBotTTSStartedMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
type: Literal["bot-tts-started"] = "bot-tts-started"
class RTVIBotTTSStoppedMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
type: Literal["bot-tts-stopped"] = "bot-tts-stopped"
class RTVITextMessageData(BaseModel):
text: str
class RTVIBotLLMTextMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
type: Literal["bot-llm-text"] = "bot-llm-text"
data: RTVITextMessageData
class RTVIBotTTSTextMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
type: Literal["bot-tts-text"] = "bot-tts-text"
data: RTVITextMessageData
class RTVIAudioMessageData(BaseModel):
audio: str
sample_rate: int
num_channels: int
class RTVIBotAudioMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
type: Literal["bot-audio"] = "bot-audio"
data: RTVIAudioMessageData
class RTVIBotTranscriptionMessageData(BaseModel):
text: str
class RTVIBotTranscriptionMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
type: Literal["bot-transcription"] = "bot-transcription"
data: RTVIBotTranscriptionMessageData
class RTVIUserTranscriptionMessageData(BaseModel):
text: str
user_id: str
timestamp: str
final: bool
class RTVITranscriptionMessage(BaseModel):
class RTVIUserTranscriptionMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
type: Literal["user-transcription"] = "user-transcription"
data: RTVITranscriptionMessageData
data: RTVIUserTranscriptionMessageData
class RTVIUserStartedSpeakingMessage(BaseModel):
@@ -267,186 +344,31 @@ class RTVIProcessorParams(BaseModel):
send_bot_ready: bool = True
class RTVIProcessor(FrameProcessor):
class RTVIFrameProcessor(FrameProcessor):
def __init__(self, direction: FrameDirection = FrameDirection.DOWNSTREAM, **kwargs):
super().__init__(**kwargs)
self._direction = direction
def __init__(self,
*,
config: RTVIConfig = RTVIConfig(config=[]),
params: RTVIProcessorParams = RTVIProcessorParams()):
super().__init__()
self._config = config
self._params = params
async def _push_transport_message(self, model: BaseModel, exclude_none: bool = True):
frame = TransportMessageFrame(
message=model.model_dump(exclude_none=exclude_none), urgent=True
)
await self.push_frame(frame, self._direction)
self._pipeline: FrameProcessor | None = None
self._pipeline_started = False
self._client_ready = False
self._client_ready_id = ""
self._registered_actions: Dict[str, RTVIAction] = {}
self._registered_services: Dict[str, RTVIService] = {}
self._push_frame_task = self.get_event_loop().create_task(self._push_frame_task_handler())
self._push_queue = asyncio.Queue()
self._message_task = self.get_event_loop().create_task(self._message_task_handler())
self._message_queue = asyncio.Queue()
def register_action(self, action: RTVIAction):
id = self._action_id(action.service, action.action)
self._registered_actions[id] = action
def register_service(self, service: RTVIService):
self._registered_services[service.name] = service
async def interrupt_bot(self):
await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM)
async def send_error(self, error: str):
message = RTVIError(data=RTVIErrorData(error=error, fatal=False))
await self._push_transport_message(message)
async def set_client_ready(self):
if not self._client_ready:
self._client_ready = True
await self._maybe_send_bot_ready()
async def handle_function_call(
self,
function_name: str,
tool_call_id: str,
arguments: dict,
llm: FrameProcessor,
context: OpenAILLMContext,
result_callback):
fn = RTVILLMFunctionCallMessageData(
function_name=function_name,
tool_call_id=tool_call_id,
args=arguments)
message = RTVILLMFunctionCallMessage(data=fn)
await self._push_transport_message(message, exclude_none=False)
async def handle_function_call_start(
self,
function_name: str,
llm: FrameProcessor,
context: OpenAILLMContext):
fn = RTVILLMFunctionCallStartMessageData(function_name=function_name)
message = RTVILLMFunctionCallStartMessage(data=fn)
await self._push_transport_message(message, exclude_none=False)
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
if isinstance(frame, SystemFrame):
await super().push_frame(frame, direction)
else:
await self._internal_push_frame(frame, direction)
class RTVISpeakingProcessor(RTVIFrameProcessor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# Specific system frames
if isinstance(frame, CancelFrame):
await self._cancel(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, ErrorFrame):
await self._send_error_frame(frame)
await self.push_frame(frame, direction)
# All other system frames
elif isinstance(frame, SystemFrame):
await self.push_frame(frame, direction)
# Control frames
elif isinstance(frame, StartFrame):
# Push StartFrame before start(), because we want StartFrame to be
# processed by every processor before any other frame is processed.
await self.push_frame(frame, direction)
await self._start(frame)
elif isinstance(frame, EndFrame):
# Push EndFrame before stop(), because stop() waits on the task to
# finish and the task finishes when EndFrame is processed.
await self.push_frame(frame, direction)
await self._stop(frame)
elif isinstance(frame, UserStartedSpeakingFrame) or isinstance(frame, UserStoppedSpeakingFrame):
await self.push_frame(frame, direction)
if isinstance(frame, (UserStartedSpeakingFrame, UserStoppedSpeakingFrame)):
await self._handle_interruptions(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, BotStartedSpeakingFrame) or isinstance(frame, BotStoppedSpeakingFrame):
elif isinstance(frame, (BotStartedSpeakingFrame, BotStoppedSpeakingFrame)):
await self._handle_bot_speaking(frame)
await self.push_frame(frame, direction)
# Data frames
elif isinstance(frame, TranscriptionFrame) or isinstance(frame, InterimTranscriptionFrame):
await self._handle_transcriptions(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, TransportMessageFrame):
await self._message_queue.put(frame)
# Other frames
else:
await self.push_frame(frame, direction)
async def cleanup(self):
if self._pipeline:
await self._pipeline.cleanup()
async def _start(self, frame: StartFrame):
self._pipeline_started = True
await self._maybe_send_bot_ready()
async def _stop(self, frame: EndFrame):
# We need to cancel the message task handler because that one is not
# processing EndFrames.
self._message_task.cancel()
await self._message_task
await self._push_frame_task
async def _cancel(self, frame: CancelFrame):
self._message_task.cancel()
await self._message_task
self._push_frame_task.cancel()
await self._push_frame_task
async def _internal_push_frame(
self,
frame: Frame | None,
direction: FrameDirection | None = FrameDirection.DOWNSTREAM):
await self._push_queue.put((frame, direction))
async def _push_frame_task_handler(self):
running = True
while running:
try:
(frame, direction) = await self._push_queue.get()
await super().push_frame(frame, direction)
self._push_queue.task_done()
running = not isinstance(frame, EndFrame)
except asyncio.CancelledError:
break
async def _push_transport_message(self, model: BaseModel, exclude_none: bool = True):
frame = TransportMessageFrame(
message=model.model_dump(exclude_none=exclude_none),
urgent=True)
await self.push_frame(frame)
async def _handle_transcriptions(self, frame: Frame):
# TODO(aleix): Once we add support for using custom pipelines, the STTs will
# be in the pipeline after this processor.
message = None
if isinstance(frame, TranscriptionFrame):
message = RTVITranscriptionMessage(
data=RTVITranscriptionMessageData(
text=frame.text,
user_id=frame.user_id,
timestamp=frame.timestamp,
final=True))
elif isinstance(frame, InterimTranscriptionFrame):
message = RTVITranscriptionMessage(
data=RTVITranscriptionMessageData(
text=frame.text,
user_id=frame.user_id,
timestamp=frame.timestamp,
final=False))
if message:
await self._push_transport_message(message)
async def _handle_interruptions(self, frame: Frame):
message = None
@@ -468,23 +390,295 @@ class RTVIProcessor(FrameProcessor):
if message:
await self._push_transport_message(message)
class RTVIUserTranscriptionProcessor(RTVIFrameProcessor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction)
if isinstance(frame, (TranscriptionFrame, InterimTranscriptionFrame)):
await self._handle_user_transcriptions(frame)
async def _handle_user_transcriptions(self, frame: Frame):
message = None
if isinstance(frame, TranscriptionFrame):
message = RTVIUserTranscriptionMessage(
data=RTVIUserTranscriptionMessageData(
text=frame.text, user_id=frame.user_id, timestamp=frame.timestamp, final=True
)
)
elif isinstance(frame, InterimTranscriptionFrame):
message = RTVIUserTranscriptionMessage(
data=RTVIUserTranscriptionMessageData(
text=frame.text, user_id=frame.user_id, timestamp=frame.timestamp, final=False
)
)
if message:
await self._push_transport_message(message)
class RTVIBotLLMProcessor(RTVIFrameProcessor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction)
if isinstance(frame, LLMFullResponseStartFrame):
await self._push_transport_message(RTVIBotLLMStartedMessage())
elif isinstance(frame, LLMFullResponseEndFrame):
await self._push_transport_message(RTVIBotLLMStoppedMessage())
class RTVIBotTTSProcessor(RTVIFrameProcessor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction)
if isinstance(frame, TTSStartedFrame):
await self._push_transport_message(RTVIBotTTSStartedMessage())
elif isinstance(frame, TTSStoppedFrame):
await self._push_transport_message(RTVIBotTTSStoppedMessage())
class RTVIBotLLMTextProcessor(RTVIFrameProcessor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction)
if isinstance(frame, TextFrame):
await self._handle_text(frame)
async def _handle_text(self, frame: TextFrame):
message = RTVIBotLLMTextMessage(data=RTVITextMessageData(text=frame.text))
await self._push_transport_message(message)
class RTVIBotTTSTextProcessor(RTVIFrameProcessor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction)
if isinstance(frame, TextFrame):
await self._handle_text(frame)
async def _handle_text(self, frame: TextFrame):
message = RTVIBotTTSTextMessage(data=RTVITextMessageData(text=frame.text))
await self._push_transport_message(message)
class RTVIBotAudioProcessor(RTVIFrameProcessor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
await self.push_frame(frame, direction)
if isinstance(frame, OutputAudioRawFrame):
await self._handle_audio(frame)
async def _handle_audio(self, frame: OutputAudioRawFrame):
encoded = base64.b64encode(frame.audio).decode("utf-8")
message = RTVIBotAudioMessage(
data=RTVIAudioMessageData(
audio=encoded, sample_rate=frame.sample_rate, num_channels=frame.num_channels
)
)
await self._push_transport_message(message)
class RTVIProcessor(FrameProcessor):
def __init__(
self,
*,
config: RTVIConfig = RTVIConfig(config=[]),
params: RTVIProcessorParams = RTVIProcessorParams(),
**kwargs,
):
super().__init__(**kwargs)
self._config = config
self._params = params
self._pipeline: FrameProcessor | None = None
self._pipeline_started = False
self._client_ready = False
self._client_ready_id = ""
self._registered_actions: Dict[str, RTVIAction] = {}
self._registered_services: Dict[str, RTVIService] = {}
# A task to process incoming action frames.
self._action_task = self.get_event_loop().create_task(self._action_task_handler())
self._action_queue = asyncio.Queue()
# A task to process incoming transport messages.
self._message_task = self.get_event_loop().create_task(self._message_task_handler())
self._message_queue = asyncio.Queue()
self._register_event_handler("on_bot_ready")
def register_action(self, action: RTVIAction):
id = self._action_id(action.service, action.action)
self._registered_actions[id] = action
def register_service(self, service: RTVIService):
self._registered_services[service.name] = service
async def interrupt_bot(self):
await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM)
async def send_error(self, error: str):
message = RTVIError(data=RTVIErrorData(error=error, fatal=False))
await self._push_transport_message(message)
async def set_client_ready(self):
if not self._client_ready:
self._client_ready = True
await self._maybe_send_bot_ready()
async def handle_message(self, message: RTVIMessage):
await self._message_queue.put(message)
async def handle_function_call(
self,
function_name: str,
tool_call_id: str,
arguments: dict,
llm: FrameProcessor,
context: OpenAILLMContext,
result_callback,
):
fn = RTVILLMFunctionCallMessageData(
function_name=function_name, tool_call_id=tool_call_id, args=arguments
)
message = RTVILLMFunctionCallMessage(data=fn)
await self._push_transport_message(message, exclude_none=False)
async def handle_function_call_start(
self, function_name: str, llm: FrameProcessor, context: OpenAILLMContext
):
fn = RTVILLMFunctionCallStartMessageData(function_name=function_name)
message = RTVILLMFunctionCallStartMessage(data=fn)
await self._push_transport_message(message, exclude_none=False)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# Specific system frames
if isinstance(frame, StartFrame):
# Push StartFrame before start(), because we want StartFrame to be
# processed by every processor before any other frame is processed.
await self.push_frame(frame, direction)
await self._start(frame)
elif isinstance(frame, CancelFrame):
await self._cancel(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, ErrorFrame):
await self._send_error_frame(frame)
await self.push_frame(frame, direction)
# All other system frames
elif isinstance(frame, SystemFrame):
await self.push_frame(frame, direction)
# Control frames
elif isinstance(frame, EndFrame):
# Push EndFrame before stop(), because stop() waits on the task to
# finish and the task finishes when EndFrame is processed.
await self.push_frame(frame, direction)
await self._stop(frame)
# Data frames
elif isinstance(frame, TransportMessageFrame):
await self._handle_transport_message(frame)
elif isinstance(frame, RTVIActionFrame):
await self._action_queue.put(frame)
# Other frames
else:
await self.push_frame(frame, direction)
async def cleanup(self):
if self._pipeline:
await self._pipeline.cleanup()
async def _start(self, frame: StartFrame):
self._pipeline_started = True
await self._maybe_send_bot_ready()
async def _stop(self, frame: EndFrame):
if self._action_task:
self._action_task.cancel()
await self._action_task
self._action_task = None
if self._message_task:
self._message_task.cancel()
await self._message_task
self._message_task = None
async def _cancel(self, frame: CancelFrame):
if self._action_task:
self._action_task.cancel()
await self._action_task
self._action_task = None
if self._message_task:
self._message_task.cancel()
await self._message_task
self._message_task = None
async def _push_transport_message(self, model: BaseModel, exclude_none: bool = True):
frame = TransportMessageFrame(
message=model.model_dump(exclude_none=exclude_none), urgent=True
)
await self.push_frame(frame)
async def _action_task_handler(self):
while True:
try:
frame = await self._action_queue.get()
await self._handle_action(frame.message_id, frame.rtvi_action_run)
self._action_queue.task_done()
except asyncio.CancelledError:
break
async def _message_task_handler(self):
while True:
try:
frame = await self._message_queue.get()
await self._handle_message(frame)
message = await self._message_queue.get()
await self._handle_message(message)
self._message_queue.task_done()
except asyncio.CancelledError:
break
async def _handle_message(self, frame: TransportMessageFrame):
async def _handle_transport_message(self, frame: TransportMessageFrame):
try:
message = RTVIMessage.model_validate(frame.message)
await self._message_queue.put(message)
except ValidationError as e:
await self.send_error(f"Invalid incoming message: {e}")
logger.warning(f"Invalid incoming message: {e}")
return
await self.send_error(f"Invalid RTVI transport message: {e}")
logger.warning(f"Invalid RTVI transport message: {e}")
async def _handle_message(self, message: RTVIMessage):
try:
match message.type:
case "client-ready":
@@ -500,7 +694,8 @@ class RTVIProcessor(FrameProcessor):
await self._handle_update_config(message.id, update_config)
case "action":
action = RTVIActionRun.model_validate(message.data)
await self._handle_action(message.id, action)
action_frame = RTVIActionFrame(message_id=message.id, rtvi_action_run=action)
await self._action_queue.put(action_frame)
case "llm-function-call-result":
data = RTVILLMFunctionCallResultData.model_validate(message.data)
await self._handle_function_call_result(data)
@@ -509,8 +704,8 @@ class RTVIProcessor(FrameProcessor):
await self._send_error_response(message.id, f"Unsupported type {message.type}")
except ValidationError as e:
await self._send_error_response(message.id, f"Invalid incoming message: {e}")
logger.warning(f"Invalid incoming message: {e}")
await self._send_error_response(message.id, f"Invalid message: {e}")
logger.warning(f"Invalid message: {e}")
except Exception as e:
await self._send_error_response(message.id, f"Exception processing message: {e}")
logger.warning(f"Exception processing message: {e}")
@@ -567,10 +762,11 @@ class RTVIProcessor(FrameProcessor):
function_name=data.function_name,
tool_call_id=data.tool_call_id,
arguments=data.arguments,
result=data.result)
result=data.result,
)
await self.push_frame(frame)
async def _handle_action(self, request_id: str, data: RTVIActionRun):
async def _handle_action(self, request_id: str | None, data: RTVIActionRun):
action_id = self._action_id(data.service, data.action)
if action_id not in self._registered_actions:
await self._send_error_response(request_id, f"Action {action_id} not registered")
@@ -581,13 +777,17 @@ class RTVIProcessor(FrameProcessor):
for arg in data.arguments:
arguments[arg.name] = arg.value
result = await action.handler(self, action.service, arguments)
message = RTVIActionResponse(id=request_id, data=RTVIActionResponseData(result=result))
await self._push_transport_message(message)
# Only send a response if request_id is present. Things that don't care about
# action responses (such as webhooks) don't set a request_id
if request_id:
message = RTVIActionResponse(id=request_id, data=RTVIActionResponseData(result=result))
await self._push_transport_message(message)
async def _maybe_send_bot_ready(self):
if self._pipeline_started and self._client_ready:
await self._send_bot_ready()
await self._update_config(self._config, False)
await self._send_bot_ready()
await self._call_event_handler("on_bot_ready")
async def _send_bot_ready(self):
if not self._params.send_bot_ready:
@@ -595,9 +795,8 @@ class RTVIProcessor(FrameProcessor):
message = RTVIBotReady(
id=self._client_ready_id,
data=RTVIBotReadyData(
version=RTVI_PROTOCOL_VERSION,
config=self._config.config))
data=RTVIBotReadyData(version=RTVI_PROTOCOL_VERSION, config=self._config.config),
)
await self._push_transport_message(message)
async def _send_error_frame(self, frame: ErrorFrame):

92
src/pipecat/processors/gstreamer/pipeline_source.py
View File

@@ -9,26 +9,29 @@ import asyncio
from pydantic import BaseModel
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,
EndFrame,
Frame,
ImageRawFrame,
OutputAudioRawFrame,
OutputImageRawFrame,
StartFrame,
SystemFrame)
SystemFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from loguru import logger
try:
import gi
gi.require_version('Gst', '1.0')
gi.require_version('GstApp', '1.0')
gi.require_version("Gst", "1.0")
gi.require_version("GstApp", "1.0")
from gi.repository import Gst, GstApp
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use GStreamer, you need to `pip install pipecat-ai[gstreamer]`. Also, you need to install GStreamer in your system.")
"In order to use GStreamer, you need to `pip install pipecat-ai[gstreamer]`. Also, you need to install GStreamer in your system."
)
raise Exception(f"Missing module: {e}")
@@ -62,78 +65,42 @@ class GStreamerPipelineSource(FrameProcessor):
bus.add_signal_watch()
bus.connect("message", self._on_gstreamer_message)
# Create push frame task. This is the task that will push frames in
# order. We also guarantee that all frames are pushed in the same task.
self._create_push_task()
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# Specific system frames
if isinstance(frame, CancelFrame):
if isinstance(frame, StartFrame):
# Push StartFrame before start(), because we want StartFrame to be
# processed by every processor before any other frame is processed.
await self.push_frame(frame, direction)
await self._start(frame)
elif isinstance(frame, CancelFrame):
await self._cancel(frame)
await self.push_frame(frame, direction)
# All other system frames
elif isinstance(frame, SystemFrame):
await self.push_frame(frame, direction)
# Control frames
elif isinstance(frame, StartFrame):
# Push StartFrame before start(), because we want StartFrame to be
# processed by every processor before any other frame is processed.
await self._internal_push_frame(frame, direction)
await self._start(frame)
elif isinstance(frame, EndFrame):
# Push EndFrame before stop(), because stop() waits on the task to
# finish and the task finishes when EndFrame is processed.
await self._internal_push_frame(frame, direction)
await self.push_frame(frame, direction)
await self._stop(frame)
# Other frames
else:
await self._internal_push_frame(frame, direction)
await self.push_frame(frame, direction)
async def _start(self, frame: StartFrame):
self._player.set_state(Gst.State.PLAYING)
async def _stop(self, frame: EndFrame):
self._player.set_state(Gst.State.NULL)
# Wait for the push frame task to finish. It will finish when the
# EndFrame is actually processed.
await self._push_frame_task
async def _cancel(self, frame: CancelFrame):
self._player.set_state(Gst.State.NULL)
# Cancel all the tasks and wait for them to finish.
self._push_frame_task.cancel()
await self._push_frame_task
#
# Push frames task
#
def _create_push_task(self):
loop = self.get_event_loop()
self._push_queue = asyncio.Queue()
self._push_frame_task = loop.create_task(self._push_frame_task_handler())
async def _internal_push_frame(
self,
frame: Frame | None,
direction: FrameDirection | None = FrameDirection.DOWNSTREAM):
await self._push_queue.put((frame, direction))
async def _push_frame_task_handler(self):
running = True
while running:
try:
(frame, direction) = await self._push_queue.get()
await self.push_frame(frame, direction)
running = not isinstance(frame, EndFrame)
self._push_queue.task_done()
except asyncio.CancelledError:
break
#
# GStreaner
# GStreamer
#
def _on_gstreamer_message(self, bus: Gst.Bus, message: Gst.Message):
@@ -156,7 +123,8 @@ class GStreamerPipelineSource(FrameProcessor):
audioresample = Gst.ElementFactory.make("audioresample", None)
audiocapsfilter = Gst.ElementFactory.make("capsfilter", None)
audiocaps = Gst.Caps.from_string(
f"audio/x-raw,format=S16LE,rate={self._out_params.audio_sample_rate},channels={self._out_params.audio_channels},layout=interleaved")
f"audio/x-raw,format=S16LE,rate={self._out_params.audio_sample_rate},channels={self._out_params.audio_channels},layout=interleaved"
)
audiocapsfilter.set_property("caps", audiocaps)
appsink_audio = Gst.ElementFactory.make("appsink", None)
appsink_audio.set_property("emit-signals", True)
@@ -188,7 +156,8 @@ class GStreamerPipelineSource(FrameProcessor):
videoscale = Gst.ElementFactory.make("videoscale", None)
videocapsfilter = Gst.ElementFactory.make("capsfilter", None)
videocaps = Gst.Caps.from_string(
f"video/x-raw,format=RGB,width={self._out_params.video_width},height={self._out_params.video_height}")
f"video/x-raw,format=RGB,width={self._out_params.video_width},height={self._out_params.video_height}"
)
videocapsfilter.set_property("caps", videocaps)
appsink_video = Gst.ElementFactory.make("appsink", None)
@@ -218,20 +187,23 @@ class GStreamerPipelineSource(FrameProcessor):
def _appsink_audio_new_sample(self, appsink: GstApp.AppSink):
buffer = appsink.pull_sample().get_buffer()
(_, info) = buffer.map(Gst.MapFlags.READ)
frame = AudioRawFrame(audio=info.data,
sample_rate=self._out_params.audio_sample_rate,
num_channels=self._out_params.audio_channels)
asyncio.run_coroutine_threadsafe(self._internal_push_frame(frame), self.get_event_loop())
frame = OutputAudioRawFrame(
audio=info.data,
sample_rate=self._out_params.audio_sample_rate,
num_channels=self._out_params.audio_channels,
)
asyncio.run_coroutine_threadsafe(self.push_frame(frame), self.get_event_loop())
buffer.unmap(info)
return Gst.FlowReturn.OK
def _appsink_video_new_sample(self, appsink: GstApp.AppSink):
buffer = appsink.pull_sample().get_buffer()
(_, info) = buffer.map(Gst.MapFlags.READ)
frame = ImageRawFrame(
frame = OutputImageRawFrame(
image=info.data,
size=(self._out_params.video_width, self._out_params.video_height),
format="RGB")
asyncio.run_coroutine_threadsafe(self._internal_push_frame(frame), self.get_event_loop())
format="RGB",
)
asyncio.run_coroutine_threadsafe(self.push_frame(frame), self.get_event_loop())
buffer.unmap(info)
return Gst.FlowReturn.OK

30
src/pipecat/processors/idle_frame_processor.py
View File

@@ -8,28 +8,24 @@ import asyncio
from typing import Awaitable, Callable, List
from pipecat.frames.frames import Frame, SystemFrame
from pipecat.processors.async_frame_processor import AsyncFrameProcessor
from pipecat.processors.frame_processor import FrameDirection
from pipecat.frames.frames import Frame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class IdleFrameProcessor(AsyncFrameProcessor):
class IdleFrameProcessor(FrameProcessor):
"""This class waits to receive any frame or list of desired frames within a
given timeout. If the timeout is reached before receiving any of those
frames the provided callback will be called.
The callback can then be used to push frames downstream by using
`queue_frame()` (or `push_frame()` for system frames).
"""
def __init__(
self,
*,
callback: Callable[["IdleFrameProcessor"], Awaitable[None]],
timeout: float,
types: List[type] = [],
**kwargs):
self,
*,
callback: Callable[["IdleFrameProcessor"], Awaitable[None]],
timeout: float,
types: List[type] = [],
**kwargs,
):
super().__init__(**kwargs)
self._callback = callback
@@ -41,10 +37,7 @@ class IdleFrameProcessor(AsyncFrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, SystemFrame):
await self.push_frame(frame, direction)
else:
await self.queue_frame(frame, direction)
await self.push_frame(frame, direction)
# If we are not waiting for any specific frame set the event, otherwise
# check if we have received one of the desired frames.
@@ -55,7 +48,6 @@ class IdleFrameProcessor(AsyncFrameProcessor):
if isinstance(frame, t):
self._idle_event.set()
# If we are not waiting for any specific frame set the event, otherwise
async def cleanup(self):
self._idle_task.cancel()
await self._idle_task

5
src/pipecat/processors/logger.py
View File

@@ -8,6 +8,7 @@ from pipecat.frames.frames import BotSpeakingFrame, Frame, AudioRawFrame, Transp
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from loguru import logger
from typing import Optional
logger = logger.opt(ansi=True)
@@ -19,7 +20,9 @@ class FrameLogger(FrameProcessor):
ignored_frame_types: Optional[list] = [
BotSpeakingFrame,
AudioRawFrame,
TransportMessageFrame]):
TransportMessageFrame,
],
):
super().__init__()
self._prefix = prefix
self._color = color

80
src/pipecat/processors/metrics/frame_processor_metrics.py Normal file
View File

@@ -0,0 +1,80 @@
import time
from pipecat.frames.frames import MetricsFrame
from pipecat.metrics.metrics import (
LLMTokenUsage,
LLMUsageMetricsData,
MetricsData,
ProcessingMetricsData,
TTFBMetricsData,
TTSUsageMetricsData,
)
from loguru import logger
class FrameProcessorMetrics:
def __init__(self):
self._start_ttfb_time = 0
self._start_processing_time = 0
self._should_report_ttfb = True
def _processor_name(self):
return self._core_metrics_data.processor
def _model_name(self):
return self._core_metrics_data.model
def set_core_metrics_data(self, data: MetricsData):
self._core_metrics_data = data
def set_processor_name(self, name: str):
self._core_metrics_data = MetricsData(processor=name)
async def start_ttfb_metrics(self, report_only_initial_ttfb):
if self._should_report_ttfb:
self._start_ttfb_time = time.time()
self._should_report_ttfb = not report_only_initial_ttfb
async def stop_ttfb_metrics(self):
if self._start_ttfb_time == 0:
return None
value = time.time() - self._start_ttfb_time
logger.debug(f"{self._processor_name()} TTFB: {value}")
ttfb = TTFBMetricsData(
processor=self._processor_name(), value=value, model=self._model_name()
)
self._start_ttfb_time = 0
return MetricsFrame(data=[ttfb])
async def start_processing_metrics(self):
self._start_processing_time = time.time()
async def stop_processing_metrics(self):
if self._start_processing_time == 0:
return None
value = time.time() - self._start_processing_time
logger.debug(f"{self._processor_name()} processing time: {value}")
processing = ProcessingMetricsData(
processor=self._processor_name(), value=value, model=self._model_name()
)
self._start_processing_time = 0
return MetricsFrame(data=[processing])
async def start_llm_usage_metrics(self, tokens: LLMTokenUsage):
logger.debug(
f"{self._processor_name()} prompt tokens: {tokens.prompt_tokens}, completion tokens: {tokens.completion_tokens}"
)
value = LLMUsageMetricsData(
processor=self._processor_name(), model=self._model_name(), value=tokens
)
return MetricsFrame(data=[value])
async def start_tts_usage_metrics(self, text: str):
characters = TTSUsageMetricsData(
processor=self._processor_name(), model=self._model_name(), value=len(text)
)
logger.debug(f"{self._processor_name()} usage characters: {characters.value}")
return MetricsFrame(data=[characters])

55
src/pipecat/processors/metrics/sentry.py Normal file
View File

@@ -0,0 +1,55 @@
import time
from loguru import logger
try:
import sentry_sdk
sentry_available = sentry_sdk.is_initialized()
if not sentry_available:
logger.warning("Sentry SDK not initialized. Sentry features will be disabled.")
except ImportError:
sentry_available = False
logger.warning("Sentry SDK not installed. Sentry features will be disabled.")
from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMetrics
class SentryMetrics(FrameProcessorMetrics):
def __init__(self):
super().__init__()
self._ttfb_metrics_span = None
self._processing_metrics_span = None
async def start_ttfb_metrics(self, report_only_initial_ttfb):
if self._should_report_ttfb:
self._start_ttfb_time = time.time()
if sentry_available:
self._ttfb_metrics_span = sentry_sdk.start_span(
op="ttfb",
description=f"TTFB for {self._processor_name()}",
start_timestamp=self._start_ttfb_time,
)
logger.debug(f"Sentry Span ID: {self._ttfb_metrics_span.span_id} Description: {
self._ttfb_metrics_span.description} started.")
self._should_report_ttfb = not report_only_initial_ttfb
async def stop_ttfb_metrics(self):
stop_time = time.time()
if sentry_available:
self._ttfb_metrics_span.finish(end_timestamp=stop_time)
async def start_processing_metrics(self):
self._start_processing_time = time.time()
if sentry_available:
self._processing_metrics_span = sentry_sdk.start_span(
op="processing",
description=f"Processing for {self._processor_name()}",
start_timestamp=self._start_processing_time,
)
logger.debug(f"Sentry Span ID: {self._processing_metrics_span.span_id} Description: {
self._processing_metrics_span.description} started.")
async def stop_processing_metrics(self):
stop_time = time.time()
if sentry_available:
self._processing_metrics_span.finish(end_timestamp=stop_time)

28
src/pipecat/processors/user_idle_processor.py
View File

@@ -11,29 +11,26 @@ from typing import Awaitable, Callable
from pipecat.frames.frames import (
BotSpeakingFrame,
Frame,
SystemFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame)
from pipecat.processors.async_frame_processor import AsyncFrameProcessor
from pipecat.processors.frame_processor import FrameDirection
UserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class UserIdleProcessor(AsyncFrameProcessor):
class UserIdleProcessor(FrameProcessor):
"""This class is useful to check if the user is interacting with the bot
within a given timeout. If the timeout is reached before any interaction
occurred the provided callback will be called.
The callback can then be used to push frames downstream by using
`queue_frame()` (or `push_frame()` for system frames).
"""
def __init__(
self,
*,
callback: Callable[["UserIdleProcessor"], Awaitable[None]],
timeout: float,
**kwargs):
self,
*,
callback: Callable[["UserIdleProcessor"], Awaitable[None]],
timeout: float,
**kwargs,
):
super().__init__(**kwargs)
self._callback = callback
@@ -46,10 +43,7 @@ class UserIdleProcessor(AsyncFrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, SystemFrame):
await self.push_frame(frame, direction)
else:
await self.queue_frame(frame, direction)
await self.push_frame(frame, direction)
# We shouldn't call the idle callback if the user or the bot are speaking.
if isinstance(frame, UserStartedSpeakingFrame):

1
src/pipecat/serializers/base_serializer.py
View File

@@ -10,7 +10,6 @@ from pipecat.frames.frames import Frame
class FrameSerializer(ABC):
@abstractmethod
def serialize(self, frame: Frame) -> str | bytes | None:
pass

15
src/pipecat/serializers/livekit.py
View File

@@ -7,7 +7,7 @@
import ctypes
import pickle
from pipecat.frames.frames import AudioRawFrame, Frame
from pipecat.frames.frames import Frame, InputAudioRawFrame, OutputAudioRawFrame
from pipecat.serializers.base_serializer import FrameSerializer
from loguru import logger
@@ -16,18 +16,13 @@ try:
from livekit.rtc import AudioFrame
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use LiveKit, you need to `pip install pipecat-ai[livekit]`.")
logger.error("In order to use LiveKit, you need to `pip install pipecat-ai[livekit]`.")
raise Exception(f"Missing module: {e}")
class LivekitFrameSerializer(FrameSerializer):
SERIALIZABLE_TYPES = {
AudioRawFrame: "audio",
}
def serialize(self, frame: Frame) -> str | bytes | None:
if not isinstance(frame, AudioRawFrame):
if not isinstance(frame, OutputAudioRawFrame):
return None
audio_frame = AudioFrame(
data=frame.audio,
@@ -38,8 +33,8 @@ class LivekitFrameSerializer(FrameSerializer):
return pickle.dumps(audio_frame)
def deserialize(self, data: str | bytes) -> Frame | None:
audio_frame: AudioFrame = pickle.loads(data)['frame']
return AudioRawFrame(
audio_frame: AudioFrame = pickle.loads(data)["frame"]
return InputAudioRawFrame(
audio=bytes(audio_frame.data),
sample_rate=audio_frame.sample_rate,
num_channels=audio_frame.num_channels,

20
src/pipecat/serializers/protobuf.py
View File

@@ -18,7 +18,7 @@ class ProtobufFrameSerializer(FrameSerializer):
SERIALIZABLE_TYPES = {
TextFrame: "text",
AudioRawFrame: "audio",
TranscriptionFrame: "transcription"
TranscriptionFrame: "transcription",
}
SERIALIZABLE_FIELDS = {v: k for k, v in SERIALIZABLE_TYPES.items()}
@@ -29,14 +29,15 @@ class ProtobufFrameSerializer(FrameSerializer):
def serialize(self, frame: Frame) -> str | bytes | None:
proto_frame = frame_protos.Frame()
if type(frame) not in self.SERIALIZABLE_TYPES:
raise ValueError(
f"Frame type {type(frame)} is not serializable. You may need to add it to ProtobufFrameSerializer.SERIALIZABLE_FIELDS.")
logger.warning(f"Frame type {type(frame)} is not serializable")
return None
# ignoring linter errors; we check that type(frame) is in this dict above
proto_optional_name = self.SERIALIZABLE_TYPES[type(frame)] # type: ignore
for field in dataclasses.fields(frame): # type: ignore
setattr(getattr(proto_frame, proto_optional_name), field.name,
getattr(frame, field.name))
value = getattr(frame, field.name)
if value:
setattr(getattr(proto_frame, proto_optional_name), field.name, value)
result = proto_frame.SerializeToString()
return result
@@ -48,8 +49,8 @@ class ProtobufFrameSerializer(FrameSerializer):
>>> serializer = ProtobufFrameSerializer()
>>> serializer.deserialize(
... serializer.serialize(AudioFrame(data=b'1234567890')))
AudioFrame(data=b'1234567890')
... serializer.serialize(OutputAudioFrame(data=b'1234567890')))
InputAudioFrame(data=b'1234567890')
>>> serializer.deserialize(
... serializer.serialize(TextFrame(text='hello world')))
@@ -75,10 +76,13 @@ class ProtobufFrameSerializer(FrameSerializer):
# Remove special fields if needed
id = getattr(args, "id")
name = getattr(args, "name")
pts = getattr(args, "pts")
if not id:
del args_dict["id"]
if not name:
del args_dict["name"]
if not pts:
del args_dict["pts"]
# Create the instance
instance = class_name(**args_dict)
@@ -88,5 +92,7 @@ class ProtobufFrameSerializer(FrameSerializer):
setattr(instance, "id", getattr(args, "id"))
if name:
setattr(instance, "name", getattr(args, "name"))
if pts:
setattr(instance, "pts", getattr(args, "pts"))
return instance

21
src/pipecat/serializers/twilio.py
View File

@@ -19,10 +19,6 @@ class TwilioFrameSerializer(FrameSerializer):
twilio_sample_rate: int = 8000
sample_rate: int = 16000
SERIALIZABLE_TYPES = {
AudioRawFrame: "audio",
}
def __init__(self, stream_sid: str, params: InputParams = InputParams()):
self._stream_sid = stream_sid
self._params = params
@@ -31,15 +27,12 @@ class TwilioFrameSerializer(FrameSerializer):
if isinstance(frame, AudioRawFrame):
data = frame.audio
serialized_data = pcm_to_ulaw(
data, frame.sample_rate, self._params.twilio_sample_rate)
serialized_data = pcm_to_ulaw(data, frame.sample_rate, self._params.twilio_sample_rate)
payload = base64.b64encode(serialized_data).decode("utf-8")
answer = {
"event": "media",
"streamSid": self._stream_sid,
"media": {
"payload": payload
}
"media": {"payload": payload},
}
return json.dumps(answer)
@@ -58,11 +51,9 @@ class TwilioFrameSerializer(FrameSerializer):
payload = base64.b64decode(payload_base64)
deserialized_data = ulaw_to_pcm(
payload,
self._params.twilio_sample_rate,
self._params.sample_rate)
payload, self._params.twilio_sample_rate, self._params.sample_rate
)
audio_frame = AudioRawFrame(
audio=deserialized_data,
num_channels=1,
sample_rate=self._params.sample_rate)
audio=deserialized_data, num_channels=1, sample_rate=self._params.sample_rate
)
return audio_frame

391
src/pipecat/services/ai_services.py
View File

@@ -7,9 +7,10 @@
import asyncio
import io
import wave
from abc import abstractmethod
from typing import AsyncGenerator, Optional
from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple
from loguru import logger
from pipecat.frames.frames import (
AudioRawFrame,
@@ -18,32 +19,41 @@ from pipecat.frames.frames import (
ErrorFrame,
Frame,
LLMFullResponseEndFrame,
STTLanguageUpdateFrame,
STTModelUpdateFrame,
StartFrame,
StartInterruptionFrame,
TTSLanguageUpdateFrame,
TTSModelUpdateFrame,
STTUpdateSettingsFrame,
TextFrame,
TTSAudioRawFrame,
TTSSpeakFrame,
TTSStartedFrame,
TTSStoppedFrame,
TTSVoiceUpdateFrame,
TextFrame,
TTSUpdateSettingsFrame,
UserImageRequestFrame,
VisionImageRawFrame
VisionImageRawFrame,
)
from pipecat.processors.async_frame_processor import AsyncFrameProcessor
from pipecat.metrics.metrics import MetricsData
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.transcriptions.language import Language
from pipecat.utils.audio import calculate_audio_volume
from pipecat.utils.string import match_endofsentence
from pipecat.utils.time import seconds_to_nanoseconds
from pipecat.utils.utils import exp_smoothing
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
class AIService(FrameProcessor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._model_name: str = ""
self._settings: Dict[str, Any] = {}
@property
def model_name(self) -> str:
return self._model_name
def set_model_name(self, model: str):
self._model_name = model
self.set_core_metrics_data(MetricsData(processor=self.name, model=self._model_name))
async def start(self, frame: StartFrame):
pass
@@ -54,6 +64,16 @@ class AIService(FrameProcessor):
async def cancel(self, frame: CancelFrame):
pass
async def _update_settings(self, settings: Dict[str, Any]):
for key, value in settings.items():
if key in self._settings:
logger.debug(f"Updating setting {key} to: [{value}] for {self.name}")
self._settings[key] = value
elif key == "model":
self.set_model_name(value)
else:
logger.warning(f"Unknown setting for {self.name} service: {key}")
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -64,7 +84,7 @@ class AIService(FrameProcessor):
elif isinstance(frame, EndFrame):
await self.stop(frame)
async def process_generator(self, generator: AsyncGenerator[Frame, None]):
async def process_generator(self, generator: AsyncGenerator[Frame | None, None]):
async for f in generator:
if f:
if isinstance(f, ErrorFrame):
@@ -73,30 +93,6 @@ class AIService(FrameProcessor):
await self.push_frame(f)
class AsyncAIService(AsyncFrameProcessor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
async def start(self, frame: StartFrame):
pass
async def stop(self, frame: EndFrame):
pass
async def cancel(self, frame: CancelFrame):
pass
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame):
await self.start(frame)
elif isinstance(frame, CancelFrame):
await self.cancel(frame)
elif isinstance(frame, EndFrame):
await self.stop(frame)
class LLMService(AIService):
"""This class is a no-op but serves as a base class for LLM services."""
@@ -125,12 +121,14 @@ class LLMService(AIService):
return function_name in self._callbacks.keys()
async def call_function(
self,
*,
context: OpenAILLMContext,
tool_call_id: str,
function_name: str,
arguments: str) -> None:
self,
*,
context: OpenAILLMContext,
tool_call_id: str,
function_name: str,
arguments: str,
run_llm: bool = True,
) -> None:
f = None
if function_name in self._callbacks.keys():
f = self._callbacks[function_name]
@@ -143,7 +141,9 @@ class LLMService(AIService):
function_name=function_name,
tool_call_id=tool_call_id,
arguments=arguments,
llm=self)
llm=self,
run_llm=run_llm,
)
# QUESTION FOR CB: maybe this isn't needed anymore?
async def call_start_function(self, context: OpenAILLMContext, function_name: str):
@@ -153,41 +153,55 @@ class LLMService(AIService):
return await self._start_callbacks[None](function_name, self, context)
async def request_image_frame(self, user_id: str, *, text_content: str | None = None):
await self.push_frame(UserImageRequestFrame(user_id=user_id, context=text_content),
FrameDirection.UPSTREAM)
await self.push_frame(
UserImageRequestFrame(user_id=user_id, context=text_content), FrameDirection.UPSTREAM
)
class TTSService(AIService):
def __init__(
self,
*,
aggregate_sentences: bool = True,
# if True, subclass is responsible for pushing TextFrames and LLMFullResponseEndFrames
push_text_frames: bool = True,
# if True, TTSService will push TTSStoppedFrames, otherwise subclass must do it
push_stop_frames: bool = False,
# if push_stop_frames is True, wait for this idle period before pushing TTSStoppedFrame
stop_frame_timeout_s: float = 0.8,
**kwargs):
self,
*,
aggregate_sentences: bool = True,
# if True, TTSService will push TextFrames and LLMFullResponseEndFrames,
# otherwise subclass must do it
push_text_frames: bool = True,
# if True, TTSService will push TTSStoppedFrames, otherwise subclass must do it
push_stop_frames: bool = False,
# if push_stop_frames is True, wait for this idle period before pushing TTSStoppedFrame
stop_frame_timeout_s: float = 1.0,
# TTS output sample rate
sample_rate: int = 16000,
**kwargs,
):
super().__init__(**kwargs)
self._aggregate_sentences: bool = aggregate_sentences
self._push_text_frames: bool = push_text_frames
self._push_stop_frames: bool = push_stop_frames
self._stop_frame_timeout_s: float = stop_frame_timeout_s
self._sample_rate: int = sample_rate
self._voice_id: str = ""
self._settings: Dict[str, Any] = {}
self._stop_frame_task: Optional[asyncio.Task] = None
self._stop_frame_queue: asyncio.Queue = asyncio.Queue()
self._current_sentence: str = ""
@property
def sample_rate(self) -> int:
return self._sample_rate
@abstractmethod
async def set_model(self, model: str):
pass
self.set_model_name(model)
@abstractmethod
async def set_voice(self, voice: str):
pass
def set_voice(self, voice: str):
self._voice_id = voice
@abstractmethod
async def set_language(self, language: Language):
async def flush_audio(self):
pass
# Converts the text to audio.
@@ -195,66 +209,6 @@ class TTSService(AIService):
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
pass
async def say(self, text: str):
await self.process_frame(TextFrame(text=text), FrameDirection.DOWNSTREAM)
async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
self._current_sentence = ""
await self.push_frame(frame, direction)
async def _process_text_frame(self, frame: TextFrame):
text: str | None = None
if not self._aggregate_sentences:
text = frame.text
else:
self._current_sentence += frame.text
if match_endofsentence(self._current_sentence):
text = self._current_sentence
self._current_sentence = ""
if text:
await self._push_tts_frames(text)
async def _push_tts_frames(self, text: str, text_passthrough: bool = True):
text = text.strip()
if not text:
return
await self.start_processing_metrics()
await self.process_generator(self.run_tts(text))
await self.stop_processing_metrics()
if self._push_text_frames:
# We send the original text after the audio. This way, if we are
# interrupted, the text is not added to the assistant context.
await self.push_frame(TextFrame(text))
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame):
await self._process_text_frame(frame)
elif isinstance(frame, StartInterruptionFrame):
await self._handle_interruption(frame, direction)
elif isinstance(frame, LLMFullResponseEndFrame) or isinstance(frame, EndFrame):
sentence = self._current_sentence
self._current_sentence = ""
await self._push_tts_frames(sentence)
if isinstance(frame, LLMFullResponseEndFrame):
if self._push_text_frames:
await self.push_frame(frame, direction)
else:
await self.push_frame(frame, direction)
elif isinstance(frame, TTSSpeakFrame):
await self._push_tts_frames(frame.text, False)
elif isinstance(frame, TTSModelUpdateFrame):
await self.set_model(frame.model)
elif isinstance(frame, TTSVoiceUpdateFrame):
await self.set_voice(frame.voice)
elif isinstance(frame, TTSLanguageUpdateFrame):
await self.set_language(frame.language)
else:
await self.push_frame(frame, direction)
async def start(self, frame: StartFrame):
await super().start(frame)
if self._push_stop_frames:
@@ -274,23 +228,102 @@ class TTSService(AIService):
await self._stop_frame_task
self._stop_frame_task = None
async def _update_settings(self, settings: Dict[str, Any]):
for key, value in settings.items():
if key in self._settings:
logger.debug(f"Updating TTS setting {key} to: [{value}]")
self._settings[key] = value
if key == "language":
self._settings[key] = Language(value)
elif key == "model":
self.set_model_name(value)
elif key == "voice":
self.set_voice(value)
else:
logger.warning(f"Unknown setting for TTS service: {key}")
async def say(self, text: str):
aggregate_sentences = self._aggregate_sentences
self._aggregate_sentences = False
await self.process_frame(TextFrame(text=text), FrameDirection.DOWNSTREAM)
self._aggregate_sentences = aggregate_sentences
await self.flush_audio()
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame):
await self._process_text_frame(frame)
elif isinstance(frame, StartInterruptionFrame):
await self._handle_interruption(frame, direction)
elif isinstance(frame, LLMFullResponseEndFrame) or isinstance(frame, EndFrame):
sentence = self._current_sentence
self._current_sentence = ""
await self._push_tts_frames(sentence)
if isinstance(frame, LLMFullResponseEndFrame):
if self._push_text_frames:
await self.push_frame(frame, direction)
else:
await self.push_frame(frame, direction)
elif isinstance(frame, TTSSpeakFrame):
await self._push_tts_frames(frame.text)
await self.flush_audio()
elif isinstance(frame, TTSUpdateSettingsFrame):
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
await super().push_frame(frame, direction)
if self._push_stop_frames and (
isinstance(frame, StartInterruptionFrame) or
isinstance(frame, TTSStartedFrame) or
isinstance(frame, AudioRawFrame) or
isinstance(frame, TTSStoppedFrame)):
isinstance(frame, StartInterruptionFrame)
or isinstance(frame, TTSStartedFrame)
or isinstance(frame, TTSAudioRawFrame)
or isinstance(frame, TTSStoppedFrame)
):
await self._stop_frame_queue.put(frame)
async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
self._current_sentence = ""
await self.push_frame(frame, direction)
async def _process_text_frame(self, frame: TextFrame):
text: str | None = None
if not self._aggregate_sentences:
text = frame.text
else:
self._current_sentence += frame.text
eos_end_marker = match_endofsentence(self._current_sentence)
if eos_end_marker:
text = self._current_sentence[:eos_end_marker]
self._current_sentence = self._current_sentence[eos_end_marker:]
if text:
await self._push_tts_frames(text)
async def _push_tts_frames(self, text: str):
# Don't send only whitespace. This causes problems for some TTS models. But also don't
# strip all whitespace, as whitespace can influence prosody.
if not text.strip():
return
await self.start_processing_metrics()
await self.process_generator(self.run_tts(text))
await self.stop_processing_metrics()
if self._push_text_frames:
# We send the original text after the audio. This way, if we are
# interrupted, the text is not added to the assistant context.
await self.push_frame(TextFrame(text))
async def _stop_frame_handler(self):
try:
has_started = False
while True:
try:
frame = await asyncio.wait_for(self._stop_frame_queue.get(),
self._stop_frame_timeout_s)
frame = await asyncio.wait_for(
self._stop_frame_queue.get(), self._stop_frame_timeout_s
)
if isinstance(frame, TTSStartedFrame):
has_started = True
elif isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
@@ -303,25 +336,95 @@ class TTSService(AIService):
pass
class WordTTSService(TTSService):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._initial_word_timestamp = -1
self._words_queue = asyncio.Queue()
self._words_task = self.get_event_loop().create_task(self._words_task_handler())
def start_word_timestamps(self):
if self._initial_word_timestamp == -1:
self._initial_word_timestamp = self.get_clock().get_time()
def reset_word_timestamps(self):
self._initial_word_timestamp = -1
self._word_timestamps = []
async def add_word_timestamps(self, word_times: List[Tuple[str, float]]):
for word, timestamp in word_times:
await self._words_queue.put((word, seconds_to_nanoseconds(timestamp)))
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._stop_words_task()
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._stop_words_task()
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, LLMFullResponseEndFrame) or isinstance(frame, EndFrame):
await self.flush_audio()
async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
await super()._handle_interruption(frame, direction)
self.reset_word_timestamps()
async def _stop_words_task(self):
if self._words_task:
self._words_task.cancel()
await self._words_task
self._words_task = None
async def _words_task_handler(self):
while True:
try:
(word, timestamp) = await self._words_queue.get()
if word == "LLMFullResponseEndFrame" and timestamp == 0:
await self.push_frame(LLMFullResponseEndFrame())
else:
frame = TextFrame(word)
frame.pts = self._initial_word_timestamp + timestamp
await self.push_frame(frame)
self._words_queue.task_done()
except asyncio.CancelledError:
break
except Exception as e:
logger.exception(f"{self} exception: {e}")
class STTService(AIService):
"""STTService is a base class for speech-to-text services."""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._settings: Dict[str, Any] = {}
@abstractmethod
async def set_model(self, model: str):
pass
@abstractmethod
async def set_language(self, language: Language):
pass
self.set_model_name(model)
@abstractmethod
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Returns transcript as a string"""
pass
async def _update_settings(self, settings: Dict[str, Any]):
logger.debug(f"Updating STT settings: {self._settings}")
for key, value in settings.items():
if key in self._settings:
logger.debug(f"Updating STT setting {key} to: [{value}]")
self._settings[key] = value
if key == "language":
self._settings[key] = Language(value)
elif key == "model":
self.set_model_name(value)
else:
logger.warning(f"Unknown setting for STT service: {key}")
async def process_audio_frame(self, frame: AudioRawFrame):
await self.process_generator(self.run_stt(frame.audio))
@@ -333,10 +436,8 @@ class STTService(AIService):
# In this service we accumulate audio internally and at the end we
# push a TextFrame. We don't really want to push audio frames down.
await self.process_audio_frame(frame)
elif isinstance(frame, STTModelUpdateFrame):
await self.set_model(frame.model)
elif isinstance(frame, STTLanguageUpdateFrame):
await self.set_language(frame.language)
elif isinstance(frame, STTUpdateSettingsFrame):
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)
@@ -347,14 +448,16 @@ class SegmentedSTTService(STTService):
"""
def __init__(self,
*,
min_volume: float = 0.6,
max_silence_secs: float = 0.3,
max_buffer_secs: float = 1.5,
sample_rate: int = 16000,
num_channels: int = 1,
**kwargs):
def __init__(
self,
*,
min_volume: float = 0.6,
max_silence_secs: float = 0.3,
max_buffer_secs: float = 1.5,
sample_rate: int = 16000,
num_channels: int = 1,
**kwargs,
):
super().__init__(**kwargs)
self._min_volume = min_volume
self._max_silence_secs = max_silence_secs
@@ -383,7 +486,8 @@ class SegmentedSTTService(STTService):
silence_secs = self._silence_num_frames / self._sample_rate
buffer_secs = self._wave.getnframes() / self._sample_rate
if self._content.tell() > 0 and (
buffer_secs > self._max_buffer_secs or silence_secs > self._max_silence_secs):
buffer_secs > self._max_buffer_secs or silence_secs > self._max_silence_secs
):
self._silence_num_frames = 0
self._wave.close()
self._content.seek(0)
@@ -410,7 +514,6 @@ class SegmentedSTTService(STTService):
class ImageGenService(AIService):
def __init__(self, **kwargs):
super().__init__(**kwargs)

384
src/pipecat/services/anthropic.py
View File

@@ -5,53 +5,57 @@
#
import base64
import json
import io
import copy
from typing import List, Optional
from dataclasses import dataclass
from PIL import Image
from asyncio import CancelledError
import io
import json
import re
from asyncio import CancelledError
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
from loguru import logger
from PIL import Image
from pydantic import BaseModel, Field
from pipecat.frames.frames import (
Frame,
LLMEnablePromptCachingFrame,
LLMModelUpdateFrame,
TextFrame,
VisionImageRawFrame,
UserImageRequestFrame,
UserImageRawFrame,
LLMMessagesFrame,
LLMFullResponseStartFrame,
LLMFullResponseEndFrame,
FunctionCallResultFrame,
FunctionCallInProgressFrame,
StartInterruptionFrame
FunctionCallResultFrame,
LLMEnablePromptCachingFrame,
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMUpdateSettingsFrame,
StartInterruptionFrame,
TextFrame,
UserImageRawFrame,
UserImageRequestFrame,
VisionImageRawFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_response import (
LLMAssistantContextAggregator,
LLMUserContextAggregator,
)
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import LLMService
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
OpenAILLMContextFrame
)
from pipecat.processors.aggregators.llm_response import (
LLMUserContextAggregator,
LLMAssistantContextAggregator
)
from loguru import logger
try:
from anthropic import AsyncAnthropic, NOT_GIVEN, NotGiven
from anthropic import NOT_GIVEN, AsyncAnthropic, NotGiven
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Anthropic, you need to `pip install pipecat-ai[anthropic]`. " +
"Also, set `ANTHROPIC_API_KEY` environment variable.")
"In order to use Anthropic, you need to `pip install pipecat-ai[anthropic]`. "
+ "Also, set `ANTHROPIC_API_KEY` environment variable."
)
raise Exception(f"Missing module: {e}")
# internal use only -- todo: refactor
@dataclass
class AnthropicImageMessageFrame(Frame):
user_image_raw_frame: UserImageRawFrame
@@ -60,33 +64,46 @@ class AnthropicImageMessageFrame(Frame):
@dataclass
class AnthropicContextAggregatorPair:
_user: 'AnthropicUserContextAggregator'
_assistant: 'AnthropicAssistantContextAggregator'
_user: "AnthropicUserContextAggregator"
_assistant: "AnthropicAssistantContextAggregator"
def user(self) -> 'AnthropicUserContextAggregator':
def user(self) -> "AnthropicUserContextAggregator":
return self._user
def assistant(self) -> 'AnthropicAssistantContextAggregator':
def assistant(self) -> "AnthropicAssistantContextAggregator":
return self._assistant
class AnthropicLLMService(LLMService):
"""This class implements inference with Anthropic's AI models
"""
"""This class implements inference with Anthropic's AI models"""
class InputParams(BaseModel):
enable_prompt_caching_beta: Optional[bool] = False
max_tokens: Optional[int] = Field(default_factory=lambda: 4096, ge=1)
temperature: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0)
top_k: Optional[int] = Field(default_factory=lambda: NOT_GIVEN, ge=0)
top_p: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0)
extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
def __init__(
self,
*,
api_key: str,
model: str = "claude-3-5-sonnet-20240620",
max_tokens: int = 4096,
enable_prompt_caching_beta: bool = False,
**kwargs):
self,
*,
api_key: str,
model: str = "claude-3-5-sonnet-20240620",
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(**kwargs)
self._client = AsyncAnthropic(api_key=api_key)
self._model = model
self._max_tokens = max_tokens
self._enable_prompt_caching_beta = enable_prompt_caching_beta
self.set_model_name(model)
self._settings = {
"max_tokens": params.max_tokens,
"enable_prompt_caching_beta": params.enable_prompt_caching_beta or False,
"temperature": params.temperature,
"top_k": params.top_k,
"top_p": params.top_p,
"extra": params.extra if isinstance(params.extra, dict) else {},
}
def can_generate_metrics(self) -> bool:
return True
@@ -96,13 +113,14 @@ class AnthropicLLMService(LLMService):
return self._enable_prompt_caching_beta
@staticmethod
def create_context_aggregator(context: OpenAILLMContext) -> AnthropicContextAggregatorPair:
def create_context_aggregator(
context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True
) -> AnthropicContextAggregatorPair:
user = AnthropicUserContextAggregator(context)
assistant = AnthropicAssistantContextAggregator(user)
return AnthropicContextAggregatorPair(
_user=user,
_assistant=assistant
assistant = AnthropicAssistantContextAggregator(
user, expect_stripped_words=assistant_expect_stripped_words
)
return AnthropicContextAggregatorPair(_user=user, _assistant=assistant)
async def _process_context(self, context: OpenAILLMContext):
# Usage tracking. We track the usage reported by Anthropic in prompt_tokens and
@@ -121,78 +139,103 @@ class AnthropicLLMService(LLMService):
await self.start_processing_metrics()
logger.debug(
f"Generating chat: {context.system} | {context.get_messages_for_logging()}")
f"Generating chat: {context.system} | {context.get_messages_for_logging()}"
)
messages = context.messages
if self._enable_prompt_caching_beta:
if self._settings["enable_prompt_caching_beta"]:
messages = context.get_messages_with_cache_control_markers()
api_call = self._client.messages.create
if self._enable_prompt_caching_beta:
if self._settings["enable_prompt_caching_beta"]:
api_call = self._client.beta.prompt_caching.messages.create
await self.start_ttfb_metrics()
response = await api_call(
tools=context.tools or [],
system=context.system,
messages=messages,
model=self._model,
max_tokens=self._max_tokens,
stream=True)
params = {
"tools": context.tools or [],
"system": context.system,
"messages": messages,
"model": self.model_name,
"max_tokens": self._settings["max_tokens"],
"stream": True,
"temperature": self._settings["temperature"],
"top_k": self._settings["top_k"],
"top_p": self._settings["top_p"],
}
params.update(self._settings["extra"])
response = await api_call(**params)
await self.stop_ttfb_metrics()
# Function calling
tool_use_block = None
json_accumulator = ''
json_accumulator = ""
async for event in response:
# logger.debug(f"Anthropic LLM event: {event}")
# Aggregate streaming content, create frames, trigger events
if (event.type == "content_block_delta"):
if hasattr(event.delta, 'text'):
if event.type == "content_block_delta":
if hasattr(event.delta, "text"):
await self.push_frame(TextFrame(event.delta.text))
completion_tokens_estimate += self._estimate_tokens(event.delta.text)
elif hasattr(event.delta, 'partial_json') and tool_use_block:
elif hasattr(event.delta, "partial_json") and tool_use_block:
json_accumulator += event.delta.partial_json
completion_tokens_estimate += self._estimate_tokens(
event.delta.partial_json)
elif (event.type == "content_block_start"):
event.delta.partial_json
)
elif event.type == "content_block_start":
if event.content_block.type == "tool_use":
tool_use_block = event.content_block
json_accumulator = ''
elif ((event.type == "message_delta" and
hasattr(event.delta, 'stop_reason')
and event.delta.stop_reason == 'tool_use')):
json_accumulator = ""
elif (
event.type == "message_delta"
and hasattr(event.delta, "stop_reason")
and event.delta.stop_reason == "tool_use"
):
if tool_use_block:
await self.call_function(context=context,
tool_call_id=tool_use_block.id,
function_name=tool_use_block.name,
arguments=json.loads(json_accumulator) if json_accumulator else dict()
)
await self.call_function(
context=context,
tool_call_id=tool_use_block.id,
function_name=tool_use_block.name,
arguments=json.loads(json_accumulator) if json_accumulator else dict(),
)
# Calculate usage. Do this here in its own if statement, because there may be usage
# data embedded in messages that we do other processing for, above.
if hasattr(event, "usage"):
prompt_tokens += event.usage.input_tokens if hasattr(
event.usage, "input_tokens") else 0
completion_tokens += event.usage.output_tokens if hasattr(
event.usage, "output_tokens") else 0
prompt_tokens += (
event.usage.input_tokens if hasattr(event.usage, "input_tokens") else 0
)
completion_tokens += (
event.usage.output_tokens if hasattr(event.usage, "output_tokens") else 0
)
elif hasattr(event, "message") and hasattr(event.message, "usage"):
prompt_tokens += event.message.usage.input_tokens if hasattr(
event.message.usage, "input_tokens") else 0
completion_tokens += event.message.usage.output_tokens if hasattr(
event.message.usage, "output_tokens") else 0
prompt_tokens += (
event.message.usage.input_tokens
if hasattr(event.message.usage, "input_tokens")
else 0
)
completion_tokens += (
event.message.usage.output_tokens
if hasattr(event.message.usage, "output_tokens")
else 0
)
if hasattr(event.message.usage, "cache_creation_input_tokens"):
cache_creation_input_tokens += event.message.usage.cache_creation_input_tokens
cache_creation_input_tokens += (
event.message.usage.cache_creation_input_tokens
)
logger.debug(f"Cache creation input tokens: {cache_creation_input_tokens}")
if hasattr(event.message.usage, "cache_read_input_tokens"):
cache_read_input_tokens += event.message.usage.cache_read_input_tokens
logger.debug(f"Cache read input tokens: {cache_read_input_tokens}")
total_input_tokens = prompt_tokens + cache_creation_input_tokens + cache_read_input_tokens
total_input_tokens = (
prompt_tokens + cache_creation_input_tokens + cache_read_input_tokens
)
if total_input_tokens >= 1024:
context.turns_above_cache_threshold += 1
@@ -207,12 +250,16 @@ class AnthropicLLMService(LLMService):
finally:
await self.stop_processing_metrics()
await self.push_frame(LLMFullResponseEndFrame())
comp_tokens = completion_tokens if not use_completion_tokens_estimate else completion_tokens_estimate
comp_tokens = (
completion_tokens
if not use_completion_tokens_estimate
else completion_tokens_estimate
)
await self._report_usage_metrics(
prompt_tokens=prompt_tokens,
completion_tokens=comp_tokens,
cache_creation_input_tokens=cache_creation_input_tokens,
cache_read_input_tokens=cache_read_input_tokens
cache_read_input_tokens=cache_read_input_tokens,
)
async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -229,12 +276,11 @@ class AnthropicLLMService(LLMService):
# UserImageRawFrames coming through the pipeline and add them
# to the context.
context = AnthropicLLMContext.from_image_frame(frame)
elif isinstance(frame, LLMModelUpdateFrame):
logger.debug(f"Switching LLM model to: [{frame.model}]")
self._model = frame.model
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
elif isinstance(frame, LLMEnablePromptCachingFrame):
logger.debug(f"Setting enable prompt caching to: [{frame.enable}]")
self._enable_prompt_caching_beta = frame.enable
self._settings["enable_prompt_caching_beta"] = frame.enable
else:
await self.push_frame(frame, direction)
@@ -242,24 +288,28 @@ class AnthropicLLMService(LLMService):
await self._process_context(context)
def _estimate_tokens(self, text: str) -> int:
return int(len(re.split(r'[^\w]+', text)) * 1.3)
return int(len(re.split(r"[^\w]+", text)) * 1.3)
async def _report_usage_metrics(
self,
prompt_tokens: int,
completion_tokens: int,
cache_creation_input_tokens: int,
cache_read_input_tokens: int):
if prompt_tokens or completion_tokens or cache_creation_input_tokens or cache_read_input_tokens:
tokens = {
"processor": self.name,
"model": self._model,
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"cache_creation_input_tokens": cache_creation_input_tokens,
"cache_read_input_tokens": cache_read_input_tokens,
"total_tokens": prompt_tokens + completion_tokens
}
self,
prompt_tokens: int,
completion_tokens: int,
cache_creation_input_tokens: int,
cache_read_input_tokens: int,
):
if (
prompt_tokens
or completion_tokens
or cache_creation_input_tokens
or cache_read_input_tokens
):
tokens = LLMTokenUsage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
cache_creation_input_tokens=cache_creation_input_tokens,
cache_read_input_tokens=cache_read_input_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
await self.start_llm_usage_metrics(tokens)
@@ -270,10 +320,9 @@ class AnthropicLLMContext(OpenAILLMContext):
tools: list[dict] | None = None,
tool_choice: dict | None = None,
*,
system: str | NotGiven = NOT_GIVEN
system: str | NotGiven = NOT_GIVEN,
):
super().__init__(messages=messages, tools=tools, tool_choice=tool_choice)
self._user_image_request_context = {}
# For beta prompt caching. This is a counter that tracks the number of turns
# we've seen above the cache threshold. We reset this when we reset the
@@ -303,10 +352,8 @@ class AnthropicLLMContext(OpenAILLMContext):
def from_image_frame(cls, frame: VisionImageRawFrame) -> "AnthropicLLMContext":
context = cls()
context.add_image_frame_message(
format=frame.format,
size=frame.size,
image=frame.image,
text=frame.text)
format=frame.format, size=frame.size, image=frame.image, text=frame.text
)
return context
def set_messages(self, messages: List):
@@ -315,18 +362,23 @@ class AnthropicLLMContext(OpenAILLMContext):
self._restructure_from_openai_messages()
def add_image_frame_message(
self, *, format: str, size: tuple[int, int], image: bytes, text: str = None):
self, *, format: str, size: tuple[int, int], image: bytes, text: str = None
):
buffer = io.BytesIO()
Image.frombytes(format, size, image).save(buffer, format="JPEG")
encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
# Anthropic docs say that the image should be the first content block in the message.
content = [{"type": "image",
"source": {
"type": "base64",
"media_type": "image/jpeg",
"data": encoded_image,
}}]
content = [
{
"type": "image",
"source": {
"type": "base64",
"media_type": "image/jpeg",
"data": encoded_image,
},
}
]
if text:
content.append({"type": "text", "text": text})
self.add_message({"role": "user", "content": content})
@@ -340,8 +392,9 @@ class AnthropicLLMContext(OpenAILLMContext):
# if the last message has just a content string, convert it to a list
# in the proper format
if isinstance(self.messages[-1]["content"], str):
self.messages[-1]["content"] = [{"type": "text",
"text": self.messages[-1]["content"]}]
self.messages[-1]["content"] = [
{"type": "text", "text": self.messages[-1]["content"]}
]
# if this message has just a content string, convert it to a list
# in the proper format
if isinstance(message["content"], str):
@@ -362,8 +415,11 @@ class AnthropicLLMContext(OpenAILLMContext):
if isinstance(messages[-1]["content"], str):
messages[-1]["content"] = [{"type": "text", "text": messages[-1]["content"]}]
messages[-1]["content"][-1]["cache_control"] = {"type": "ephemeral"}
if (self.turns_above_cache_threshold >= 2 and
len(messages) > 2 and messages[-3]["role"] == "user"):
if (
self.turns_above_cache_threshold >= 2
and len(messages) > 2
and messages[-3]["role"] == "user"
):
if isinstance(messages[-3]["content"], str):
messages[-3]["content"] = [{"type": "text", "text": messages[-3]["content"]}]
messages[-3]["content"][-1]["cache_control"] = {"type": "ephemeral"}
@@ -417,12 +473,13 @@ class AnthropicUserContextAggregator(LLMUserContextAggregator):
# The LLM sends a UserImageRequestFrame upstream. Cache any context provided with
# that frame so we can use it when we assemble the image message in the assistant
# context aggregator.
if (frame.context):
if frame.context:
if isinstance(frame.context, str):
self._context._user_image_request_context[frame.user_id] = frame.context
else:
logger.error(
f"Unexpected UserImageRequestFrame context type: {type(frame.context)}")
f"Unexpected UserImageRequestFrame context type: {type(frame.context)}"
)
del self._context._user_image_request_context[frame.user_id]
else:
if frame.user_id in self._context._user_image_request_context:
@@ -439,6 +496,7 @@ class AnthropicUserContextAggregator(LLMUserContextAggregator):
except Exception as e:
logger.error(f"Error processing frame: {e}")
#
# Claude returns a text content block along with a tool use content block. This works quite nicely
# with streaming. We get the text first, so we can start streaming it right away. Then we get the
@@ -450,8 +508,8 @@ class AnthropicUserContextAggregator(LLMUserContextAggregator):
class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
def __init__(self, user_context_aggregator: AnthropicUserContextAggregator):
super().__init__(context=user_context_aggregator._context)
def __init__(self, user_context_aggregator: AnthropicUserContextAggregator, **kwargs):
super().__init__(context=user_context_aggregator._context, **kwargs)
self._user_context_aggregator = user_context_aggregator
self._function_call_in_progress = None
self._function_call_result = None
@@ -466,13 +524,16 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
elif isinstance(frame, FunctionCallInProgressFrame):
self._function_call_in_progress = frame
elif isinstance(frame, FunctionCallResultFrame):
if (self._function_call_in_progress and self._function_call_in_progress.tool_call_id ==
frame.tool_call_id):
if (
self._function_call_in_progress
and self._function_call_in_progress.tool_call_id == frame.tool_call_id
):
self._function_call_in_progress = None
self._function_call_result = frame
else:
logger.warning(
"FunctionCallResultFrame tool_call_id != InProgressFrame tool_call_id")
"FunctionCallResultFrame tool_call_id != InProgressFrame tool_call_id"
)
self._function_call_in_progress = None
self._function_call_result = None
elif isinstance(frame, AnthropicImageMessageFrame):
@@ -485,38 +546,39 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
run_llm = False
aggregation = self._aggregation
self._aggregation = ""
self._reset()
try:
if self._function_call_result:
frame = self._function_call_result
self._function_call_result = None
if frame.result:
self._context.add_message({
"role": "assistant",
"content": [
{
"type": "text",
"text": aggregation
},
{
"type": "tool_use",
"id": frame.tool_call_id,
"name": frame.function_name,
"input": frame.arguments
}
]
})
self._context.add_message({
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": frame.tool_call_id,
"content": json.dumps(frame.result)
}
]
})
self._context.add_message(
{
"role": "assistant",
"content": [
{"type": "text", "text": aggregation},
{
"type": "tool_use",
"id": frame.tool_call_id,
"name": frame.function_name,
"input": frame.arguments,
},
],
}
)
self._context.add_message(
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": frame.tool_call_id,
"content": json.dumps(frame.result),
}
],
}
)
run_llm = True
else:
self._context.add_message({"role": "assistant", "content": aggregation})
@@ -528,11 +590,15 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
format=frame.user_image_raw_frame.format,
size=frame.user_image_raw_frame.size,
image=frame.user_image_raw_frame.image,
text=frame.text)
text=frame.text,
)
run_llm = True
if run_llm:
await self._user_context_aggregator.push_context_frame()
frame = OpenAILLMContextFrame(self._context)
await self.push_frame(frame)
except Exception as e:
logger.error(f"Error processing frame: {e}")

213
src/pipecat/services/aws.py Normal file
View File

@@ -0,0 +1,213 @@
#
# Copyright (c) 2024, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import AsyncGenerator, Optional
from loguru import logger
from pydantic import BaseModel
from pipecat.frames.frames import (
ErrorFrame,
Frame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.ai_services import TTSService
from pipecat.transcriptions.language import Language
try:
import boto3
from botocore.exceptions import BotoCoreError, ClientError
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Deepgram, you need to `pip install pipecat-ai[aws]`. Also, set `AWS_SECRET_ACCESS_KEY`, `AWS_ACCESS_KEY_ID`, and `AWS_REGION` environment variable."
)
raise Exception(f"Missing module: {e}")
def language_to_aws_language(language: Language) -> str | None:
match language:
case Language.CA:
return "ca-ES"
case Language.ZH:
return "cmn-CN"
case Language.DA:
return "da-DK"
case Language.NL:
return "nl-NL"
case Language.NL_BE:
return "nl-BE"
case Language.EN:
return "en-US"
case Language.EN_US:
return "en-US"
case Language.EN_AU:
return "en-AU"
case Language.EN_GB:
return "en-GB"
case Language.EN_NZ:
return "en-NZ"
case Language.EN_IN:
return "en-IN"
case Language.FI:
return "fi-FI"
case Language.FR:
return "fr-FR"
case Language.FR_CA:
return "fr-CA"
case Language.DE:
return "de-DE"
case Language.HI:
return "hi-IN"
case Language.IT:
return "it-IT"
case Language.JA:
return "ja-JP"
case Language.KO:
return "ko-KR"
case Language.NO:
return "nb-NO"
case Language.PL:
return "pl-PL"
case Language.PT:
return "pt-PT"
case Language.PT_BR:
return "pt-BR"
case Language.RO:
return "ro-RO"
case Language.RU:
return "ru-RU"
case Language.ES:
return "es-ES"
case Language.SV:
return "sv-SE"
case Language.TR:
return "tr-TR"
return None
class AWSTTSService(TTSService):
class InputParams(BaseModel):
engine: Optional[str] = None
language: Optional[Language] = Language.EN
pitch: Optional[str] = None
rate: Optional[str] = None
volume: Optional[str] = None
def __init__(
self,
*,
api_key: str,
aws_access_key_id: str,
region: str,
voice_id: str = "Joanna",
sample_rate: int = 16000,
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
self._polly_client = boto3.client(
"polly",
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=api_key,
region_name=region,
)
self._settings = {
"sample_rate": sample_rate,
"engine": params.engine,
"language": params.language if params.language else Language.EN,
"pitch": params.pitch,
"rate": params.rate,
"volume": params.volume,
}
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
return True
def _construct_ssml(self, text: str) -> str:
ssml = "<speak>"
language = language_to_aws_language(self._settings["language"])
ssml += f"<lang xml:lang='{language}'>"
prosody_attrs = []
# Prosody tags are only supported for standard and neural engines
if self._settings["engine"] != "generative":
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings['rate']}'")
if self._settings["pitch"]:
prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings['volume']}'")
if prosody_attrs:
ssml += f"<prosody {' '.join(prosody_attrs)}>"
else:
logger.warning("Prosody tags are not supported for generative engine. Ignoring.")
ssml += text
if prosody_attrs:
ssml += "</prosody>"
ssml += "</lang>"
ssml += "</speak>"
return ssml
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
try:
await self.start_ttfb_metrics()
# Construct the parameters dictionary
ssml = self._construct_ssml(text)
params = {
"Text": ssml,
"TextType": "ssml",
"OutputFormat": "pcm",
"VoiceId": self._voice_id,
"Engine": self._settings["engine"],
"SampleRate": str(self._settings["sample_rate"]),
}
# Filter out None values
filtered_params = {k: v for k, v in params.items() if v is not None}
response = self._polly_client.synthesize_speech(**filtered_params)
await self.start_tts_usage_metrics(text)
yield TTSStartedFrame()
if "AudioStream" in response:
with response["AudioStream"] as stream:
audio_data = stream.read()
chunk_size = 8192
for i in range(0, len(audio_data), chunk_size):
chunk = audio_data[i : i + chunk_size]
if len(chunk) > 0:
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
yield frame
yield TTSStoppedFrame()
except (BotoCoreError, ClientError) as error:
logger.exception(f"{self} error generating TTS: {error}")
error_message = f"AWS Polly TTS error: {str(error)}"
yield ErrorFrame(error=error_message)
finally:
yield TTSStoppedFrame()

289
src/pipecat/services/azure.py
View File

@@ -4,59 +4,60 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import aiohttp
import asyncio
import io
from typing import AsyncGenerator, Optional
import aiohttp
from loguru import logger
from PIL import Image
from typing import AsyncGenerator
from pydantic import BaseModel
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,
EndFrame,
ErrorFrame,
Frame,
StartFrame,
SystemFrame,
TranscriptionFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
TranscriptionFrame,
URLImageRawFrame)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import AsyncAIService, TTSService, ImageGenService
URLImageRawFrame,
)
from pipecat.services.ai_services import ImageGenService, STTService, TTSService
from pipecat.services.openai import BaseOpenAILLMService
from pipecat.transcriptions import language
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from loguru import logger
# See .env.example for Azure configuration needed
try:
from openai import AsyncAzureOpenAI
from azure.cognitiveservices.speech import (
CancellationReason,
ResultReason,
SpeechConfig,
SpeechRecognizer,
SpeechSynthesizer,
ResultReason,
CancellationReason,
)
from azure.cognitiveservices.speech.audio import AudioStreamFormat, PushAudioInputStream
from azure.cognitiveservices.speech.audio import (
AudioStreamFormat,
PushAudioInputStream,
)
from azure.cognitiveservices.speech.dialog import AudioConfig
from openai import AsyncAzureOpenAI
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Azure, you need to `pip install pipecat-ai[azure]`. Also, set `AZURE_SPEECH_API_KEY` and `AZURE_SPEECH_REGION` environment variables.")
"In order to use Azure, you need to `pip install pipecat-ai[azure]`. Also, set `AZURE_SPEECH_API_KEY` and `AZURE_SPEECH_REGION` environment variables."
)
raise Exception(f"Missing module: {e}")
class AzureLLMService(BaseOpenAILLMService):
def __init__(
self,
*,
api_key: str,
endpoint: str,
model: str,
api_version: str = "2023-12-01-preview"):
self, *, api_key: str, endpoint: str, model: str, api_version: str = "2023-12-01-preview"
):
# Initialize variables before calling parent __init__() because that
# will call create_client() and we need those values there.
self._endpoint = endpoint
@@ -71,46 +72,205 @@ class AzureLLMService(BaseOpenAILLMService):
)
def language_to_azure_language(language: Language) -> str | None:
match language:
case Language.BG:
return "bg-BG"
case Language.CA:
return "ca-ES"
case Language.ZH:
return "zh-CN"
case Language.ZH_TW:
return "zh-TW"
case Language.CS:
return "cs-CZ"
case Language.DA:
return "da-DK"
case Language.NL:
return "nl-NL"
case Language.EN:
return "en-US"
case Language.EN_US:
return "en-US"
case Language.EN_AU:
return "en-AU"
case Language.EN_GB:
return "en-GB"
case Language.EN_NZ:
return "en-NZ"
case Language.EN_IN:
return "en-IN"
case Language.ET:
return "et-EE"
case Language.FI:
return "fi-FI"
case Language.NL_BE:
return "nl-BE"
case Language.FR:
return "fr-FR"
case Language.FR_CA:
return "fr-CA"
case Language.DE:
return "de-DE"
case Language.DE_CH:
return "de-CH"
case Language.EL:
return "el-GR"
case Language.HI:
return "hi-IN"
case Language.HU:
return "hu-HU"
case Language.ID:
return "id-ID"
case Language.IT:
return "it-IT"
case Language.JA:
return "ja-JP"
case Language.KO:
return "ko-KR"
case Language.LV:
return "lv-LV"
case Language.LT:
return "lt-LT"
case Language.MS:
return "ms-MY"
case Language.NO:
return "nb-NO"
case Language.PL:
return "pl-PL"
case Language.PT:
return "pt-PT"
case Language.PT_BR:
return "pt-BR"
case Language.RO:
return "ro-RO"
case Language.RU:
return "ru-RU"
case Language.SK:
return "sk-SK"
case Language.ES:
return "es-ES"
case Language.SV:
return "sv-SE"
case Language.TH:
return "th-TH"
case Language.TR:
return "tr-TR"
case Language.UK:
return "uk-UA"
case Language.VI:
return "vi-VN"
return None
class AzureTTSService(TTSService):
def __init__(self, *, api_key: str, region: str, voice="en-US-SaraNeural", **kwargs):
super().__init__(**kwargs)
class InputParams(BaseModel):
emphasis: Optional[str] = None
language: Optional[Language] = Language.EN
pitch: Optional[str] = None
rate: Optional[str] = "1.05"
role: Optional[str] = None
style: Optional[str] = None
style_degree: Optional[str] = None
volume: Optional[str] = None
def __init__(
self,
*,
api_key: str,
region: str,
voice="en-US-SaraNeural",
sample_rate: int = 16000,
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
speech_config = SpeechConfig(subscription=api_key, region=region)
self._speech_synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=None)
self._voice = voice
self._settings = {
"sample_rate": sample_rate,
"emphasis": params.emphasis,
"language": params.language if params.language else Language.EN,
"pitch": params.pitch,
"rate": params.rate,
"role": params.role,
"style": params.style,
"style_degree": params.style_degree,
"volume": params.volume,
}
self.set_voice(voice)
def can_generate_metrics(self) -> bool:
return True
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice = voice
def _construct_ssml(self, text: str) -> str:
language = language_to_azure_language(self._settings["language"])
ssml = (
f"<speak version='1.0' xml:lang='{language}' "
"xmlns='http://www.w3.org/2001/10/synthesis' "
"xmlns:mstts='http://www.w3.org/2001/mstts'>"
f"<voice name='{self._voice_id}'>"
"<mstts:silence type='Sentenceboundary' value='20ms' />"
)
if self._settings["style"]:
ssml += f"<mstts:express-as style='{self._settings['style']}'"
if self._settings["style_degree"]:
ssml += f" styledegree='{self._settings['style_degree']}'"
if self._settings["role"]:
ssml += f" role='{self._settings['role']}'"
ssml += ">"
prosody_attrs = []
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings['rate']}'")
if self._settings["pitch"]:
prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings['volume']}'")
ssml += f"<prosody {' '.join(prosody_attrs)}>"
if self._settings["emphasis"]:
ssml += f"<emphasis level='{self._settings['emphasis']}'>"
ssml += text
if self._settings["emphasis"]:
ssml += "</emphasis>"
ssml += "</prosody>"
if self._settings["style"]:
ssml += "</mstts:express-as>"
ssml += "</voice></speak>"
return ssml
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
await self.start_ttfb_metrics()
ssml = (
"<speak version='1.0' xml:lang='en-US' xmlns='http://www.w3.org/2001/10/synthesis' "
"xmlns:mstts='http://www.w3.org/2001/mstts'>"
f"<voice name='{self._voice}'>"
"<mstts:silence type='Sentenceboundary' value='20ms' />"
"<mstts:express-as style='lyrical' styledegree='2' role='SeniorFemale'>"
"<prosody rate='1.05'>"
f"{text}"
"</prosody></mstts:express-as></voice></speak> ")
ssml = self._construct_ssml(text)
result = await asyncio.to_thread(self._speech_synthesizer.speak_ssml, (ssml))
if result.reason == ResultReason.SynthesizingAudioCompleted:
await self.start_tts_usage_metrics(text)
await self.stop_ttfb_metrics()
await self.push_frame(TTSStartedFrame())
yield TTSStartedFrame()
# Azure always sends a 44-byte header. Strip it off.
yield AudioRawFrame(audio=result.audio_data[44:], sample_rate=16000, num_channels=1)
await self.push_frame(TTSStoppedFrame())
yield TTSAudioRawFrame(
audio=result.audio_data[44:],
sample_rate=self._settings["sample_rate"],
num_channels=1,
)
yield TTSStoppedFrame()
elif result.reason == ResultReason.Canceled:
cancellation_details = result.cancellation_details
logger.warning(f"Speech synthesis canceled: {cancellation_details.reason}")
@@ -118,16 +278,17 @@ class AzureTTSService(TTSService):
logger.error(f"{self} error: {cancellation_details.error_details}")
class AzureSTTService(AsyncAIService):
class AzureSTTService(STTService):
def __init__(
self,
*,
api_key: str,
region: str,
language="en-US",
sample_rate=16000,
channels=1,
**kwargs):
self,
*,
api_key: str,
region: str,
language="en-US",
sample_rate=16000,
channels=1,
**kwargs,
):
super().__init__(**kwargs)
speech_config = SpeechConfig(subscription=api_key, region=region)
@@ -138,18 +299,15 @@ class AzureSTTService(AsyncAIService):
audio_config = AudioConfig(stream=self._audio_stream)
self._speech_recognizer = SpeechRecognizer(
speech_config=speech_config, audio_config=audio_config)
speech_config=speech_config, audio_config=audio_config
)
self._speech_recognizer.recognized.connect(self._on_handle_recognized)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, SystemFrame):
await self.push_frame(frame, direction)
elif isinstance(frame, AudioRawFrame):
self._audio_stream.write(frame.audio)
else:
await self._push_queue.put((frame, direction))
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
await self.start_processing_metrics()
self._audio_stream.write(audio)
await self.stop_processing_metrics()
yield None
async def start(self, frame: StartFrame):
await super().start(frame)
@@ -168,11 +326,10 @@ class AzureSTTService(AsyncAIService):
def _on_handle_recognized(self, event):
if event.result.reason == ResultReason.RecognizedSpeech and len(event.result.text) > 0:
frame = TranscriptionFrame(event.result.text, "", time_now_iso8601())
asyncio.run_coroutine_threadsafe(self.queue_frame(frame), self.get_event_loop())
asyncio.run_coroutine_threadsafe(self.push_frame(frame), self.get_event_loop())
class AzureImageGenServiceREST(ImageGenService):
def __init__(
self,
*,
@@ -188,16 +345,14 @@ class AzureImageGenServiceREST(ImageGenService):
self._api_key = api_key
self._azure_endpoint = endpoint
self._api_version = api_version
self._model = model
self.set_model_name(model)
self._image_size = image_size
self._aiohttp_session = aiohttp_session
async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]:
url = f"{self._azure_endpoint}openai/images/generations:submit?api-version={self._api_version}"
headers = {
"api-key": self._api_key,
"Content-Type": "application/json"}
headers = {"api-key": self._api_key, "Content-Type": "application/json"}
body = {
# Enter your prompt text here
@@ -239,8 +394,6 @@ class AzureImageGenServiceREST(ImageGenService):
image_stream = io.BytesIO(await response.content.read())
image = Image.open(image_stream)
frame = URLImageRawFrame(
url=image_url,
image=image.tobytes(),
size=image.size,
format=image.format)
url=image_url, image=image.tobytes(), size=image.size, format=image.format
)
yield frame

325
src/pipecat/services/cartesia.py
View File

@@ -4,40 +4,40 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
import base64
import json
import uuid
import base64
import asyncio
import time
from typing import AsyncGenerator, List, Optional, Union
from typing import AsyncGenerator
from loguru import logger
from pydantic.main import BaseModel
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
ErrorFrame,
Frame,
AudioRawFrame,
StartInterruptionFrame,
LLMFullResponseEndFrame,
StartFrame,
EndFrame,
StartInterruptionFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
TextFrame,
LLMFullResponseEndFrame
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import TTSService, WordTTSService
from pipecat.transcriptions.language import Language
from pipecat.services.ai_services import TTSService
from loguru import logger
# See .env.example for Cartesia configuration needed
try:
import websockets
from cartesia import AsyncCartesia
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Cartesia, you need to `pip install pipecat-ai[cartesia]`. Also, set `CARTESIA_API_KEY` environment variable.")
"In order to use Cartesia, you need to `pip install pipecat-ai[cartesia]`. Also, set `CARTESIA_API_KEY` environment variable."
)
raise Exception(f"Missing module: {e}")
@@ -60,67 +60,96 @@ def language_to_cartesia_language(language: Language) -> str | None:
return None
class CartesiaTTSService(TTSService):
class CartesiaTTSService(WordTTSService):
class InputParams(BaseModel):
encoding: Optional[str] = "pcm_s16le"
sample_rate: Optional[int] = 16000
container: Optional[str] = "raw"
language: Optional[Language] = Language.EN
speed: Optional[Union[str, float]] = ""
emotion: Optional[List[str]] = []
def __init__(
self,
*,
api_key: str,
voice_id: str,
cartesia_version: str = "2024-06-10",
url: str = "wss://api.cartesia.ai/tts/websocket",
model_id: str = "sonic-english",
encoding: str = "pcm_s16le",
sample_rate: int = 16000,
language: str = "en",
**kwargs):
super().__init__(**kwargs)
self,
*,
api_key: str,
voice_id: str,
cartesia_version: str = "2024-06-10",
url: str = "wss://api.cartesia.ai/tts/websocket",
model: str = "sonic-english",
params: InputParams = InputParams(),
**kwargs,
):
# Aggregating sentences still gives cleaner-sounding results and fewer
# artifacts than streaming one word at a time. On average, waiting for
# a full sentence should only "cost" us 15ms or so with GPT-4o or a Llama 3
# model, and it's worth it for the better audio quality.
self._aggregate_sentences = True
# we don't want to automatically push LLM response text frames, because the
# context aggregators will add them to the LLM context even if we're
# interrupted. cartesia gives us word-by-word timestamps. we can use those
# to generate text frames ourselves aligned with the playout timing of the audio!
self._push_text_frames = False
# artifacts than streaming one word at a time. On average, waiting for a
# full sentence should only "cost" us 15ms or so with GPT-4o or a Llama
# 3 model, and it's worth it for the better audio quality.
#
# We also don't want to automatically push LLM response text frames,
# because the context aggregators will add them to the LLM context even
# if we're interrupted. Cartesia gives us word-by-word timestamps. We
# can use those to generate text frames ourselves aligned with the
# playout timing of the audio!
super().__init__(
aggregate_sentences=True,
push_text_frames=False,
sample_rate=params.sample_rate,
**kwargs,
)
self._api_key = api_key
self._cartesia_version = cartesia_version
self._url = url
self._voice_id = voice_id
self._model_id = model_id
self._output_format = {
"container": "raw",
"encoding": encoding,
"sample_rate": sample_rate,
self._settings = {
"output_format": {
"container": params.container,
"encoding": params.encoding,
"sample_rate": params.sample_rate,
},
"language": params.language if params.language else Language.EN,
"speed": params.speed,
"emotion": params.emotion,
}
self._language = language
self.set_model_name(model)
self.set_voice(voice_id)
self._websocket = None
self._context_id = None
self._context_id_start_timestamp = None
self._timestamped_words_buffer = []
self._receive_task = None
self._context_appending_task = None
def can_generate_metrics(self) -> bool:
return True
async def set_model(self, model: str):
logger.debug(f"Switching TTS model to: [{model}]")
self._model_id = model
await super().set_model(model)
logger.debug(f"Switching TTS model to: [{model}]")
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice_id = voice
def _build_msg(
self, text: str = "", continue_transcript: bool = True, add_timestamps: bool = True
):
voice_config = {}
voice_config["mode"] = "id"
voice_config["id"] = self._voice_id
async def set_language(self, language: Language):
logger.debug(f"Switching TTS language to: [{language}]")
self._language = language_to_cartesia_language(language)
if self._settings["speed"] or self._settings["emotion"]:
voice_config["__experimental_controls"] = {}
if self._settings["speed"]:
voice_config["__experimental_controls"]["speed"] = self._settings["speed"]
if self._settings["emotion"]:
voice_config["__experimental_controls"]["emotion"] = self._settings["emotion"]
msg = {
"transcript": text,
"continue": continue_transcript,
"context_id": self._context_id,
"model_id": self.model_name,
"voice": voice_config,
"output_format": self._settings["output_format"],
"language": language_to_cartesia_language(self._settings["language"]),
"add_timestamps": add_timestamps,
}
return json.dumps(msg)
async def start(self, frame: StartFrame):
await super().start(frame)
@@ -140,44 +169,48 @@ class CartesiaTTSService(TTSService):
f"{self._url}?api_key={self._api_key}&cartesia_version={self._cartesia_version}"
)
self._receive_task = self.get_event_loop().create_task(self._receive_task_handler())
self._context_appending_task = self.get_event_loop().create_task(self._context_appending_task_handler())
except Exception as e:
logger.exception(f"{self} initialization error: {e}")
logger.error(f"{self} initialization error: {e}")
self._websocket = None
async def _disconnect(self):
try:
await self.stop_all_metrics()
if self._context_appending_task:
self._context_appending_task.cancel()
await self._context_appending_task
self._context_appending_task = None
if self._receive_task:
self._receive_task.cancel()
await self._receive_task
self._receive_task = None
if self._websocket:
await self._websocket.close()
self._websocket = None
if self._receive_task:
self._receive_task.cancel()
await self._receive_task
self._receive_task = None
self._context_id = None
self._context_id_start_timestamp = None
self._timestamped_words_buffer = []
except Exception as e:
logger.exception(f"{self} error closing websocket: {e}")
logger.error(f"{self} error closing websocket: {e}")
def _get_websocket(self):
if self._websocket:
return self._websocket
raise Exception("Websocket not connected")
async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
await super()._handle_interruption(frame, direction)
self._context_id = None
self._context_id_start_timestamp = None
self._timestamped_words_buffer = []
await self.stop_all_metrics()
await self.push_frame(LLMFullResponseEndFrame())
self._context_id = None
async def flush_audio(self):
if not self._context_id or not self._websocket:
return
logger.trace("Flushing audio")
msg = self._build_msg(text="", continue_transcript=False)
await self._websocket.send(msg)
async def _receive_task_handler(self):
try:
async for message in self._websocket:
async for message in self._get_websocket():
msg = json.loads(message)
if not msg or msg["context_id"] != self._context_id:
continue
@@ -188,20 +221,18 @@ class CartesiaTTSService(TTSService):
# because we are likely still playing out audio and need the
# timestamp to set send context frames.
self._context_id = None
self._timestamped_words_buffer.append(("LLMFullResponseEndFrame", 0))
await self.add_word_timestamps([("LLMFullResponseEndFrame", 0)])
elif msg["type"] == "timestamps":
# logger.debug(f"TIMESTAMPS: {msg}")
self._timestamped_words_buffer.extend(
list(zip(msg["word_timestamps"]["words"], msg["word_timestamps"]["end"]))
await self.add_word_timestamps(
list(zip(msg["word_timestamps"]["words"], msg["word_timestamps"]["start"]))
)
elif msg["type"] == "chunk":
await self.stop_ttfb_metrics()
if not self._context_id_start_timestamp:
self._context_id_start_timestamp = time.time()
frame = AudioRawFrame(
self.start_word_timestamps()
frame = TTSAudioRawFrame(
audio=base64.b64decode(msg["data"]),
sample_rate=self._output_format["sample_rate"],
num_channels=1
sample_rate=self._settings["output_format"]["sample_rate"],
num_channels=1,
)
await self.push_frame(frame)
elif msg["type"] == "error":
@@ -214,28 +245,7 @@ class CartesiaTTSService(TTSService):
except asyncio.CancelledError:
pass
except Exception as e:
logger.exception(f"{self} exception: {e}")
async def _context_appending_task_handler(self):
try:
while True:
await asyncio.sleep(0.1)
if not self._context_id_start_timestamp:
continue
elapsed_seconds = time.time() - self._context_id_start_timestamp
# Pop all words from self._timestamped_words_buffer that are
# older than the elapsed time and print a message about them to
# the console.
while self._timestamped_words_buffer and self._timestamped_words_buffer[0][1] <= elapsed_seconds:
word, timestamp = self._timestamped_words_buffer.pop(0)
if word == "LLMFullResponseEndFrame" and timestamp == 0:
await self.push_frame(LLMFullResponseEndFrame())
continue
await self.push_frame(TextFrame(word))
except asyncio.CancelledError:
pass
except Exception as e:
logger.exception(f"{self} exception: {e}")
logger.error(f"{self} exception: {e}")
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
@@ -245,32 +255,109 @@ class CartesiaTTSService(TTSService):
await self._connect()
if not self._context_id:
await self.push_frame(TTSStartedFrame())
await self.start_ttfb_metrics()
yield TTSStartedFrame()
self._context_id = str(uuid.uuid4())
msg = {
"transcript": text + " ",
"continue": True,
"context_id": self._context_id,
"model_id": self._model_id,
"voice": {
"mode": "id",
"id": self._voice_id
},
"output_format": self._output_format,
"language": self._language,
"add_timestamps": True,
}
msg = self._build_msg(text=text)
try:
await self._websocket.send(json.dumps(msg))
await self._get_websocket().send(msg)
await self.start_tts_usage_metrics(text)
except Exception as e:
logger.error(f"{self} error sending message: {e}")
await self.push_frame(TTSStoppedFrame())
yield TTSStoppedFrame()
await self._disconnect()
await self._connect()
return
yield None
except Exception as e:
logger.exception(f"{self} exception: {e}")
logger.error(f"{self} exception: {e}")
class CartesiaHttpTTSService(TTSService):
class InputParams(BaseModel):
encoding: Optional[str] = "pcm_s16le"
sample_rate: Optional[int] = 16000
container: Optional[str] = "raw"
language: Optional[Language] = Language.EN
speed: Optional[Union[str, float]] = ""
emotion: Optional[List[str]] = []
def __init__(
self,
*,
api_key: str,
voice_id: str,
model: str = "sonic-english",
base_url: str = "https://api.cartesia.ai",
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(**kwargs)
self._api_key = api_key
self._settings = {
"output_format": {
"container": params.container,
"encoding": params.encoding,
"sample_rate": params.sample_rate,
},
"language": params.language if params.language else Language.EN,
"speed": params.speed,
"emotion": params.emotion,
}
self.set_voice(voice_id)
self.set_model_name(model)
self._client = AsyncCartesia(api_key=api_key, base_url=base_url)
def can_generate_metrics(self) -> bool:
return True
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._client.close()
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._client.close()
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
await self.start_ttfb_metrics()
yield TTSStartedFrame()
try:
voice_controls = None
if self._settings["speed"] or self._settings["emotion"]:
voice_controls = {}
if self._settings["speed"]:
voice_controls["speed"] = self._settings["speed"]
if self._settings["emotion"]:
voice_controls["emotion"] = self._settings["emotion"]
output = await self._client.tts.sse(
model_id=self._model_name,
transcript=text,
voice_id=self._voice_id,
output_format=self._settings["output_format"],
language=language_to_cartesia_language(self._settings["language"]),
stream=False,
_experimental_voice_controls=voice_controls,
)
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(
audio=output["audio"],
sample_rate=self._settings["output_format"]["sample_rate"],
num_channels=1,
)
yield frame
except Exception as e:
logger.error(f"{self} exception: {e}")
await self.start_tts_usage_metrics(text)
yield TTSStoppedFrame()

188
src/pipecat/services/deepgram.py
View File

@@ -4,145 +4,157 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import aiohttp
import asyncio
from typing import AsyncGenerator
from loguru import logger
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,
EndFrame,
ErrorFrame,
Frame,
InterimTranscriptionFrame,
StartFrame,
TranscriptionFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
TranscriptionFrame)
)
from pipecat.services.ai_services import STTService, TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from loguru import logger
# See .env.example for Deepgram configuration needed
try:
from deepgram import (
AsyncListenWebSocketClient,
DeepgramClient,
DeepgramClientOptions,
LiveTranscriptionEvents,
LiveOptions,
LiveResultResponse
LiveResultResponse,
LiveTranscriptionEvents,
SpeakOptions,
)
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Deepgram, you need to `pip install pipecat-ai[deepgram]`. Also, set `DEEPGRAM_API_KEY` environment variable.")
"In order to use Deepgram, you need to `pip install pipecat-ai[deepgram]`. Also, set `DEEPGRAM_API_KEY` environment variable."
)
raise Exception(f"Missing module: {e}")
class DeepgramTTSService(TTSService):
def __init__(
self,
*,
api_key: str,
aiohttp_session: aiohttp.ClientSession,
voice: str = "aura-helios-en",
base_url: str = "https://api.deepgram.com/v1/speak",
sample_rate: int = 16000,
encoding: str = "linear16",
**kwargs):
self,
*,
api_key: str,
voice: str = "aura-helios-en",
sample_rate: int = 16000,
encoding: str = "linear16",
**kwargs,
):
super().__init__(**kwargs)
self._voice = voice
self._api_key = api_key
self._base_url = base_url
self._sample_rate = sample_rate
self._encoding = encoding
self._aiohttp_session = aiohttp_session
self._settings = {
"sample_rate": sample_rate,
"encoding": encoding,
}
self.set_voice(voice)
self._deepgram_client = DeepgramClient(api_key=api_key)
def can_generate_metrics(self) -> bool:
return True
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice = voice
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
base_url = self._base_url
request_url = f"{base_url}?model={self._voice}&encoding={self._encoding}&container=none&sample_rate={self._sample_rate}"
headers = {"authorization": f"token {self._api_key}"}
body = {"text": text}
options = SpeakOptions(
model=self._voice_id,
encoding=self._settings["encoding"],
sample_rate=self._settings["sample_rate"],
container="none",
)
try:
await self.start_ttfb_metrics()
async with self._aiohttp_session.post(request_url, headers=headers, json=body) as r:
if r.status != 200:
response_text = await r.text()
# If we get a a "Bad Request: Input is unutterable", just print out a debug log.
# All other unsuccesful requests should emit an error frame. If not specifically
# handled by the running PipelineTask, the ErrorFrame will cancel the task.
if "unutterable" in response_text:
logger.debug(f"Unutterable text: [{text}]")
return
logger.error(
f"{self} error getting audio (status: {r.status}, error: {response_text})")
yield ErrorFrame(f"Error getting audio (status: {r.status}, error: {response_text})")
return
response = await asyncio.to_thread(
self._deepgram_client.speak.v("1").stream, {"text": text}, options
)
await self.start_tts_usage_metrics(text)
await self.start_tts_usage_metrics(text)
yield TTSStartedFrame()
# The response.stream_memory is already a BytesIO object
audio_buffer = response.stream_memory
if audio_buffer is None:
raise ValueError("No audio data received from Deepgram")
# Read and yield the audio data in chunks
audio_buffer.seek(0) # Ensure we're at the start of the buffer
chunk_size = 8192 # Use a fixed buffer size
while True:
await self.stop_ttfb_metrics()
chunk = audio_buffer.read(chunk_size)
if not chunk:
break
frame = TTSAudioRawFrame(
audio=chunk, sample_rate=self._settings["sample_rate"], num_channels=1
)
yield frame
yield TTSStoppedFrame()
await self.push_frame(TTSStartedFrame())
async for data in r.content:
await self.stop_ttfb_metrics()
frame = AudioRawFrame(audio=data, sample_rate=self._sample_rate, num_channels=1)
yield frame
await self.push_frame(TTSStoppedFrame())
except Exception as e:
logger.exception(f"{self} exception: {e}")
yield ErrorFrame(f"Error getting audio: {str(e)}")
class DeepgramSTTService(STTService):
def __init__(self,
*,
api_key: str,
url: str = "",
live_options: LiveOptions = LiveOptions(
encoding="linear16",
language="en-US",
model="nova-2-conversationalai",
sample_rate=16000,
channels=1,
interim_results=True,
smart_format=True,
punctuate=True,
profanity_filter=True,
),
**kwargs):
def __init__(
self,
*,
api_key: str,
url: str = "",
live_options: LiveOptions = LiveOptions(
encoding="linear16",
language=Language.EN,
model="nova-2-conversationalai",
sample_rate=16000,
channels=1,
interim_results=True,
smart_format=True,
punctuate=True,
profanity_filter=True,
vad_events=False,
),
**kwargs,
):
super().__init__(**kwargs)
self._live_options = live_options
self._settings = vars(live_options)
self._client = DeepgramClient(
api_key, config=DeepgramClientOptions(url=url, options={"keepalive": "true"}))
api_key, config=DeepgramClientOptions(url=url, options={"keepalive": "true"})
)
self._connection: AsyncListenWebSocketClient = self._client.listen.asyncwebsocket.v("1")
self._connection.on(LiveTranscriptionEvents.Transcript, self._on_message)
if self.vad_enabled:
self._connection.on(LiveTranscriptionEvents.SpeechStarted, self._on_speech_started)
@property
def vad_enabled(self):
return self._settings["vad_events"]
def can_generate_metrics(self) -> bool:
return self.vad_enabled
async def set_model(self, model: str):
await super().set_model(model)
logger.debug(f"Switching STT model to: [{model}]")
self._live_options.model = model
await self._disconnect()
await self._connect()
async def set_language(self, language: Language):
logger.debug(f"Switching STT language to: [{language}]")
self._live_options.language = language
self._settings["model"] = model
await self._disconnect()
await self._connect()
@@ -159,13 +171,11 @@ class DeepgramSTTService(STTService):
await self._disconnect()
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
await self.start_processing_metrics()
await self._connection.send(audio)
yield None
await self.stop_processing_metrics()
async def _connect(self):
if await self._connection.start(self._live_options):
if await self._connection.start(self._settings):
logger.debug(f"{self}: Connected to Deepgram")
else:
logger.error(f"{self}: Unable to connect to Deepgram")
@@ -175,6 +185,10 @@ class DeepgramSTTService(STTService):
await self._connection.finish()
logger.debug(f"{self}: Disconnected from Deepgram")
async def _on_speech_started(self, *args, **kwargs):
await self.start_ttfb_metrics()
await self.start_processing_metrics()
async def _on_message(self, *args, **kwargs):
result: LiveResultResponse = kwargs["result"]
if len(result.channel.alternatives) == 0:
@@ -186,7 +200,13 @@ class DeepgramSTTService(STTService):
language = result.channel.alternatives[0].languages[0]
language = Language(language)
if len(transcript) > 0:
await self.stop_ttfb_metrics()
if is_final:
await self.push_frame(TranscriptionFrame(transcript, "", time_now_iso8601(), language))
await self.push_frame(
TranscriptionFrame(transcript, "", time_now_iso8601(), language)
)
await self.stop_processing_metrics()
else:
await self.push_frame(InterimTranscriptionFrame(transcript, "", time_now_iso8601(), language))
await self.push_frame(
InterimTranscriptionFrame(transcript, "", time_now_iso8601(), language)
)

404
src/pipecat/services/elevenlabs.py
View File

@@ -4,21 +4,36 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import AsyncGenerator, Literal
from pydantic import BaseModel
from pipecat.frames.frames import AudioRawFrame, Frame, TTSStartedFrame, TTSStoppedFrame
from pipecat.services.ai_services import TTSService
import asyncio
import base64
import json
from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional, Tuple
from loguru import logger
from pydantic import BaseModel, model_validator
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
Frame,
StartFrame,
StartInterruptionFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import WordTTSService
from pipecat.transcriptions.language import Language
# See .env.example for ElevenLabs configuration needed
try:
from elevenlabs.client import AsyncElevenLabs
import websockets
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use ElevenLabs, you need to `pip install pipecat-ai[elevenlabs]`. Also, set `ELEVENLABS_API_KEY` environment variable.")
"In order to use ElevenLabs, you need to `pip install pipecat-ai[elevenlabs]`. Also, set `ELEVENLABS_API_KEY` environment variable."
)
raise Exception(f"Missing module: {e}")
@@ -35,59 +50,354 @@ def sample_rate_from_output_format(output_format: str) -> int:
return 16000
class ElevenLabsTTSService(TTSService):
def language_to_elevenlabs_language(language: Language) -> str | None:
match language:
case Language.BG:
return "bg"
case Language.ZH:
return "zh"
case Language.CS:
return "cs"
case Language.DA:
return "da"
case Language.NL:
return "nl"
case (
Language.EN
| Language.EN_US
| Language.EN_AU
| Language.EN_GB
| Language.EN_NZ
| Language.EN_IN
):
return "en"
case Language.FI:
return "fi"
case Language.FR | Language.FR_CA:
return "fr"
case Language.DE | Language.DE_CH:
return "de"
case Language.EL:
return "el"
case Language.HI:
return "hi"
case Language.HU:
return "hu"
case Language.ID:
return "id"
case Language.IT:
return "it"
case Language.JA:
return "ja"
case Language.KO:
return "ko"
case Language.MS:
return "ms"
case Language.NO:
return "no"
case Language.PL:
return "pl"
case Language.PT:
return "pt-PT"
case Language.PT_BR:
return "pt-BR"
case Language.RO:
return "ro"
case Language.RU:
return "ru"
case Language.SK:
return "sk"
case Language.ES:
return "es"
case Language.SV:
return "sv"
case Language.TR:
return "tr"
case Language.UK:
return "uk"
case Language.VI:
return "vi"
return None
def calculate_word_times(
alignment_info: Mapping[str, Any], cumulative_time: float
) -> List[Tuple[str, float]]:
zipped_times = list(zip(alignment_info["chars"], alignment_info["charStartTimesMs"]))
words = "".join(alignment_info["chars"]).split(" ")
# Calculate start time for each word. We do this by finding a space character
# and using the previous word time, also taking into account there might not
# be a space at the end.
times = []
for i, (a, b) in enumerate(zipped_times):
if a == " " or i == len(zipped_times) - 1:
t = cumulative_time + (zipped_times[i - 1][1] / 1000.0)
times.append(t)
word_times = list(zip(words, times))
return word_times
class ElevenLabsTTSService(WordTTSService):
class InputParams(BaseModel):
language: Optional[Language] = Language.EN
output_format: Literal["pcm_16000", "pcm_22050", "pcm_24000", "pcm_44100"] = "pcm_16000"
optimize_streaming_latency: Optional[str] = None
stability: Optional[float] = None
similarity_boost: Optional[float] = None
style: Optional[float] = None
use_speaker_boost: Optional[bool] = None
@model_validator(mode="after")
def validate_voice_settings(self):
stability = self.stability
similarity_boost = self.similarity_boost
if (stability is None) != (similarity_boost is None):
raise ValueError(
"Both 'stability' and 'similarity_boost' must be provided when using voice settings"
)
return self
def __init__(
self,
*,
api_key: str,
voice_id: str,
model: str = "eleven_turbo_v2_5",
params: InputParams = InputParams(),
**kwargs):
super().__init__(**kwargs)
self,
*,
api_key: str,
voice_id: str,
model: str = "eleven_turbo_v2_5",
url: str = "wss://api.elevenlabs.io",
params: InputParams = InputParams(),
**kwargs,
):
# Aggregating sentences still gives cleaner-sounding results and fewer
# artifacts than streaming one word at a time. On average, waiting for a
# full sentence should only "cost" us 15ms or so with GPT-4o or a Llama
# 3 model, and it's worth it for the better audio quality.
#
# We also don't want to automatically push LLM response text frames,
# because the context aggregators will add them to the LLM context even
# if we're interrupted. ElevenLabs gives us word-by-word timestamps. We
# can use those to generate text frames ourselves aligned with the
# playout timing of the audio!
#
# Finally, ElevenLabs doesn't provide information on when the bot stops
# speaking for a while, so we want the parent class to send TTSStopFrame
# after a short period not receiving any audio.
super().__init__(
aggregate_sentences=True,
push_text_frames=False,
push_stop_frames=True,
stop_frame_timeout_s=2.0,
sample_rate=sample_rate_from_output_format(params.output_format),
**kwargs,
)
self._voice_id = voice_id
self._model = model
self._params = params
self._client = AsyncElevenLabs(api_key=api_key)
self._sample_rate = sample_rate_from_output_format(params.output_format)
self._api_key = api_key
self._url = url
self._settings = {
"sample_rate": sample_rate_from_output_format(params.output_format),
"language": params.language if params.language else Language.EN,
"output_format": params.output_format,
"optimize_streaming_latency": params.optimize_streaming_latency,
"stability": params.stability,
"similarity_boost": params.similarity_boost,
"style": params.style,
"use_speaker_boost": params.use_speaker_boost,
}
self.set_model_name(model)
self.set_voice(voice_id)
self._voice_settings = self._set_voice_settings()
# Websocket connection to ElevenLabs.
self._websocket = None
# Indicates if we have sent TTSStartedFrame. It will reset to False when
# there's an interruption or TTSStoppedFrame.
self._started = False
self._cumulative_time = 0
def can_generate_metrics(self) -> bool:
return True
async def set_model(self, model: str):
logger.debug(f"Switching TTS model to: [{model}]")
self._model = model
def _set_voice_settings(self):
voice_settings = {}
if (
self._settings["stability"] is not None
and self._settings["similarity_boost"] is not None
):
voice_settings["stability"] = self._settings["stability"]
voice_settings["similarity_boost"] = self._settings["similarity_boost"]
if self._settings["style"] is not None:
voice_settings["style"] = self._settings["style"]
if self._settings["use_speaker_boost"] is not None:
voice_settings["use_speaker_boost"] = self._settings["use_speaker_boost"]
else:
if self._settings["style"] is not None:
logger.warning(
"'style' is set but will not be applied because 'stability' and 'similarity_boost' are not both set."
)
if self._settings["use_speaker_boost"] is not None:
logger.warning(
"'use_speaker_boost' is set but will not be applied because 'stability' and 'similarity_boost' are not both set."
)
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice_id = voice
return voice_settings or None
async def set_model(self, model: str):
await super().set_model(model)
logger.debug(f"Switching TTS model to: [{model}]")
await self._disconnect()
await self._connect()
async def _update_settings(self, settings: Dict[str, Any]):
prev_voice = self._voice_id
await super()._update_settings(settings)
if not prev_voice == self._voice_id:
await self._disconnect()
await self._connect()
logger.debug(f"Switching TTS voice to: [{self._voice_id}]")
async def start(self, frame: StartFrame):
await super().start(frame)
await self._connect()
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._disconnect()
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._disconnect()
async def flush_audio(self):
if self._websocket:
msg = {"text": " ", "flush": True}
await self._websocket.send(json.dumps(msg))
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
await super().push_frame(frame, direction)
if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
self._started = False
if isinstance(frame, TTSStoppedFrame):
await self.add_word_timestamps([("LLMFullResponseEndFrame", 0)])
async def _connect(self):
try:
voice_id = self._voice_id
model = self.model_name
output_format = self._settings["output_format"]
url = f"{self._url}/v1/text-to-speech/{voice_id}/stream-input?model_id={model}&output_format={output_format}"
if self._settings["optimize_streaming_latency"]:
url += f"&optimize_streaming_latency={self._settings['optimize_streaming_latency']}"
# Language can only be used with the 'eleven_turbo_v2_5' model
language = language_to_elevenlabs_language(self._settings["language"])
if model == "eleven_turbo_v2_5":
url += f"&language_code={language}"
else:
logger.debug(
f"Language code [{language}] not applied. Language codes can only be used with the 'eleven_turbo_v2_5' model."
)
self._websocket = await websockets.connect(url)
self._receive_task = self.get_event_loop().create_task(self._receive_task_handler())
self._keepalive_task = self.get_event_loop().create_task(self._keepalive_task_handler())
# According to ElevenLabs, we should always start with a single space.
msg: Dict[str, Any] = {
"text": " ",
"xi_api_key": self._api_key,
}
if self._voice_settings:
msg["voice_settings"] = self._voice_settings
await self._websocket.send(json.dumps(msg))
except Exception as e:
logger.error(f"{self} initialization error: {e}")
self._websocket = None
async def _disconnect(self):
try:
await self.stop_all_metrics()
if self._websocket:
await self._websocket.send(json.dumps({"text": ""}))
await self._websocket.close()
self._websocket = None
if self._receive_task:
self._receive_task.cancel()
await self._receive_task
self._receive_task = None
if self._keepalive_task:
self._keepalive_task.cancel()
await self._keepalive_task
self._keepalive_task = None
self._started = False
except Exception as e:
logger.error(f"{self} error closing websocket: {e}")
async def _receive_task_handler(self):
try:
async for message in self._websocket:
msg = json.loads(message)
if msg.get("audio"):
await self.stop_ttfb_metrics()
self.start_word_timestamps()
audio = base64.b64decode(msg["audio"])
frame = TTSAudioRawFrame(audio, self._settings["sample_rate"], 1)
await self.push_frame(frame)
if msg.get("alignment"):
word_times = calculate_word_times(msg["alignment"], self._cumulative_time)
await self.add_word_timestamps(word_times)
self._cumulative_time = word_times[-1][1]
except asyncio.CancelledError:
pass
except Exception as e:
logger.error(f"{self} exception: {e}")
async def _keepalive_task_handler(self):
while True:
try:
await asyncio.sleep(10)
await self._send_text("")
except asyncio.CancelledError:
break
except Exception as e:
logger.error(f"{self} exception: {e}")
async def _send_text(self, text: str):
if self._websocket:
msg = {"text": text + " "}
await self._websocket.send(json.dumps(msg))
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
await self.start_tts_usage_metrics(text)
await self.start_ttfb_metrics()
try:
if not self._websocket:
await self._connect()
results = await self._client.generate(
text=text,
voice=self._voice_id,
model=self._model,
output_format=self._params.output_format
)
try:
if not self._started:
await self.start_ttfb_metrics()
yield TTSStartedFrame()
self._started = True
self._cumulative_time = 0
tts_started = False
async for audio in results:
# This is so we send TTSStartedFrame when we have the first audio
# bytes.
if not tts_started:
await self.push_frame(TTSStartedFrame())
tts_started = True
await self.stop_ttfb_metrics()
frame = AudioRawFrame(audio, self._sample_rate, 1)
yield frame
await self.push_frame(TTSStoppedFrame())
await self._send_text(text)
await self.start_tts_usage_metrics(text)
except Exception as e:
logger.error(f"{self} error sending message: {e}")
yield TTSStoppedFrame()
await self._disconnect()
await self._connect()
return
yield None
except Exception as e:
logger.error(f"{self} exception: {e}")

21
src/pipecat/services/fal.py
View File

@@ -8,13 +8,14 @@ import aiohttp
import io
import os
from PIL import Image
from pydantic import BaseModel
from typing import AsyncGenerator, Optional, Union, Dict
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.ai_services import ImageGenService
from PIL import Image
from loguru import logger
try:
@@ -22,7 +23,8 @@ try:
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Fal, you need to `pip install pipecat-ai[fal]`. Also, set `FAL_KEY` environment variable.")
"In order to use Fal, you need to `pip install pipecat-ai[fal]`. Also, set `FAL_KEY` environment variable."
)
raise Exception(f"Missing module: {e}")
@@ -43,9 +45,10 @@ class FalImageGenService(ImageGenService):
aiohttp_session: aiohttp.ClientSession,
model: str = "fal-ai/fast-sdxl",
key: str | None = None,
**kwargs,
):
super().__init__()
self._model = model
super().__init__(**kwargs)
self.set_model_name(model)
self._params = params
self._aiohttp_session = aiohttp_session
if key:
@@ -55,8 +58,8 @@ class FalImageGenService(ImageGenService):
logger.debug(f"Generating image from prompt: {prompt}")
response = await fal_client.run_async(
self._model,
arguments={"prompt": prompt, **self._params.model_dump(exclude_none=True)}
self.model_name,
arguments={"prompt": prompt, **self._params.model_dump(exclude_none=True)},
)
image_url = response["images"][0]["url"] if response else None
@@ -76,8 +79,6 @@ class FalImageGenService(ImageGenService):
image = Image.open(image_stream)
frame = URLImageRawFrame(
url=image_url,
image=image.tobytes(),
size=image.size,
format=image.format)
url=image_url, image=image.tobytes(), size=image.size, format=image.format
)
yield frame

15
src/pipecat/services/fireworks.py
View File

@@ -13,13 +13,16 @@ try:
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Fireworks, you need to `pip install pipecat-ai[fireworks]`. Also, set the `FIREWORKS_API_KEY` environment variable.")
"In order to use Fireworks, you need to `pip install pipecat-ai[fireworks]`. Also, set the `FIREWORKS_API_KEY` environment variable."
)
raise Exception(f"Missing module: {e}")
class FireworksLLMService(BaseOpenAILLMService):
def __init__(self,
*,
model: str = "accounts/fireworks/models/firefunction-v1",
base_url: str = "https://api.fireworks.ai/inference/v1"):
super().__init__(model, base_url)
def __init__(
self,
*,
model: str = "accounts/fireworks/models/firefunction-v1",
base_url: str = "https://api.fireworks.ai/inference/v1",
):
super().__init__(model=model, base_url=base_url)

159
src/pipecat/services/gladia.py
View File

@@ -6,67 +6,142 @@
import base64
import json
from typing import AsyncGenerator, Optional
from typing import Optional
from loguru import logger
from pydantic.main import BaseModel
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,
EndFrame,
Frame,
InterimTranscriptionFrame,
StartFrame,
SystemFrame,
TranscriptionFrame)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import AsyncAIService
TranscriptionFrame,
)
from pipecat.services.ai_services import STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from loguru import logger
# See .env.example for Gladia configuration needed
try:
import websockets
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Gladia, you need to `pip install pipecat-ai[gladia]`. Also, set `GLADIA_API_KEY` environment variable.")
"In order to use Gladia, you need to `pip install pipecat-ai[gladia]`. Also, set `GLADIA_API_KEY` environment variable."
)
raise Exception(f"Missing module: {e}")
class GladiaSTTService(AsyncAIService):
def language_to_gladia_language(language: Language) -> str | None:
match language:
case Language.BG:
return "bulgarian"
case Language.CA:
return "catalan"
case Language.ZH:
return "chinese"
case Language.CS:
return "czech"
case Language.DA:
return "danish"
case Language.NL:
return "dutch"
case (
Language.EN
| Language.EN_US
| Language.EN_AU
| Language.EN_GB
| Language.EN_NZ
| Language.EN_IN
):
return "english"
case Language.ET:
return "estonian"
case Language.FI:
return "finnish"
case Language.FR | Language.FR_CA:
return "french"
case Language.DE | Language.DE_CH:
return "german"
case Language.EL:
return "greek"
case Language.HI:
return "hindi"
case Language.HU:
return "hungarian"
case Language.ID:
return "indonesian"
case Language.IT:
return "italian"
case Language.JA:
return "japanese"
case Language.KO:
return "korean"
case Language.LV:
return "latvian"
case Language.LT:
return "lithuanian"
case Language.MS:
return "malay"
case Language.NO:
return "norwegian"
case Language.PL:
return "polish"
case Language.PT | Language.PT_BR:
return "portuguese"
case Language.RO:
return "romanian"
case Language.RU:
return "russian"
case Language.SK:
return "slovak"
case Language.ES:
return "spanish"
case Language.SV:
return "slovenian"
case Language.TH:
return "thai"
case Language.TR:
return "turkish"
case Language.UK:
return "ukrainian"
case Language.VI:
return "vietnamese"
return None
class GladiaSTTService(STTService):
class InputParams(BaseModel):
sample_rate: Optional[int] = 16000
language: Optional[str] = "english"
language: Optional[Language] = Language.EN
transcription_hint: Optional[str] = None
endpointing: Optional[int] = 200
prosody: Optional[bool] = None
def __init__(self,
*,
api_key: str,
url: str = "wss://api.gladia.io/audio/text/audio-transcription",
confidence: float = 0.5,
params: InputParams = InputParams(),
**kwargs):
def __init__(
self,
*,
api_key: str,
url: str = "wss://api.gladia.io/audio/text/audio-transcription",
confidence: float = 0.5,
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(**kwargs)
self._api_key = api_key
self._url = url
self._params = params
self._settings = {
"sample_rate": params.sample_rate,
"language": params.language if params.language else Language.EN,
"transcription_hint": params.transcription_hint,
"endpointing": params.endpointing,
"prosody": params.prosody,
}
self._confidence = confidence
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, SystemFrame):
await self.push_frame(frame, direction)
elif isinstance(frame, AudioRawFrame):
await self._send_audio(frame)
else:
await self.queue_frame(frame, direction)
async def start(self, frame: StartFrame):
await super().start(frame)
self._websocket = await websockets.connect(self._url)
@@ -81,21 +156,29 @@ class GladiaSTTService(AsyncAIService):
await super().cancel(frame)
await self._websocket.close()
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
await self.start_processing_metrics()
await self._send_audio(audio)
await self.stop_processing_metrics()
yield None
async def _setup_gladia(self):
configuration = {
"x_gladia_key": self._api_key,
"encoding": "WAV/PCM",
"model_type": "fast",
"language_behaviour": "manual",
**self._params.model_dump(exclude_none=True)
"sample_rate": self._settings["sample_rate"],
"language": language_to_gladia_language(self._settings["language"]),
"transcription_hint": self._settings["transcription_hint"],
"endpointing": self._settings["endpointing"],
"prosody": self._settings["prosody"],
}
await self._websocket.send(json.dumps(configuration))
async def _send_audio(self, frame: AudioRawFrame):
message = {
'frames': base64.b64encode(frame.audio).decode("utf-8")
}
async def _send_audio(self, audio: bytes):
message = {"frames": base64.b64encode(audio).decode("utf-8")}
await self._websocket.send(json.dumps(message))
async def _receive_task_handler(self):
@@ -113,6 +196,10 @@ class GladiaSTTService(AsyncAIService):
transcript = utterance["transcription"]
if confidence >= self._confidence:
if type == "final":
await self.queue_frame(TranscriptionFrame(transcript, "", time_now_iso8601()))
await self.push_frame(
TranscriptionFrame(transcript, "", time_now_iso8601())
)
else:
await self.queue_frame(InterimTranscriptionFrame(transcript, "", time_now_iso8601()))
await self.push_frame(
InterimTranscriptionFrame(transcript, "", time_now_iso8601())
)

309
src/pipecat/services/google.py
View File

@@ -5,31 +5,43 @@
#
import asyncio
from typing import List
from pipecat.frames.frames import (
Frame,
LLMModelUpdateFrame,
TextFrame,
VisionImageRawFrame,
LLMMessagesFrame,
LLMFullResponseStartFrame,
LLMFullResponseEndFrame
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import LLMService
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext, OpenAILLMContextFrame
import json
from typing import AsyncGenerator, List, Literal, Optional
from loguru import logger
from pydantic import BaseModel
from pipecat.frames.frames import (
ErrorFrame,
Frame,
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMUpdateSettingsFrame,
TextFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
VisionImageRawFrame,
)
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import LLMService, TTSService
from pipecat.transcriptions.language import Language
try:
import google.generativeai as gai
import google.ai.generativelanguage as glm
import google.generativeai as gai
from google.cloud import texttospeech_v1
from google.oauth2 import service_account
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Google AI, you need to `pip install pipecat-ai[google]`. Also, set `GOOGLE_API_KEY` environment variable.")
"In order to use Google AI, you need to `pip install pipecat-ai[google]`. Also, set the environment variable GOOGLE_API_KEY for the GoogleLLMService and GOOGLE_APPLICATION_CREDENTIALS for the GoogleTTSService`."
)
raise Exception(f"Missing module: {e}")
@@ -50,10 +62,10 @@ class GoogleLLMService(LLMService):
return True
def _create_client(self, model: str):
self.set_model_name(model)
self._client = gai.GenerativeModel(model)
def _get_messages_from_openai_context(
self, context: OpenAILLMContext) -> List[glm.Content]:
def _get_messages_from_openai_context(self, context: OpenAILLMContext) -> List[glm.Content]:
openai_messages = context.get_messages()
google_messages = []
@@ -68,10 +80,12 @@ class GoogleLLMService(LLMService):
parts = [glm.Part(text=content)]
if "mime_type" in message:
parts.append(
glm.Part(inline_data=glm.Blob(
mime_type=message["mime_type"],
data=message["data"].getvalue()
)))
glm.Part(
inline_data=glm.Blob(
mime_type=message["mime_type"], data=message["data"].getvalue()
)
)
)
google_messages.append({"role": role, "parts": parts})
return google_messages
@@ -102,7 +116,8 @@ class GoogleLLMService(LLMService):
# Google LLMs seem to flag safety issues a lot!
if chunk.candidates[0].finish_reason == 3:
logger.debug(
f"LLM refused to generate content for safety reasons - {messages}.")
f"LLM refused to generate content for safety reasons - {messages}."
)
else:
logger.exception(f"{self} error: {e}")
@@ -122,11 +137,251 @@ class GoogleLLMService(LLMService):
context = OpenAILLMContext.from_messages(frame.messages)
elif isinstance(frame, VisionImageRawFrame):
context = OpenAILLMContext.from_image_frame(frame)
elif isinstance(frame, LLMModelUpdateFrame):
logger.debug(f"Switching LLM model to: [{frame.model}]")
self._create_client(frame.model)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)
if context:
await self._process_context(context)
def language_to_google_language(language: Language) -> str | None:
match language:
case Language.BG:
return "bg-BG"
case Language.CA:
return "ca-ES"
case Language.ZH:
return "cmn-CN"
case Language.ZH_TW:
return "cmn-TW"
case Language.CS:
return "cs-CZ"
case Language.DA:
return "da-DK"
case Language.NL:
return "nl-NL"
case Language.EN:
return "en-US"
case Language.EN_US:
return "en-US"
case Language.EN_AU:
return "en-AU"
case Language.EN_GB:
return "en-GB"
case Language.EN_IN:
return "en-IN"
case Language.ET:
return "et-EE"
case Language.FI:
return "fi-FI"
case Language.NL_BE:
return "nl-BE"
case Language.FR:
return "fr-FR"
case Language.FR_CA:
return "fr-CA"
case Language.DE:
return "de-DE"
case Language.EL:
return "el-GR"
case Language.HI:
return "hi-IN"
case Language.HU:
return "hu-HU"
case Language.ID:
return "id-ID"
case Language.IT:
return "it-IT"
case Language.JA:
return "ja-JP"
case Language.KO:
return "ko-KR"
case Language.LV:
return "lv-LV"
case Language.LT:
return "lt-LT"
case Language.MS:
return "ms-MY"
case Language.NO:
return "nb-NO"
case Language.PL:
return "pl-PL"
case Language.PT:
return "pt-PT"
case Language.PT_BR:
return "pt-BR"
case Language.RO:
return "ro-RO"
case Language.RU:
return "ru-RU"
case Language.SK:
return "sk-SK"
case Language.ES:
return "es-ES"
case Language.SV:
return "sv-SE"
case Language.TH:
return "th-TH"
case Language.TR:
return "tr-TR"
case Language.UK:
return "uk-UA"
case Language.VI:
return "vi-VN"
return None
class GoogleTTSService(TTSService):
class InputParams(BaseModel):
pitch: Optional[str] = None
rate: Optional[str] = None
volume: Optional[str] = None
emphasis: Optional[Literal["strong", "moderate", "reduced", "none"]] = None
language: Optional[Language] = Language.EN
gender: Optional[Literal["male", "female", "neutral"]] = None
google_style: Optional[Literal["apologetic", "calm", "empathetic", "firm", "lively"]] = None
def __init__(
self,
*,
credentials: Optional[str] = None,
credentials_path: Optional[str] = None,
voice_id: str = "en-US-Neural2-A",
sample_rate: int = 24000,
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
self._settings = {
"sample_rate": sample_rate,
"pitch": params.pitch,
"rate": params.rate,
"volume": params.volume,
"emphasis": params.emphasis,
"language": params.language if params.language else Language.EN,
"gender": params.gender,
"google_style": params.google_style,
}
self.set_voice(voice_id)
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path
)
def _create_client(
self, credentials: Optional[str], credentials_path: Optional[str]
) -> texttospeech_v1.TextToSpeechAsyncClient:
creds: Optional[service_account.Credentials] = None
# Create a Google Cloud service account for the Cloud Text-to-Speech API
# Using either the provided credentials JSON string or the path to a service account JSON
# file, create a Google Cloud service account and use it to authenticate with the API.
if credentials:
# Use provided credentials JSON string
json_account_info = json.loads(credentials)
creds = service_account.Credentials.from_service_account_info(json_account_info)
elif credentials_path:
# Use service account JSON file if provided
creds = service_account.Credentials.from_service_account_file(credentials_path)
return texttospeech_v1.TextToSpeechAsyncClient(credentials=creds)
def can_generate_metrics(self) -> bool:
return True
def _construct_ssml(self, text: str) -> str:
ssml = "<speak>"
# Voice tag
voice_attrs = [f"name='{self._voice_id}'"]
language = language_to_google_language(self._settings["language"])
voice_attrs.append(f"language='{language}'")
if self._settings["gender"]:
voice_attrs.append(f"gender='{self._settings['gender']}'")
ssml += f"<voice {' '.join(voice_attrs)}>"
# Prosody tag
prosody_attrs = []
if self._settings["pitch"]:
prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings['rate']}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings['volume']}'")
if prosody_attrs:
ssml += f"<prosody {' '.join(prosody_attrs)}>"
# Emphasis tag
if self._settings["emphasis"]:
ssml += f"<emphasis level='{self._settings['emphasis']}'>"
# Google style tag
if self._settings["google_style"]:
ssml += f"<google:style name='{self._settings['google_style']}'>"
ssml += text
# Close tags
if self._settings["google_style"]:
ssml += "</google:style>"
if self._settings["emphasis"]:
ssml += "</emphasis>"
if prosody_attrs:
ssml += "</prosody>"
ssml += "</voice></speak>"
return ssml
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
try:
await self.start_ttfb_metrics()
ssml = self._construct_ssml(text)
synthesis_input = texttospeech_v1.SynthesisInput(ssml=ssml)
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._settings["language"], name=self._voice_id
)
audio_config = texttospeech_v1.AudioConfig(
audio_encoding=texttospeech_v1.AudioEncoding.LINEAR16,
sample_rate_hertz=self._settings["sample_rate"],
)
request = texttospeech_v1.SynthesizeSpeechRequest(
input=synthesis_input, voice=voice, audio_config=audio_config
)
response = await self._client.synthesize_speech(request=request)
await self.start_tts_usage_metrics(text)
yield TTSStartedFrame()
# Skip the first 44 bytes to remove the WAV header
audio_content = response.audio_content[44:]
# Read and yield audio data in chunks
chunk_size = 8192
for i in range(0, len(audio_content), chunk_size):
chunk = audio_content[i : i + chunk_size]
if not chunk:
break
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
yield frame
await asyncio.sleep(0) # Allow other tasks to run
yield TTSStoppedFrame()
except Exception as e:
logger.exception(f"{self} error generating TTS: {e}")
error_message = f"TTS generation error: {str(e)}"
yield ErrorFrame(error=error_message)
finally:
yield TTSStoppedFrame()

95
src/pipecat/services/lmnt.py
View File

@@ -5,24 +5,24 @@
#
import asyncio
from typing import AsyncGenerator
from pipecat.processors.frame_processor import FrameDirection
from loguru import logger
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,
EndFrame,
ErrorFrame,
Frame,
StartFrame,
StartInterruptionFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import TTSService
from loguru import logger
from pipecat.transcriptions.language import Language
# See .env.example for LMNT configuration needed
try:
@@ -30,47 +30,73 @@ try:
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use LMNT, you need to `pip install pipecat-ai[lmnt]`. Also, set `LMNT_API_KEY` environment variable.")
"In order to use LMNT, you need to `pip install pipecat-ai[lmnt]`. Also, set `LMNT_API_KEY` environment variable."
)
raise Exception(f"Missing module: {e}")
def language_to_lmnt_language(language: Language) -> str | None:
match language:
case Language.DE:
return "de"
case (
Language.EN
| Language.EN_US
| Language.EN_AU
| Language.EN_GB
| Language.EN_NZ
| Language.EN_IN
):
return "en"
case Language.ES:
return "es"
case Language.FR | Language.FR_CA:
return "fr"
case Language.PT | Language.PT_BR:
return "pt"
case Language.ZH | Language.ZH_TW:
return "zh"
case Language.KO:
return "ko"
return None
class LmntTTSService(TTSService):
def __init__(
self,
*,
api_key: str,
voice_id: str,
sample_rate: int = 24000,
language: str = "en",
**kwargs):
super().__init__(**kwargs)
self,
*,
api_key: str,
voice_id: str,
sample_rate: int = 24000,
language: Language = Language.EN,
**kwargs,
):
# Let TTSService produce TTSStoppedFrames after a short delay of
# no activity.
self._push_stop_frames = True
super().__init__(push_stop_frames=True, sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._voice_id = voice_id
self._output_format = {
"container": "raw",
"encoding": "pcm_s16le",
"sample_rate": sample_rate,
self._settings = {
"output_format": {
"container": "raw",
"encoding": "pcm_s16le",
"sample_rate": sample_rate,
},
"language": language,
}
self._language = language
self.set_voice(voice_id)
self._speech = None
self._connection = None
self._receive_task = None
# Indicates if we have sent TTSStartedFrame. It will reset to False when
# there's an interruption or TTSStoppedFrame.
self._started = False
def can_generate_metrics(self) -> bool:
return True
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice_id = voice
async def start(self, frame: StartFrame):
await super().start(frame)
await self._connect()
@@ -92,7 +118,10 @@ class LmntTTSService(TTSService):
try:
self._speech = Speech()
self._connection = await self._speech.synthesize_streaming(
self._voice_id, format="raw", sample_rate=self._output_format["sample_rate"])
self._voice_id,
format="raw",
sample_rate=self._settings["output_format"]["sample_rate"],
)
self._receive_task = self.get_event_loop().create_task(self._receive_task_handler())
except Exception as e:
logger.exception(f"{self} initialization error: {e}")
@@ -126,10 +155,10 @@ class LmntTTSService(TTSService):
await self.push_error(ErrorFrame(f'{self} error: {msg["error"]}'))
elif "audio" in msg:
await self.stop_ttfb_metrics()
frame = AudioRawFrame(
frame = TTSAudioRawFrame(
audio=msg["audio"],
sample_rate=self._output_format["sample_rate"],
num_channels=1
sample_rate=self._settings["output_format"]["sample_rate"],
num_channels=1,
)
await self.push_frame(frame)
else:
@@ -147,8 +176,8 @@ class LmntTTSService(TTSService):
await self._connect()
if not self._started:
await self.push_frame(TTSStartedFrame())
await self.start_ttfb_metrics()
yield TTSStartedFrame()
self._started = True
try:
@@ -157,7 +186,7 @@ class LmntTTSService(TTSService):
await self.start_tts_usage_metrics(text)
except Exception as e:
logger.error(f"{self} error sending message: {e}")
await self.push_frame(TTSStoppedFrame())
yield TTSStoppedFrame()
await self._disconnect()
await self._connect()
return

18
src/pipecat/services/moondream.py
View File

@@ -31,6 +31,7 @@ def detect_device():
"""
try:
import intel_extension_for_pytorch
if torch.xpu.is_available():
return torch.device("xpu"), torch.float32
except ImportError:
@@ -45,13 +46,11 @@ def detect_device():
class MoondreamService(VisionService):
def __init__(
self,
*,
model="vikhyatk/moondream2",
revision="2024-04-02",
use_cpu=False
self, *, model="vikhyatk/moondream2", revision="2024-08-26", use_cpu=False, **kwargs
):
super().__init__()
super().__init__(**kwargs)
self.set_model_name(model)
if not use_cpu:
device, dtype = detect_device()
@@ -72,7 +71,7 @@ class MoondreamService(VisionService):
async def run_vision(self, frame: VisionImageRawFrame) -> AsyncGenerator[Frame, None]:
if not self._model:
logger.error(f"{self} error: Moondream model not available")
logger.error(f"{self} error: Moondream model not available ({self.model_name})")
yield ErrorFrame("Moondream model not available")
return
@@ -82,9 +81,8 @@ class MoondreamService(VisionService):
image = Image.frombytes(frame.format, frame.size, frame.image)
image_embeds = self._model.encode_image(image)
description = self._model.answer_question(
image_embeds=image_embeds,
question=frame.text,
tokenizer=self._tokenizer)
image_embeds=image_embeds, question=frame.text, tokenizer=self._tokenizer
)
return description
description = await asyncio.to_thread(get_image_description, frame)

1
src/pipecat/services/ollama.py
View File

@@ -8,6 +8,5 @@ from pipecat.services.openai import BaseOpenAILLMService
class OLLamaLLMService(BaseOpenAILLMService):
def __init__(self, *, model: str = "llama2", base_url: str = "http://localhost:11434/v1"):
super().__init__(model=model, base_url=base_url, api_key="ollama")

423
src/pipecat/services/openai.py
View File

@@ -4,57 +4,76 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import aiohttp
import base64
import io
import json
import httpx
from dataclasses import dataclass
from typing import Any, AsyncGenerator, Dict, List, Literal, Optional
from typing import AsyncGenerator, List, Literal
import aiohttp
import httpx
from loguru import logger
from PIL import Image
from pydantic import BaseModel, Field
from pipecat.frames.frames import (
AudioRawFrame,
ErrorFrame,
Frame,
FunctionCallInProgressFrame,
FunctionCallResultFrame,
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMModelUpdateFrame,
LLMUpdateSettingsFrame,
StartInterruptionFrame,
TextFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
TextFrame,
URLImageRawFrame,
UserImageRawFrame,
UserImageRequestFrame,
VisionImageRawFrame,
FunctionCallResultFrame,
FunctionCallInProgressFrame,
StartInterruptionFrame
)
from pipecat.processors.aggregators.llm_response import LLMUserContextAggregator, LLMAssistantContextAggregator
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_response import (
LLMAssistantContextAggregator,
LLMUserContextAggregator,
)
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
OpenAILLMContextFrame
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import (
ImageGenService,
LLMService,
TTSService
)
from pipecat.services.ai_services import ImageGenService, LLMService, TTSService
try:
from openai import AsyncOpenAI, AsyncStream, DefaultAsyncHttpxClient, BadRequestError
from openai import (
NOT_GIVEN,
AsyncOpenAI,
AsyncStream,
BadRequestError,
DefaultAsyncHttpxClient,
)
from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable.")
"In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable."
)
raise Exception(f"Missing module: {e}")
ValidVoice = Literal["alloy", "echo", "fable", "onyx", "nova", "shimmer"]
VALID_VOICES: Dict[str, ValidVoice] = {
"alloy": "alloy",
"echo": "echo",
"fable": "fable",
"onyx": "onyx",
"nova": "nova",
"shimmer": "shimmer",
}
class OpenAIUnhandledFunctionException(Exception):
pass
@@ -70,9 +89,37 @@ class BaseOpenAILLMService(LLMService):
calls from the LLM.
"""
def __init__(self, *, model: str, api_key=None, base_url=None, **kwargs):
class InputParams(BaseModel):
frequency_penalty: Optional[float] = Field(
default_factory=lambda: NOT_GIVEN, ge=-2.0, le=2.0
)
presence_penalty: Optional[float] = Field(
default_factory=lambda: NOT_GIVEN, ge=-2.0, le=2.0
)
seed: Optional[int] = Field(default_factory=lambda: NOT_GIVEN, ge=0)
temperature: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=2.0)
top_p: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0)
extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
def __init__(
self,
*,
model: str,
api_key=None,
base_url=None,
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(**kwargs)
self._model: str = model
self._settings = {
"frequency_penalty": params.frequency_penalty,
"presence_penalty": params.presence_penalty,
"seed": params.seed,
"temperature": params.temperature,
"top_p": params.top_p,
"extra": params.extra if isinstance(params.extra, dict) else {},
}
self.set_model_name(model)
self._client = self.create_client(api_key=api_key, base_url=base_url, **kwargs)
def create_client(self, api_key=None, base_url=None, **kwargs):
@@ -81,30 +128,40 @@ class BaseOpenAILLMService(LLMService):
base_url=base_url,
http_client=DefaultAsyncHttpxClient(
limits=httpx.Limits(
max_keepalive_connections=100,
max_connections=1000,
keepalive_expiry=None)))
max_keepalive_connections=100, max_connections=1000, keepalive_expiry=None
)
),
)
def can_generate_metrics(self) -> bool:
return True
async def get_chat_completions(
self,
context: OpenAILLMContext,
messages: List[ChatCompletionMessageParam]) -> AsyncStream[ChatCompletionChunk]:
chunks = await self._client.chat.completions.create(
model=self._model,
stream=True,
messages=messages,
tools=context.tools,
tool_choice=context.tool_choice,
stream_options={"include_usage": True}
)
self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam]
) -> AsyncStream[ChatCompletionChunk]:
params = {
"model": self.model_name,
"stream": True,
"messages": messages,
"tools": context.tools,
"tool_choice": context.tool_choice,
"stream_options": {"include_usage": True},
"frequency_penalty": self._settings["frequency_penalty"],
"presence_penalty": self._settings["presence_penalty"],
"seed": self._settings["seed"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
}
params.update(self._settings["extra"])
chunks = await self._client.chat.completions.create(**params)
return chunks
async def _stream_chat_completions(
self, context: OpenAILLMContext) -> AsyncStream[ChatCompletionChunk]:
logger.debug(f"Generating chat: {context.get_messages_json()}")
self, context: OpenAILLMContext
) -> AsyncStream[ChatCompletionChunk]:
logger.debug(f"Generating chat: {context.get_messages_for_logging()}")
messages: List[ChatCompletionMessageParam] = context.get_messages()
@@ -115,7 +172,10 @@ class BaseOpenAILLMService(LLMService):
text = message["content"]
message["content"] = [
{"type": "text", "text": text},
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}}
{
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"},
},
]
del message["data"]
del message["mime_type"]
@@ -125,25 +185,27 @@ class BaseOpenAILLMService(LLMService):
return chunks
async def _process_context(self, context: OpenAILLMContext):
functions_list = []
arguments_list = []
tool_id_list = []
func_idx = 0
function_name = ""
arguments = ""
tool_call_id = ""
await self.start_ttfb_metrics()
chunk_stream: AsyncStream[ChatCompletionChunk] = (
await self._stream_chat_completions(context)
chunk_stream: AsyncStream[ChatCompletionChunk] = await self._stream_chat_completions(
context
)
async for chunk in chunk_stream:
if chunk.usage:
tokens = {
"processor": self.name,
"model": self._model,
"prompt_tokens": chunk.usage.prompt_tokens,
"completion_tokens": chunk.usage.completion_tokens,
"total_tokens": chunk.usage.total_tokens
}
tokens = LLMTokenUsage(
prompt_tokens=chunk.usage.prompt_tokens,
completion_tokens=chunk.usage.completion_tokens,
total_tokens=chunk.usage.total_tokens,
)
await self.start_llm_usage_metrics(tokens)
if len(chunk.choices) == 0:
@@ -164,6 +226,14 @@ class BaseOpenAILLMService(LLMService):
# yield a frame containing the function name and the arguments.
tool_call = chunk.choices[0].delta.tool_calls[0]
if tool_call.index != func_idx:
functions_list.append(function_name)
arguments_list.append(arguments)
tool_id_list.append(tool_call_id)
function_name = ""
arguments = ""
tool_call_id = ""
func_idx += 1
if tool_call.function and tool_call.function.name:
function_name += tool_call.function.name
tool_call_id = tool_call.id
@@ -179,26 +249,29 @@ class BaseOpenAILLMService(LLMService):
# the context, and re-prompt to get a chat answer. If we don't have a registered
# handler, raise an exception.
if function_name and arguments:
if self.has_function(function_name):
await self._handle_function_call(context, tool_call_id, function_name, arguments)
else:
raise OpenAIUnhandledFunctionException(
f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function.")
# added to the list as last function name and arguments not added to the list
functions_list.append(function_name)
arguments_list.append(arguments)
tool_id_list.append(tool_call_id)
async def _handle_function_call(
self,
context,
tool_call_id,
function_name,
arguments
):
arguments = json.loads(arguments)
await self.call_function(
context=context,
tool_call_id=tool_call_id,
function_name=function_name,
arguments=arguments
)
total_items = len(functions_list)
for index, (function_name, arguments, tool_id) in enumerate(
zip(functions_list, arguments_list, tool_id_list), start=1
):
if self.has_function(function_name):
run_llm = index == total_items
arguments = json.loads(arguments)
await self.call_function(
context=context,
function_name=function_name,
arguments=arguments,
tool_call_id=tool_id,
run_llm=run_llm,
)
else:
raise OpenAIUnhandledFunctionException(
f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function."
)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -210,9 +283,8 @@ class BaseOpenAILLMService(LLMService):
context = OpenAILLMContext.from_messages(frame.messages)
elif isinstance(frame, VisionImageRawFrame):
context = OpenAILLMContext.from_image_frame(frame)
elif isinstance(frame, LLMModelUpdateFrame):
logger.debug(f"Switching LLM model to: [{frame.model}]")
self._model = frame.model
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)
@@ -226,33 +298,38 @@ class BaseOpenAILLMService(LLMService):
@dataclass
class OpenAIContextAggregatorPair:
_user: 'OpenAIUserContextAggregator'
_assistant: 'OpenAIAssistantContextAggregator'
_user: "OpenAIUserContextAggregator"
_assistant: "OpenAIAssistantContextAggregator"
def user(self) -> 'OpenAIUserContextAggregator':
def user(self) -> "OpenAIUserContextAggregator":
return self._user
def assistant(self) -> 'OpenAIAssistantContextAggregator':
def assistant(self) -> "OpenAIAssistantContextAggregator":
return self._assistant
class OpenAILLMService(BaseOpenAILLMService):
def __init__(self, *, model: str = "gpt-4o", **kwargs):
super().__init__(model=model, **kwargs)
def __init__(
self,
*,
model: str = "gpt-4o",
params: BaseOpenAILLMService.InputParams = BaseOpenAILLMService.InputParams(),
**kwargs,
):
super().__init__(model=model, params=params, **kwargs)
@staticmethod
def create_context_aggregator(context: OpenAILLMContext) -> OpenAIContextAggregatorPair:
def create_context_aggregator(
context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True
) -> OpenAIContextAggregatorPair:
user = OpenAIUserContextAggregator(context)
assistant = OpenAIAssistantContextAggregator(user)
return OpenAIContextAggregatorPair(
_user=user,
_assistant=assistant
assistant = OpenAIAssistantContextAggregator(
user, expect_stripped_words=assistant_expect_stripped_words
)
return OpenAIContextAggregatorPair(_user=user, _assistant=assistant)
class OpenAIImageGenService(ImageGenService):
def __init__(
self,
*,
@@ -262,7 +339,7 @@ class OpenAIImageGenService(ImageGenService):
model: str = "dall-e-3",
):
super().__init__()
self._model = model
self.set_model_name(model)
self._image_size = image_size
self._client = AsyncOpenAI(api_key=api_key)
self._aiohttp_session = aiohttp_session
@@ -271,10 +348,7 @@ class OpenAIImageGenService(ImageGenService):
logger.debug(f"Generating image from prompt: {prompt}")
image = await self._client.images.generate(
prompt=prompt,
model=self._model,
n=1,
size=self._image_size
prompt=prompt, model=self.model_name, n=1, size=self._image_size
)
image_url = image.data[0].url
@@ -304,25 +378,30 @@ class OpenAITTSService(TTSService):
"""
def __init__(
self,
*,
api_key: str | None = None,
voice: Literal["alloy", "echo", "fable", "onyx", "nova", "shimmer"] = "alloy",
model: Literal["tts-1", "tts-1-hd"] = "tts-1",
**kwargs):
super().__init__(**kwargs)
self,
*,
api_key: str | None = None,
voice: str = "alloy",
model: Literal["tts-1", "tts-1-hd"] = "tts-1",
sample_rate: int = 24000,
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
self._voice = voice
self._model = model
self._settings = {
"sample_rate": sample_rate,
}
self.set_model_name(model)
self.set_voice(voice)
self._client = AsyncOpenAI(api_key=api_key)
def can_generate_metrics(self) -> bool:
return True
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice = voice
async def set_model(self, model: str):
logger.debug(f"Switching TTS model to: [{model}]")
self.set_model_name(model)
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
@@ -330,101 +409,167 @@ class OpenAITTSService(TTSService):
await self.start_ttfb_metrics()
async with self._client.audio.speech.with_streaming_response.create(
input=text,
model=self._model,
voice=self._voice,
response_format="pcm",
input=text,
model=self.model_name,
voice=VALID_VOICES[self._voice_id],
response_format="pcm",
) as r:
if r.status_code != 200:
error = await r.text()
logger.error(
f"{self} error getting audio (status: {r.status_code}, error: {error})")
yield ErrorFrame(f"Error getting audio (status: {r.status_code}, error: {error})")
f"{self} error getting audio (status: {r.status_code}, error: {error})"
)
yield ErrorFrame(
f"Error getting audio (status: {r.status_code}, error: {error})"
)
return
await self.start_tts_usage_metrics(text)
await self.push_frame(TTSStartedFrame())
yield TTSStartedFrame()
async for chunk in r.iter_bytes(8192):
if len(chunk) > 0:
await self.stop_ttfb_metrics()
frame = AudioRawFrame(chunk, 24_000, 1)
frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
yield frame
await self.push_frame(TTSStoppedFrame())
yield TTSStoppedFrame()
except BadRequestError as e:
logger.exception(f"{self} error generating TTS: {e}")
# internal use only -- todo: refactor
@dataclass
class OpenAIImageMessageFrame(Frame):
user_image_raw_frame: UserImageRawFrame
text: Optional[str] = None
class OpenAIUserContextAggregator(LLMUserContextAggregator):
def __init__(self, context: OpenAILLMContext):
super().__init__(context=context)
async def process_frame(self, frame, direction):
await super().process_frame(frame, direction)
# Our parent method has already called push_frame(). So we can't interrupt the
# flow here and we don't need to call push_frame() ourselves.
try:
if isinstance(frame, UserImageRequestFrame):
# The LLM sends a UserImageRequestFrame upstream. Cache any context provided with
# that frame so we can use it when we assemble the image message in the assistant
# context aggregator.
if frame.context:
if isinstance(frame.context, str):
self._context._user_image_request_context[frame.user_id] = frame.context
else:
logger.error(
f"Unexpected UserImageRequestFrame context type: {type(frame.context)}"
)
del self._context._user_image_request_context[frame.user_id]
else:
if frame.user_id in self._context._user_image_request_context:
del self._context._user_image_request_context[frame.user_id]
elif isinstance(frame, UserImageRawFrame):
# Push a new AnthropicImageMessageFrame with the text context we cached
# downstream to be handled by our assistant context aggregator. This is
# necessary so that we add the message to the context in the right order.
text = self._context._user_image_request_context.get(frame.user_id) or ""
if text:
del self._context._user_image_request_context[frame.user_id]
frame = OpenAIImageMessageFrame(user_image_raw_frame=frame, text=text)
await self.push_frame(frame)
except Exception as e:
logger.error(f"Error processing frame: {e}")
class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator):
def __init__(self, user_context_aggregator: OpenAIUserContextAggregator):
super().__init__(context=user_context_aggregator._context)
def __init__(self, user_context_aggregator: OpenAIUserContextAggregator, **kwargs):
super().__init__(context=user_context_aggregator._context, **kwargs)
self._user_context_aggregator = user_context_aggregator
self._function_call_in_progress = None
self._function_calls_in_progress = {}
self._function_call_result = None
self._pending_image_frame_message = None
async def process_frame(self, frame, direction):
await super().process_frame(frame, direction)
# See note above about not calling push_frame() here.
if isinstance(frame, StartInterruptionFrame):
self._function_call_in_progress = None
self._function_calls_in_progress.clear()
self._function_call_finished = None
elif isinstance(frame, FunctionCallInProgressFrame):
self._function_call_in_progress = frame
self._function_calls_in_progress[frame.tool_call_id] = frame
elif isinstance(frame, FunctionCallResultFrame):
if self._function_call_in_progress and self._function_call_in_progress.tool_call_id == frame.tool_call_id:
self._function_call_in_progress = None
if frame.tool_call_id in self._function_calls_in_progress:
del self._function_calls_in_progress[frame.tool_call_id]
self._function_call_result = frame
# TODO-CB: Kwin wants us to refactor this out of here but I REFUSE
await self._push_aggregation()
else:
logger.warning(
f"FunctionCallResultFrame tool_call_id does not match FunctionCallInProgressFrame tool_call_id")
self._function_call_in_progress = None
"FunctionCallResultFrame tool_call_id does not match any function call in progress"
)
self._function_call_result = None
elif isinstance(frame, OpenAIImageMessageFrame):
self._pending_image_frame_message = frame
await self._push_aggregation()
async def _push_aggregation(self):
if not (self._aggregation or self._function_call_result):
if not (
self._aggregation or self._function_call_result or self._pending_image_frame_message
):
return
run_llm = False
aggregation = self._aggregation
self._aggregation = ""
self._reset()
try:
if self._function_call_result:
frame = self._function_call_result
self._function_call_result = None
if frame.result:
self._context.add_message({
"role": "assistant",
"tool_calls": [
{
"id": frame.tool_call_id,
"function": {
"name": frame.function_name,
"arguments": json.dumps(frame.arguments)
},
"type": "function"
}
]
})
self._context.add_message({
"role": "tool",
"content": json.dumps(frame.result),
"tool_call_id": frame.tool_call_id
})
run_llm = True
self._context.add_message(
{
"role": "assistant",
"tool_calls": [
{
"id": frame.tool_call_id,
"function": {
"name": frame.function_name,
"arguments": json.dumps(frame.arguments),
},
"type": "function",
}
],
}
)
self._context.add_message(
{
"role": "tool",
"content": json.dumps(frame.result),
"tool_call_id": frame.tool_call_id,
}
)
run_llm = frame.run_llm
else:
self._context.add_message({"role": "assistant", "content": aggregation})
if self._pending_image_frame_message:
frame = self._pending_image_frame_message
self._pending_image_frame_message = None
self._context.add_image_frame_message(
format=frame.user_image_raw_frame.format,
size=frame.user_image_raw_frame.size,
image=frame.user_image_raw_frame.image,
text=frame.text,
)
run_llm = True
if run_llm:
await self._user_context_aggregator.push_context_frame()
frame = OpenAILLMContextFrame(self._context)
await self.push_frame(frame)
except Exception as e:
logger.error(f"Error processing frame: {e}")

45
src/pipecat/services/openpipe.py
View File

@@ -13,33 +13,35 @@ from loguru import logger
try:
from openpipe import AsyncOpenAI as OpenPipeAI, AsyncStream
from openai.types.chat import (ChatCompletionMessageParam, ChatCompletionChunk)
from openai.types.chat import ChatCompletionMessageParam, ChatCompletionChunk
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use OpenPipe, you need to `pip install pipecat-ai[openpipe]`. Also, set `OPENPIPE_API_KEY` and `OPENAI_API_KEY` environment variables.")
"In order to use OpenPipe, you need to `pip install pipecat-ai[openpipe]`. Also, set `OPENPIPE_API_KEY` and `OPENAI_API_KEY` environment variables."
)
raise Exception(f"Missing module: {e}")
class OpenPipeLLMService(BaseOpenAILLMService):
def __init__(
self,
*,
model: str = "gpt-4o",
api_key: str | None = None,
base_url: str | None = None,
openpipe_api_key: str | None = None,
openpipe_base_url: str = "https://app.openpipe.ai/api/v1",
tags: Dict[str, str] | None = None,
**kwargs):
self,
*,
model: str = "gpt-4o",
api_key: str | None = None,
base_url: str | None = None,
openpipe_api_key: str | None = None,
openpipe_base_url: str = "https://app.openpipe.ai/api/v1",
tags: Dict[str, str] | None = None,
**kwargs,
):
super().__init__(
model=model,
api_key=api_key,
base_url=base_url,
openpipe_api_key=openpipe_api_key,
openpipe_base_url=openpipe_base_url,
**kwargs)
**kwargs,
)
self._tags = tags
def create_client(self, api_key=None, base_url=None, **kwargs):
@@ -48,24 +50,17 @@ class OpenPipeLLMService(BaseOpenAILLMService):
client = OpenPipeAI(
api_key=api_key,
base_url=base_url,
openpipe={
"api_key": openpipe_api_key,
"base_url": openpipe_base_url
}
openpipe={"api_key": openpipe_api_key, "base_url": openpipe_base_url},
)
return client
async def get_chat_completions(
self,
context: OpenAILLMContext,
messages: List[ChatCompletionMessageParam]) -> AsyncStream[ChatCompletionChunk]:
self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam]
) -> AsyncStream[ChatCompletionChunk]:
chunks = await self._client.chat.completions.create(
model=self._model,
model=self.model_name,
stream=True,
messages=messages,
openpipe={
"tags": self._tags,
"log_request": True
}
openpipe={"tags": self._tags, "log_request": True},
)
return chunks

57
src/pipecat/services/playht.py
View File

@@ -6,29 +6,35 @@
import io
import struct
from typing import AsyncGenerator
from pipecat.frames.frames import AudioRawFrame, Frame, TTSStartedFrame, TTSStoppedFrame
from pipecat.services.ai_services import TTSService
from loguru import logger
from pipecat.frames.frames import (
Frame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.ai_services import TTSService
try:
from pyht.client import TTSOptions
from pyht.async_client import AsyncClient
from pyht.client import TTSOptions
from pyht.protos.api_pb2 import Format
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use PlayHT, you need to `pip install pipecat-ai[playht]`. Also, set `PLAY_HT_USER_ID` and `PLAY_HT_API_KEY` environment variables.")
"In order to use PlayHT, you need to `pip install pipecat-ai[playht]`. Also, set `PLAY_HT_USER_ID` and `PLAY_HT_API_KEY` environment variables."
)
raise Exception(f"Missing module: {e}")
class PlayHTTTSService(TTSService):
def __init__(self, *, api_key: str, user_id: str, voice_url: str, **kwargs):
super().__init__(**kwargs)
def __init__(
self, *, api_key: str, user_id: str, voice_url: str, sample_rate: int = 16000, **kwargs
):
super().__init__(sample_rate=sample_rate, **kwargs)
self._user_id = user_id
self._speech_key = api_key
@@ -37,11 +43,19 @@ class PlayHTTTSService(TTSService):
user_id=self._user_id,
api_key=self._speech_key,
)
self._settings = {
"sample_rate": sample_rate,
"quality": "higher",
"format": Format.FORMAT_WAV,
"voice_engine": "PlayHT2.0-turbo",
}
self.set_voice(voice_url)
self._options = TTSOptions(
voice=voice_url,
sample_rate=16000,
quality="higher",
format=Format.FORMAT_WAV)
voice=self._voice_id,
sample_rate=self._settings["sample_rate"],
quality=self._settings["quality"],
format=self._settings["format"],
)
def can_generate_metrics(self) -> bool:
return True
@@ -56,13 +70,12 @@ class PlayHTTTSService(TTSService):
await self.start_ttfb_metrics()
playht_gen = self._client.tts(
text,
voice_engine="PlayHT2.0-turbo",
options=self._options)
text, voice_engine=self._settings["voice_engine"], options=self._options
)
await self.start_tts_usage_metrics(text)
await self.push_frame(TTSStartedFrame())
yield TTSStartedFrame()
async for chunk in playht_gen:
# skip the RIFF header.
if in_header:
@@ -72,16 +85,16 @@ class PlayHTTTSService(TTSService):
else:
fh = io.BytesIO(b)
fh.seek(36)
(data, size) = struct.unpack('<4sI', fh.read(8))
while data != b'data':
(data, size) = struct.unpack("<4sI", fh.read(8))
while data != b"data":
fh.read(size)
(data, size) = struct.unpack('<4sI', fh.read(8))
(data, size) = struct.unpack("<4sI", fh.read(8))
in_header = False
else:
if len(chunk):
await self.stop_ttfb_metrics()
frame = AudioRawFrame(chunk, 16000, 1)
frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
yield frame
await self.push_frame(TTSStoppedFrame())
yield TTSStoppedFrame()
except Exception as e:
logger.exception(f"{self} error generating TTS: {e}")

27
src/pipecat/services/to_be_updated/cloudflare_ai_service.py
View File

@@ -12,15 +12,14 @@ class CloudflareAIService(AIService):
self.cloudflare_account_id = os.getenv("CLOUDFLARE_ACCOUNT_ID")
self.cloudflare_api_token = os.getenv("CLOUDFLARE_API_TOKEN")
self.api_base_url = f'https://api.cloudflare.com/client/v4/accounts/{self.cloudflare_account_id}/ai/run/'
self.headers = {"Authorization": f'Bearer {self.cloudflare_api_token}'}
self.api_base_url = (
f"https://api.cloudflare.com/client/v4/accounts/{self.cloudflare_account_id}/ai/run/"
)
self.headers = {"Authorization": f"Bearer {self.cloudflare_api_token}"}
# base endpoint, used by the others
def run(self, model, input):
response = requests.post(
f"{self.api_base_url}{model}",
headers=self.headers,
json=input)
response = requests.post(f"{self.api_base_url}{model}", headers=self.headers, json=input)
return response.json()
# https://developers.cloudflare.com/workers-ai/models/llm/
@@ -28,7 +27,7 @@ class CloudflareAIService(AIService):
input = {
"messages": [
{"role": "system", "content": "You are a friendly assistant"},
{"role": "user", "content": sentence}
{"role": "user", "content": sentence},
]
}
@@ -36,16 +35,14 @@ class CloudflareAIService(AIService):
# https://developers.cloudflare.com/workers-ai/models/translation/
def run_text_translation(self, sentence, source_language, target_language):
return self.run('@cf/meta/m2m100-1.2b', {
"text": sentence,
"source_lang": source_language,
"target_lang": target_language
})
return self.run(
"@cf/meta/m2m100-1.2b",
{"text": sentence, "source_lang": source_language, "target_lang": target_language},
)
# https://developers.cloudflare.com/workers-ai/models/sentiment-analysis/
def run_text_sentiment(self, sentence):
return self.run("@cf/huggingface/distilbert-sst-2-int8",
{"text": sentence})
return self.run("@cf/huggingface/distilbert-sst-2-int8", {"text": sentence})
# https://developers.cloudflare.com/workers-ai/models/image-classification/
def run_image_classification(self, image_url):
@@ -65,7 +62,7 @@ class CloudflareAIService(AIService):
models = {
"small": "@cf/baai/bge-small-en-v1.5", # 384 output dimensions
"medium": "@cf/baai/bge-base-en-v1.5", # 768 output dimensions
"large": "@cf/baai/bge-large-en-v1.5" # 1024 output dimensions
"large": "@cf/baai/bge-large-en-v1.5", # 1024 output dimensions
}
return self.run(models[size], {"text": texts})

8
src/pipecat/services/to_be_updated/google_ai_service.py
View File

@@ -18,14 +18,12 @@ class GoogleAIService(AIService):
)
self.audio_config = texttospeech.AudioConfig(
audio_encoding=texttospeech.AudioEncoding.LINEAR16,
sample_rate_hertz=16000
audio_encoding=texttospeech.AudioEncoding.LINEAR16, sample_rate_hertz=16000
)
def run_tts(self, sentence):
synthesis_input = texttospeech.SynthesisInput(text=sentence.strip())
result = self.client.synthesize_speech(
input=synthesis_input,
voice=self.voice,
audio_config=self.audio_config)
input=synthesis_input, voice=self.voice, audio_config=self.audio_config
)
return result

4
src/pipecat/services/to_be_updated/huggingface_ai_service.py
View File

@@ -19,8 +19,8 @@ class HuggingFaceAIService(AIService):
# models use 2-character language codes**)
def run_text_translation(self, sentence, source_language, target_language):
translator = pipeline(
f"translation",
model=f"Helsinki-NLP/opus-mt-{source_language}-{target_language}")
f"translation", model=f"Helsinki-NLP/opus-mt-{source_language}-{target_language}"
)
return translator(sentence)[0]["translation_text"]

343
src/pipecat/services/together.py
View File

@@ -4,312 +4,73 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import base64
import json
import io
import copy
from typing import List, Optional
from dataclasses import dataclass
from asyncio import CancelledError
import re
import uuid
from pipecat.frames.frames import (
Frame,
LLMModelUpdateFrame,
TextFrame,
VisionImageRawFrame,
UserImageRequestFrame,
UserImageRawFrame,
LLMMessagesFrame,
LLMFullResponseStartFrame,
LLMFullResponseEndFrame,
FunctionCallResultFrame,
FunctionCallInProgressFrame,
StartInterruptionFrame
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import LLMService
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext, OpenAILLMContextFrame
from pipecat.processors.aggregators.llm_response import LLMUserContextAggregator, LLMAssistantContextAggregator
from typing import Any, Dict, Optional
import httpx
from loguru import logger
from pydantic import BaseModel, Field
from pipecat.services.openai import OpenAILLMService
try:
from together import AsyncTogether
# Together.ai is recommending OpenAI-compatible function calling, so we've switched over
# to using the OpenAI client library here rather than the Together Python client library.
from openai import AsyncOpenAI, DefaultAsyncHttpxClient
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Together.ai, you need to `pip install pipecat-ai[together]`. Also, set `TOGETHER_API_KEY` environment variable.")
"In order to use Together.ai, you need to `pip install pipecat-ai[together]`. Also, set `TOGETHER_API_KEY` environment variable."
)
raise Exception(f"Missing module: {e}")
@dataclass
class TogetherContextAggregatorPair:
_user: 'TogetherUserContextAggregator'
_assistant: 'TogetherAssistantContextAggregator'
class TogetherLLMService(OpenAILLMService):
"""This class implements inference with Together's Llama 3.1 models"""
def user(self) -> 'TogetherUserContextAggregator':
return self._user
def assistant(self) -> 'TogetherAssistantContextAggregator':
return self._assistant
class TogetherLLMService(LLMService):
"""This class implements inference with Together's Llama 3.1 models
"""
class InputParams(BaseModel):
frequency_penalty: Optional[float] = Field(default=None, ge=-2.0, le=2.0)
max_tokens: Optional[int] = Field(default=4096, ge=1)
presence_penalty: Optional[float] = Field(default=None, ge=-2.0, le=2.0)
temperature: Optional[float] = Field(default=None, ge=0.0, le=1.0)
# Note: top_k is currently not supported by the OpenAI client library,
# so top_k is ignore right now.
top_k: Optional[int] = Field(default=None, ge=0)
top_p: Optional[float] = Field(default=None, ge=0.0, le=1.0)
extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
seed: Optional[int] = Field(default=None)
def __init__(
self,
*,
api_key: str,
model: str = "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo",
max_tokens: int = 4096,
**kwargs):
super().__init__(**kwargs)
self._client = AsyncTogether(api_key=api_key)
self._model = model
self._max_tokens = max_tokens
self,
*,
api_key: str,
base_url: str = "https://api.together.xyz/v1",
model: str = "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo",
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(api_key=api_key, base_url=base_url, model=model, params=params, **kwargs)
self.set_model_name(model)
self._settings = {
"max_tokens": params.max_tokens,
"frequency_penalty": params.frequency_penalty,
"presence_penalty": params.presence_penalty,
"seed": params.seed,
"temperature": params.temperature,
"top_p": params.top_p,
"extra": params.extra if isinstance(params.extra, dict) else {},
}
def can_generate_metrics(self) -> bool:
return True
@staticmethod
def create_context_aggregator(context: OpenAILLMContext) -> TogetherContextAggregatorPair:
user = TogetherUserContextAggregator(context)
assistant = TogetherAssistantContextAggregator(user)
return TogetherContextAggregatorPair(
_user=user,
_assistant=assistant
def create_client(self, api_key=None, base_url=None, **kwargs):
logger.debug(f"Creating Together.ai client with api {base_url}")
return AsyncOpenAI(
api_key=api_key,
base_url=base_url,
http_client=DefaultAsyncHttpxClient(
limits=httpx.Limits(
max_keepalive_connections=100, max_connections=1000, keepalive_expiry=None
)
),
)
async def _process_context(self, context: OpenAILLMContext):
try:
await self.push_frame(LLMFullResponseStartFrame())
await self.start_processing_metrics()
logger.debug(f"Generating chat: {context.get_messages_for_logging()}")
await self.start_ttfb_metrics()
stream = await self._client.chat.completions.create(
messages=context.messages,
model=self._model,
max_tokens=self._max_tokens,
stream=True,
)
# Function calling
got_first_chunk = False
accumulating_function_call = False
function_call_accumulator = ""
async for chunk in stream:
# logger.debug(f"Together LLM event: {chunk}")
if chunk.usage:
tokens = {
"processor": self.name,
"model": self._model,
"prompt_tokens": chunk.usage.prompt_tokens,
"completion_tokens": chunk.usage.completion_tokens,
"total_tokens": chunk.usage.total_tokens
}
await self.start_llm_usage_metrics(tokens)
if len(chunk.choices) == 0:
continue
if not got_first_chunk:
await self.stop_ttfb_metrics()
if chunk.choices[0].delta.content:
got_first_chunk = True
if chunk.choices[0].delta.content[0] == "<":
accumulating_function_call = True
if chunk.choices[0].delta.content:
if accumulating_function_call:
function_call_accumulator += chunk.choices[0].delta.content
else:
await self.push_frame(TextFrame(chunk.choices[0].delta.content))
if chunk.choices[0].finish_reason == 'eos' and accumulating_function_call:
await self._extract_function_call(context, function_call_accumulator)
except CancelledError as e:
# todo: implement token counting estimates for use when the user interrupts a long generation
# we do this in the anthropic.py service
raise
except Exception as e:
logger.exception(f"{self} exception: {e}")
finally:
await self.stop_processing_metrics()
await self.push_frame(LLMFullResponseEndFrame())
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
context = None
if isinstance(frame, OpenAILLMContextFrame):
context = frame.context
elif isinstance(frame, LLMMessagesFrame):
context = TogetherLLMContext.from_messages(frame.messages)
elif isinstance(frame, LLMModelUpdateFrame):
logger.debug(f"Switching LLM model to: [{frame.model}]")
self._model = frame.model
else:
await self.push_frame(frame, direction)
if context:
await self._process_context(context)
async def _extract_function_call(self, context, function_call_accumulator):
context.add_message({"role": "assistant", "content": function_call_accumulator})
function_regex = r"<function=(\w+)>(.*?)</function>"
match = re.search(function_regex, function_call_accumulator)
if match:
function_name, args_string = match.groups()
try:
arguments = json.loads(args_string)
await self.call_function(context=context,
tool_call_id=str(uuid.uuid4()),
function_name=function_name,
arguments=arguments)
return
except json.JSONDecodeError as error:
# We get here if the LLM returns a function call with invalid JSON arguments. This could happen
# because of LLM non-determinism, or maybe more often because of user error in the prompt.
# Should we do anything more than log a warning?
logger.debug(f"Error parsing function arguments: {error}")
class TogetherLLMContext(OpenAILLMContext):
def __init__(
self,
messages: list[dict] | None = None,
):
super().__init__(messages=messages)
@classmethod
def from_openai_context(cls, openai_context: OpenAILLMContext):
self = cls(
messages=openai_context.messages,
)
return self
@classmethod
def from_messages(cls, messages: List[dict]) -> "TogetherLLMContext":
return cls(messages=messages)
def add_message(self, message):
try:
self.messages.append(message)
except Exception as e:
logger.error(f"Error adding message: {e}")
def get_messages_for_logging(self) -> str:
return json.dumps(self.messages)
class TogetherUserContextAggregator(LLMUserContextAggregator):
def __init__(self, context: OpenAILLMContext | TogetherLLMContext):
super().__init__(context=context)
if isinstance(context, OpenAILLMContext):
self._context = TogetherLLMContext.from_openai_context(context)
async def push_messages_frame(self):
frame = OpenAILLMContextFrame(self._context)
await self.push_frame(frame)
async def process_frame(self, frame, direction):
await super().process_frame(frame, direction)
# Our parent method has already called push_frame(). So we can't interrupt the
# flow here and we don't need to call push_frame() ourselves. Possibly something
# to talk through (tagging @aleix). At some point we might need to refactor these
# context aggregators.
try:
if isinstance(frame, UserImageRequestFrame):
# The LLM sends a UserImageRequestFrame upstream. Cache any context provided with
# that frame so we can use it when we assemble the image message in the assistant
# context aggregator.
if (frame.context):
if isinstance(frame.context, str):
self._context._user_image_request_context[frame.user_id] = frame.context
else:
logger.error(
f"Unexpected UserImageRequestFrame context type: {type(frame.context)}")
del self._context._user_image_request_context[frame.user_id]
else:
if frame.user_id in self._context._user_image_request_context:
del self._context._user_image_request_context[frame.user_id]
except Exception as e:
logger.error(f"Error processing frame: {e}")
#
# Claude returns a text content block along with a tool use content block. This works quite nicely
# with streaming. We get the text first, so we can start streaming it right away. Then we get the
# tool_use block. While the text is streaming to TTS and the transport, we can run the tool call.
#
# But Claude is verbose. It would be nice to come up with prompt language that suppresses Claude's
# chattiness about it's tool thinking.
#
class TogetherAssistantContextAggregator(LLMAssistantContextAggregator):
def __init__(self, user_context_aggregator: TogetherUserContextAggregator):
super().__init__(context=user_context_aggregator._context)
self._user_context_aggregator = user_context_aggregator
self._function_call_in_progress = None
self._function_call_result = None
async def process_frame(self, frame, direction):
await super().process_frame(frame, direction)
# See note above about not calling push_frame() here.
if isinstance(frame, StartInterruptionFrame):
self._function_call_in_progress = None
self._function_call_finished = None
elif isinstance(frame, FunctionCallInProgressFrame):
self._function_call_in_progress = frame
elif isinstance(frame, FunctionCallResultFrame):
if self._function_call_in_progress and self._function_call_in_progress.tool_call_id == frame.tool_call_id:
self._function_call_in_progress = None
self._function_call_result = frame
await self._push_aggregation()
else:
logger.warning(
f"FunctionCallResultFrame tool_call_id does not match FunctionCallInProgressFrame tool_call_id")
self._function_call_in_progress = None
self._function_call_result = None
def add_message(self, message):
self._user_context_aggregator.add_message(message)
async def _push_aggregation(self):
if not (self._aggregation or self._function_call_result):
return
run_llm = False
aggregation = self._aggregation
self._aggregation = ""
try:
if self._function_call_result:
frame = self._function_call_result
self._function_call_result = None
self._context.add_message({
"role": "tool",
# Together expects the content here to be a string, so stringify it
"content": str(frame.result)
})
run_llm = True
else:
self._context.add_message({"role": "assistant", "content": aggregation})
if run_llm:
await self._user_context_aggregator.push_messages_frame()
except Exception as e:
logger.error(f"Error processing frame: {e}")

28
src/pipecat/services/whisper.py
View File

@@ -23,13 +23,13 @@ try:
from faster_whisper import WhisperModel
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Whisper, you need to `pip install pipecat-ai[whisper]`.")
logger.error("In order to use Whisper, you need to `pip install pipecat-ai[whisper]`.")
raise Exception(f"Missing module: {e}")
class Model(Enum):
"""Class of basic Whisper model selection options"""
TINY = "tiny"
BASE = "base"
MEDIUM = "medium"
@@ -41,18 +41,19 @@ class Model(Enum):
class WhisperSTTService(SegmentedSTTService):
"""Class to transcribe audio with a locally-downloaded Whisper model"""
def __init__(self,
*,
model: str | Model = Model.DISTIL_MEDIUM_EN,
device: str = "auto",
compute_type: str = "default",
no_speech_prob: float = 0.4,
**kwargs):
def __init__(
self,
*,
model: str | Model = Model.DISTIL_MEDIUM_EN,
device: str = "auto",
compute_type: str = "default",
no_speech_prob: float = 0.4,
**kwargs,
):
super().__init__(**kwargs)
self._device: str = device
self._compute_type = compute_type
self._model_name: str | Model = model
self.set_model_name(model if isinstance(model, str) else model.value)
self._no_speech_prob = no_speech_prob
self._model: WhisperModel | None = None
self._load()
@@ -65,9 +66,8 @@ class WhisperSTTService(SegmentedSTTService):
this model is being run, it will take time to download."""
logger.debug("Loading Whisper model...")
self._model = WhisperModel(
self._model_name.value if isinstance(self._model_name, Enum) else self._model_name,
device=self._device,
compute_type=self._compute_type)
self.model_name, device=self._device, compute_type=self._compute_type
)
logger.debug("Loaded Whisper model")
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:

118
src/pipecat/services/xtts.py
View File

@@ -4,22 +4,22 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import aiohttp
from typing import Any, AsyncGenerator, Dict
import aiohttp
import numpy as np
from loguru import logger
from pipecat.frames.frames import (
AudioRawFrame,
ErrorFrame,
Frame,
StartFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame)
TTSStoppedFrame,
)
from pipecat.services.ai_services import TTSService
from loguru import logger
import numpy as np
from pipecat.transcriptions.language import Language
try:
import resampy
@@ -37,21 +37,67 @@ except ModuleNotFoundError as e:
# https://github.com/coqui-ai/xtts-streaming-server
class XTTSService(TTSService):
def language_to_xtts_language(language: Language) -> str | None:
match language:
case Language.CS:
return "cs"
case Language.DE:
return "de"
case (
Language.EN
| Language.EN_US
| Language.EN_AU
| Language.EN_GB
| Language.EN_NZ
| Language.EN_IN
):
return "en"
case Language.ES:
return "es"
case Language.FR:
return "fr"
case Language.HI:
return "hi"
case Language.HU:
return "hu"
case Language.IT:
return "it"
case Language.JA:
return "ja"
case Language.KO:
return "ko"
case Language.NL:
return "nl"
case Language.PL:
return "pl"
case Language.PT | Language.PT_BR:
return "pt"
case Language.RU:
return "ru"
case Language.TR:
return "tr"
case Language.ZH:
return "zh-cn"
return None
class XTTSService(TTSService):
def __init__(
self,
*,
voice_id: str,
language: str,
base_url: str,
aiohttp_session: aiohttp.ClientSession,
**kwargs):
self,
*,
voice_id: str,
language: Language,
base_url: str,
aiohttp_session: aiohttp.ClientSession,
**kwargs,
):
super().__init__(**kwargs)
self._voice_id = voice_id
self._language = language
self._base_url = base_url
self._settings = {
"language": language,
"base_url": base_url,
}
self.set_voice(voice_id)
self._studio_speakers: Dict[str, Any] | None = None
self._aiohttp_session = aiohttp_session
@@ -60,20 +106,20 @@ class XTTSService(TTSService):
async def start(self, frame: StartFrame):
await super().start(frame)
async with self._aiohttp_session.get(self._base_url + "/studio_speakers") as r:
async with self._aiohttp_session.get(self._settings["base_url"] + "/studio_speakers") as r:
if r.status != 200:
text = await r.text()
logger.error(
f"{self} error getting studio speakers (status: {r.status}, error: {text})")
f"{self} error getting studio speakers (status: {r.status}, error: {text})"
)
await self.push_error(
ErrorFrame(f"Error error getting studio speakers (status: {r.status}, error: {text})"))
ErrorFrame(
f"Error error getting studio speakers (status: {r.status}, error: {text})"
)
)
return
self._studio_speakers = await r.json()
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice_id = voice
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
@@ -83,11 +129,13 @@ class XTTSService(TTSService):
embeddings = self._studio_speakers[self._voice_id]
url = self._base_url + "/tts_stream"
url = self._settings["base_url"] + "/tts_stream"
language = language_to_xtts_language(self._settings["language"])
payload = {
"text": text.replace('.', '').replace('*', ''),
"language": self._language,
"text": text.replace(".", "").replace("*", ""),
"language": language,
"speaker_embedding": embeddings["speaker_embedding"],
"gpt_cond_latent": embeddings["gpt_cond_latent"],
"add_wav_header": False,
@@ -105,7 +153,7 @@ class XTTSService(TTSService):
await self.start_tts_usage_metrics(text)
await self.push_frame(TTSStartedFrame())
yield TTSStartedFrame()
buffer = bytearray()
async for chunk in r.content.iter_chunked(1024):
@@ -115,7 +163,9 @@ class XTTSService(TTSService):
buffer.extend(chunk)
# Check if buffer has enough data for processing
while len(buffer) >= 48000: # Assuming at least 0.5 seconds of audio data at 24000 Hz
while (
len(buffer) >= 48000
): # Assuming at least 0.5 seconds of audio data at 24000 Hz
# Process the buffer up to a safe size for resampling
process_data = buffer[:48000]
# Remove processed data from buffer
@@ -128,7 +178,7 @@ class XTTSService(TTSService):
# Convert the numpy array back to bytes
resampled_audio_bytes = resampled_audio.astype(np.int16).tobytes()
# Create the frame with the resampled audio
frame = AudioRawFrame(resampled_audio_bytes, 16000, 1)
frame = TTSAudioRawFrame(resampled_audio_bytes, 16000, 1)
yield frame
# Process any remaining data in the buffer
@@ -136,7 +186,7 @@ class XTTSService(TTSService):
audio_np = np.frombuffer(buffer, dtype=np.int16)
resampled_audio = resampy.resample(audio_np, 24000, 16000)
resampled_audio_bytes = resampled_audio.astype(np.int16).tobytes()
frame = AudioRawFrame(resampled_audio_bytes, 16000, 1)
frame = TTSAudioRawFrame(resampled_audio_bytes, 16000, 1)
yield frame
await self.push_frame(TTSStoppedFrame())
yield TTSStoppedFrame()

0
src/pipecat/transcriptions/__init__.py Normal file
View File

87
src/pipecat/transcriptions/language.py
View File

@@ -9,6 +9,7 @@ import sys
from enum import Enum
if sys.version_info < (3, 11):
class StrEnum(str, Enum):
def __new__(cls, value):
obj = str.__new__(cls, value)
@@ -19,46 +20,46 @@ else:
class Language(StrEnum):
BG = "bg" # Bulgarian
CA = "ca" # Catalan
ZH = "zh" # Chinese simplified
ZH_TW = "zh-TW" # Chinese traditional
CS = "cs" # Czech
DA = "da" # Danish
NL = "nl" # Dutch
EN = "en" # English
EN_US = "en-US" # English (USA)
EN_AU = "en-AU" # English (Australia)
EN_GB = "en-GB" # English (Great Britain)
EN_NZ = "en-NZ" # English (New Zealand)
EN_IN = "en-IN" # English (India)
ET = "et" # Estonian
FI = "fi" # Finnish
NL_BE = "nl-BE" # Flemmish
FR = "fr" # French
FR_CA = "fr-CA" # French (Canada)
DE = "de" # German
DE_CH = "de-CH" # German (Switzerland)
EL = "el" # Greek
HI = "hi" # Hindi
HU = "hu" # Hungarian
ID = "id" # Indonesian
IT = "it" # Italian
JA = "ja" # Japanese
KO = "ko" # Korean
LV = "lv" # Latvian
LT = "lt" # Lithuanian
MS = "ms" # Malay
NO = "no" # Norwegian
PL = "pl" # Polish
PT = "pt" # Portuguese
PT_BR = "pt-BR" # Portuguese (Brazil)
RO = "ro" # Romanian
RU = "ru" # Russian
SK = "sk" # Slovak
ES = "es" # Spanish
SV = "sv" # Swedish
TH = "th" # Thai
TR = "tr" # Turkish
UK = "uk" # Ukrainian
VI = "vi" # Vietnamese
BG = "bg" # Bulgarian
CA = "ca" # Catalan
ZH = "zh" # Chinese simplified
ZH_TW = "zh-TW" # Chinese traditional
CS = "cs" # Czech
DA = "da" # Danish
NL = "nl" # Dutch
EN = "en" # English
EN_US = "en-US" # English (USA)
EN_AU = "en-AU" # English (Australia)
EN_GB = "en-GB" # English (Great Britain)
EN_NZ = "en-NZ" # English (New Zealand)
EN_IN = "en-IN" # English (India)
ET = "et" # Estonian
FI = "fi" # Finnish
NL_BE = "nl-BE" # Flemmish
FR = "fr" # French
FR_CA = "fr-CA" # French (Canada)
DE = "de" # German
DE_CH = "de-CH" # German (Switzerland)
EL = "el" # Greek
HI = "hi" # Hindi
HU = "hu" # Hungarian
ID = "id" # Indonesian
IT = "it" # Italian
JA = "ja" # Japanese
KO = "ko" # Korean
LV = "lv" # Latvian
LT = "lt" # Lithuanian
MS = "ms" # Malay
NO = "no" # Norwegian
PL = "pl" # Polish
PT = "pt" # Portuguese
PT_BR = "pt-BR" # Portuguese (Brazil)
RO = "ro" # Romanian
RU = "ru" # Russian
SK = "sk" # Slovak
ES = "es" # Spanish
SV = "sv" # Swedish
TH = "th" # Thai
TR = "tr" # Turkish
UK = "uk" # Ukrainian
VI = "vi" # Vietnamese

139
src/pipecat/transports/base_input.py
View File

@@ -5,31 +5,30 @@
#
import asyncio
from concurrent.futures import ThreadPoolExecutor
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from loguru import logger
from pipecat.frames.frames import (
AudioRawFrame,
BotInterruptionFrame,
CancelFrame,
StartFrame,
EndFrame,
Frame,
InputAudioRawFrame,
StartFrame,
StartInterruptionFrame,
StopInterruptionFrame,
SystemFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
VADParamsUpdateFrame)
VADParamsUpdateFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.transports.base_transport import TransportParams
from pipecat.vad.vad_analyzer import VADAnalyzer, VADState
from loguru import logger
class BaseInputTransport(FrameProcessor):
def __init__(self, params: TransportParams, **kwargs):
super().__init__(**kwargs)
@@ -37,9 +36,9 @@ class BaseInputTransport(FrameProcessor):
self._executor = ThreadPoolExecutor(max_workers=5)
# Create push frame task. This is the task that will push frames in
# order. We also guarantee that all frames are pushed in the same task.
self._create_push_task()
# Task to process incoming audio (VAD) and push audio frames downstream
# if passthrough is enabled.
self._audio_task = None
async def start(self, frame: StartFrame):
# Create audio input queue and task if needed.
@@ -49,28 +48,22 @@ class BaseInputTransport(FrameProcessor):
async def stop(self, frame: EndFrame):
# Cancel and wait for the audio input task to finish.
if self._params.audio_in_enabled or self._params.vad_enabled:
if self._audio_task and (self._params.audio_in_enabled or self._params.vad_enabled):
self._audio_task.cancel()
await self._audio_task
# Wait for the push frame task to finish. It will finish when the
# EndFrame is actually processed.
await self._push_frame_task
self._audio_task = None
async def cancel(self, frame: CancelFrame):
# Cancel all the tasks and wait for them to finish.
if self._params.audio_in_enabled or self._params.vad_enabled:
# Cancel and wait for the audio input task to finish.
if self._audio_task and (self._params.audio_in_enabled or self._params.vad_enabled):
self._audio_task.cancel()
await self._audio_task
self._push_frame_task.cancel()
await self._push_frame_task
self._audio_task = None
def vad_analyzer(self) -> VADAnalyzer | None:
return self._params.vad_analyzer
async def push_audio_frame(self, frame: AudioRawFrame):
async def push_audio_frame(self, frame: InputAudioRawFrame):
if self._params.audio_in_enabled or self._params.vad_enabled:
await self._audio_in_queue.put(frame)
@@ -82,28 +75,26 @@ class BaseInputTransport(FrameProcessor):
await super().process_frame(frame, direction)
# Specific system frames
if isinstance(frame, CancelFrame):
if isinstance(frame, StartFrame):
# Push StartFrame before start(), because we want StartFrame to be
# processed by every processor before any other frame is processed.
await self.push_frame(frame, direction)
await self.start(frame)
elif isinstance(frame, CancelFrame):
await self.cancel(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, BotInterruptionFrame):
await self._handle_interruptions(frame, False)
elif isinstance(frame, StartInterruptionFrame):
logger.debug("Bot interruption")
await self._start_interruption()
elif isinstance(frame, StopInterruptionFrame):
await self._stop_interruption()
await self.push_frame(StartInterruptionFrame())
# All other system frames
elif isinstance(frame, SystemFrame):
await self.push_frame(frame, direction)
# Control frames
elif isinstance(frame, StartFrame):
# Push StartFrame before start(), because we want StartFrame to be
# processed by every processor before any other frame is processed.
await self._internal_push_frame(frame, direction)
await self.start(frame)
elif isinstance(frame, EndFrame):
# Push EndFrame before stop(), because stop() waits on the task to
# finish and the task finishes when EndFrame is processed.
await self._internal_push_frame(frame, direction)
await self.push_frame(frame, direction)
await self.stop(frame)
elif isinstance(frame, VADParamsUpdateFrame):
vad_analyzer = self.vad_analyzer()
@@ -111,73 +102,28 @@ class BaseInputTransport(FrameProcessor):
vad_analyzer.set_params(frame.params)
# Other frames
else:
await self._internal_push_frame(frame, direction)
#
# Push frames task
#
def _create_push_task(self):
loop = self.get_event_loop()
self._push_queue = asyncio.Queue()
self._push_frame_task = loop.create_task(self._push_frame_task_handler())
async def _internal_push_frame(
self,
frame: Frame | None,
direction: FrameDirection | None = FrameDirection.DOWNSTREAM):
await self._push_queue.put((frame, direction))
async def _push_frame_task_handler(self):
running = True
while running:
try:
(frame, direction) = await self._push_queue.get()
await self.push_frame(frame, direction)
running = not isinstance(frame, EndFrame)
self._push_queue.task_done()
except asyncio.CancelledError:
break
await self.push_frame(frame, direction)
#
# Handle interruptions
#
async def _start_interruption(self):
if not self.interruptions_allowed:
return
# Cancel the task. This will stop pushing frames downstream.
self._push_frame_task.cancel()
await self._push_frame_task
# Push an out-of-band frame (i.e. not using the ordered push
# frame task) to stop everything, specially at the output
# transport.
await self.push_frame(StartInterruptionFrame())
# Create a new queue and task.
self._create_push_task()
async def _stop_interruption(self):
if not self.interruptions_allowed:
return
await self.push_frame(StopInterruptionFrame())
async def _handle_interruptions(self, frame: Frame, push_frame: bool):
async def _handle_interruptions(self, frame: Frame):
if self.interruptions_allowed:
# Make sure we notify about interruptions quickly out-of-band
if isinstance(frame, BotInterruptionFrame):
logger.debug("Bot interruption")
await self._start_interruption()
elif isinstance(frame, UserStartedSpeakingFrame):
# Make sure we notify about interruptions quickly out-of-band.
if isinstance(frame, UserStartedSpeakingFrame):
logger.debug("User started speaking")
await self._start_interruption()
# Push an out-of-band frame (i.e. not using the ordered push
# frame task) to stop everything, specially at the output
# transport.
await self.push_frame(StartInterruptionFrame())
elif isinstance(frame, UserStoppedSpeakingFrame):
logger.debug("User stopped speaking")
await self._stop_interruption()
await self.push_frame(StopInterruptionFrame())
if push_frame:
await self._internal_push_frame(frame)
await self.push_frame(frame)
#
# Audio input
@@ -188,12 +134,17 @@ class BaseInputTransport(FrameProcessor):
vad_analyzer = self.vad_analyzer()
if vad_analyzer:
state = await self.get_event_loop().run_in_executor(
self._executor, vad_analyzer.analyze_audio, audio_frames)
self._executor, vad_analyzer.analyze_audio, audio_frames
)
return state
async def _handle_vad(self, audio_frames: bytes, vad_state: VADState):
new_vad_state = await self._vad_analyze(audio_frames)
if new_vad_state != vad_state and new_vad_state != VADState.STARTING and new_vad_state != VADState.STOPPING:
if (
new_vad_state != vad_state
and new_vad_state != VADState.STARTING
and new_vad_state != VADState.STOPPING
):
frame = None
if new_vad_state == VADState.SPEAKING:
frame = UserStartedSpeakingFrame()
@@ -201,7 +152,7 @@ class BaseInputTransport(FrameProcessor):
frame = UserStoppedSpeakingFrame()
if frame:
await self._handle_interruptions(frame, True)
await self._handle_interruptions(frame)
vad_state = new_vad_state
return vad_state
@@ -210,7 +161,7 @@ class BaseInputTransport(FrameProcessor):
vad_state: VADState = VADState.QUIET
while True:
try:
frame: AudioRawFrame = await self._audio_in_queue.get()
frame: InputAudioRawFrame = await self._audio_in_queue.get()
audio_passthrough = True
@@ -222,7 +173,7 @@ class BaseInputTransport(FrameProcessor):
# Push audio downstream if passthrough.
if audio_passthrough:
await self._internal_push_frame(frame)
await self.push_frame(frame)
self._audio_in_queue.task_done()
except asyncio.CancelledError:

277
src/pipecat/transports/base_output.py
View File

@@ -8,49 +8,66 @@
import asyncio
import itertools
import time
import sys
from PIL import Image
from typing import List
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.frames.frames import (
AudioRawFrame,
BotSpeakingFrame,
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
CancelFrame,
MetricsFrame,
OutputAudioRawFrame,
OutputImageRawFrame,
SpriteFrame,
StartFrame,
EndFrame,
Frame,
ImageRawFrame,
StartInterruptionFrame,
StopInterruptionFrame,
SystemFrame,
TTSStartedFrame,
TTSStoppedFrame,
TransportMessageFrame)
TextFrame,
TransportMessageFrame,
)
from pipecat.transports.base_transport import TransportParams
from loguru import logger
from pipecat.utils.time import nanoseconds_to_seconds
class BaseOutputTransport(FrameProcessor):
def __init__(self, params: TransportParams, **kwargs):
super().__init__(**kwargs)
self._params = params
# Task to process incoming frames so we don't block upstream elements.
self._sink_task = None
# Task to process incoming frames using a clock.
self._sink_clock_task = None
# Task to write/send audio frames.
self._audio_out_task = None
# Task to write/send image frames.
self._camera_out_task = None
# These are the images that we should send to the camera at our desired
# framerate.
self._camera_images = None
# We will write 20ms audio at a time. If we receive long audio frames we
# will chunk them. This will help with interruption handling.
audio_bytes_10ms = int(self._params.audio_out_sample_rate / 100) * \
self._params.audio_out_channels * 2
audio_bytes_10ms = (
int(self._params.audio_out_sample_rate / 100) * self._params.audio_out_channels * 2
)
self._audio_chunk_size = audio_bytes_10ms * 2
self._audio_buffer = bytearray()
@@ -64,50 +81,72 @@ class BaseOutputTransport(FrameProcessor):
# Create sink frame task. This is the task that will actually write
# audio or video frames. We write audio/video in a task so we can keep
# generating frames upstream while, for example, the audio is playing.
self._create_sink_task()
# Create push frame task. This is the task that will push frames in
# order. We also guarantee that all frames are pushed in the same task.
self._create_push_task()
self._create_sink_tasks()
async def start(self, frame: StartFrame):
# Create camera output queue and task if needed.
if self._params.camera_out_enabled:
self._camera_out_queue = asyncio.Queue()
self._camera_out_task = self.get_event_loop().create_task(self._camera_out_task_handler())
self._camera_out_task = self.get_event_loop().create_task(
self._camera_out_task_handler()
)
# Create audio output queue and task if needed.
if self._params.audio_out_enabled and self._params.audio_out_is_live:
self._audio_out_queue = asyncio.Queue()
self._audio_out_task = self.get_event_loop().create_task(self._audio_out_task_handler())
async def stop(self, frame: EndFrame):
# At this point we have enqueued an EndFrame and we need to wait for
# that EndFrame to be processed by the sink tasks. We also need to wait
# for these tasks before cancelling the camera and audio tasks below
# because they might be still rendering.
if self._sink_task:
await self._sink_task
if self._sink_clock_task:
await self._sink_clock_task
# Cancel and wait for the camera output task to finish.
if self._params.camera_out_enabled:
if self._camera_out_task and self._params.camera_out_enabled:
self._camera_out_task.cancel()
await self._camera_out_task
self._camera_out_task = None
# Cancel and wait for the audio output task to finish.
if self._params.audio_out_enabled and self._params.audio_out_is_live:
if (
self._audio_out_task
and self._params.audio_out_enabled
and self._params.audio_out_is_live
):
self._audio_out_task.cancel()
await self._audio_out_task
# Wait for the push frame and sink tasks to finish. They will finish when
# the EndFrame is actually processed.
await self._push_frame_task
await self._sink_task
self._audio_out_task = None
async def cancel(self, frame: CancelFrame):
# Cancel all the tasks and wait for them to finish.
# Since we are cancelling everything it doesn't matter if we cancel sink
# tasks first or not.
if self._sink_task:
self._sink_task.cancel()
await self._sink_task
self._sink_task = None
if self._params.camera_out_enabled:
if self._sink_clock_task:
self._sink_clock_task.cancel()
await self._sink_clock_task
self._sink_clock_task = None
# Cancel and wait for the camera output task to finish.
if self._camera_out_task and self._params.camera_out_enabled:
self._camera_out_task.cancel()
await self._camera_out_task
self._camera_out_task = None
self._push_frame_task.cancel()
await self._push_frame_task
self._sink_task.cancel()
await self._sink_task
# Cancel and wait for the audio output task to finish.
if self._audio_out_task and (
self._params.audio_out_enabled and self._params.audio_out_is_live
):
self._audio_out_task.cancel()
await self._audio_out_task
self._audio_out_task = None
async def send_message(self, frame: TransportMessageFrame):
pass
@@ -115,7 +154,7 @@ class BaseOutputTransport(FrameProcessor):
async def send_metrics(self, frame: MetricsFrame):
pass
async def write_frame_to_camera(self, frame: ImageRawFrame):
async def write_frame_to_camera(self, frame: OutputImageRawFrame):
pass
async def write_raw_audio_frames(self, frames: bytes):
@@ -133,7 +172,12 @@ class BaseOutputTransport(FrameProcessor):
# immediately. Other frames require order so they are put in the sink
# queue.
#
if isinstance(frame, CancelFrame):
if isinstance(frame, StartFrame):
# Push StartFrame before start(), because we want StartFrame to be
# processed by every processor before any other frame is processed.
await self.push_frame(frame, direction)
await self.start(frame)
elif isinstance(frame, CancelFrame):
await self.cancel(frame)
await self.push_frame(frame, direction)
elif isinstance(frame, StartInterruptionFrame) or isinstance(frame, StopInterruptionFrame):
@@ -145,19 +189,20 @@ class BaseOutputTransport(FrameProcessor):
elif isinstance(frame, SystemFrame):
await self.push_frame(frame, direction)
# Control frames.
elif isinstance(frame, StartFrame):
await self._sink_queue.put(frame)
await self.start(frame)
elif isinstance(frame, EndFrame):
await self._sink_clock_queue.put((sys.maxsize, frame.id, frame))
await self._sink_queue.put(frame)
await self.stop(frame)
# Other frames.
elif isinstance(frame, AudioRawFrame):
elif isinstance(frame, OutputAudioRawFrame):
await self._handle_audio(frame)
elif isinstance(frame, ImageRawFrame) or isinstance(frame, SpriteFrame):
elif isinstance(frame, OutputImageRawFrame) or isinstance(frame, SpriteFrame):
await self._handle_image(frame)
elif isinstance(frame, TransportMessageFrame) and frame.urgent:
await self.send_message(frame)
# TODO(aleix): Images and audio should support presentation timestamps.
elif frame.pts:
await self._sink_clock_queue.put((frame.pts, frame.id, frame))
else:
await self._sink_queue.put(frame)
@@ -166,19 +211,21 @@ class BaseOutputTransport(FrameProcessor):
return
if isinstance(frame, StartInterruptionFrame):
# Stop sink task.
self._sink_task.cancel()
await self._sink_task
self._create_sink_task()
# Stop push task.
self._push_frame_task.cancel()
await self._push_frame_task
self._create_push_task()
# Stop sink tasks.
if self._sink_task:
self._sink_task.cancel()
await self._sink_task
# Stop sink clock tasks.
if self._sink_clock_task:
self._sink_clock_task.cancel()
await self._sink_clock_task
# Create sink tasks.
self._create_sink_tasks()
# Let's send a bot stopped speaking if we have to.
if self._bot_speaking:
await self._bot_stopped_speaking()
async def _handle_audio(self, frame: AudioRawFrame):
async def _handle_audio(self, frame: OutputAudioRawFrame):
if not self._params.audio_out_enabled:
return
@@ -187,12 +234,15 @@ class BaseOutputTransport(FrameProcessor):
else:
self._audio_buffer.extend(frame.audio)
while len(self._audio_buffer) >= self._audio_chunk_size:
chunk = AudioRawFrame(bytes(self._audio_buffer[:self._audio_chunk_size]),
sample_rate=frame.sample_rate, num_channels=frame.num_channels)
chunk = OutputAudioRawFrame(
bytes(self._audio_buffer[: self._audio_chunk_size]),
sample_rate=frame.sample_rate,
num_channels=frame.num_channels,
)
await self._sink_queue.put(chunk)
self._audio_buffer = self._audio_buffer[self._audio_chunk_size:]
self._audio_buffer = self._audio_buffer[self._audio_chunk_size :]
async def _handle_image(self, frame: ImageRawFrame | SpriteFrame):
async def _handle_image(self, frame: OutputImageRawFrame | SpriteFrame):
if not self._params.camera_out_enabled:
return
@@ -201,102 +251,117 @@ class BaseOutputTransport(FrameProcessor):
else:
await self._sink_queue.put(frame)
def _create_sink_task(self):
#
# Sink tasks
#
def _create_sink_tasks(self):
loop = self.get_event_loop()
self._sink_queue = asyncio.Queue()
self._sink_task = loop.create_task(self._sink_task_handler())
self._sink_clock_queue = asyncio.PriorityQueue()
self._sink_clock_task = loop.create_task(self._sink_clock_task_handler())
async def _sink_frame_handler(self, frame: Frame):
if isinstance(frame, OutputAudioRawFrame):
await self.write_raw_audio_frames(frame.audio)
await self.push_frame(frame)
await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM)
elif isinstance(frame, OutputImageRawFrame):
await self._set_camera_image(frame)
elif isinstance(frame, SpriteFrame):
await self._set_camera_images(frame.images)
elif isinstance(frame, TransportMessageFrame):
await self.send_message(frame)
elif isinstance(frame, TTSStartedFrame):
await self._bot_started_speaking()
await self.push_frame(frame)
elif isinstance(frame, TTSStoppedFrame):
await self._bot_stopped_speaking()
await self.push_frame(frame)
else:
await self.push_frame(frame)
async def _sink_task_handler(self):
running = True
while running:
try:
frame = await self._sink_queue.get()
if isinstance(frame, AudioRawFrame):
await self.write_raw_audio_frames(frame.audio)
await self._internal_push_frame(frame)
await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM)
elif isinstance(frame, ImageRawFrame):
await self._set_camera_image(frame)
elif isinstance(frame, SpriteFrame):
await self._set_camera_images(frame.images)
elif isinstance(frame, TransportMessageFrame):
await self.send_message(frame)
elif isinstance(frame, TTSStartedFrame):
await self._bot_started_speaking()
await self._internal_push_frame(frame)
elif isinstance(frame, TTSStoppedFrame):
await self._bot_stopped_speaking()
await self._internal_push_frame(frame)
else:
await self._internal_push_frame(frame)
await self._sink_frame_handler(frame)
running = not isinstance(frame, EndFrame)
self._sink_queue.task_done()
except asyncio.CancelledError:
break
except Exception as e:
logger.exception(f"{self} error processing sink queue: {e}")
async def _sink_clock_frame_handler(self, frame: Frame):
# TODO(aleix): For now we just process TextFrame. But we should process
# audio and video as well.
if isinstance(frame, TextFrame):
await self.push_frame(frame)
async def _sink_clock_task_handler(self):
running = True
while running:
try:
timestamp, _, frame = await self._sink_clock_queue.get()
# If we hit an EndFrame, we can finish right away.
running = not isinstance(frame, EndFrame)
# If we have a frame we check it's presentation timestamp. If it
# has already passed we process it, otherwise we wait until it's
# time to process it.
if running:
current_time = self.get_clock().get_time()
if timestamp <= current_time:
await self._sink_clock_frame_handler(frame)
else:
wait_time = nanoseconds_to_seconds(timestamp - current_time)
await asyncio.sleep(wait_time)
await self._sink_frame_handler(frame)
self._sink_clock_queue.task_done()
except asyncio.CancelledError:
break
except Exception as e:
logger.exception(f"{self} error processing sink clock queue: {e}")
async def _bot_started_speaking(self):
logger.debug("Bot started speaking")
self._bot_speaking = True
await self._internal_push_frame(BotStartedSpeakingFrame(), FrameDirection.UPSTREAM)
await self.push_frame(BotStartedSpeakingFrame(), FrameDirection.UPSTREAM)
async def _bot_stopped_speaking(self):
logger.debug("Bot stopped speaking")
self._bot_speaking = False
await self._internal_push_frame(BotStoppedSpeakingFrame(), FrameDirection.UPSTREAM)
#
# Push frames task
#
def _create_push_task(self):
loop = self.get_event_loop()
self._push_queue = asyncio.Queue()
self._push_frame_task = loop.create_task(self._push_frame_task_handler())
async def _internal_push_frame(
self,
frame: Frame | None,
direction: FrameDirection | None = FrameDirection.DOWNSTREAM):
await self._push_queue.put((frame, direction))
async def _push_frame_task_handler(self):
running = True
while running:
try:
(frame, direction) = await self._push_queue.get()
await self.push_frame(frame, direction)
running = not isinstance(frame, EndFrame)
self._push_queue.task_done()
except asyncio.CancelledError:
break
await self.push_frame(BotStoppedSpeakingFrame(), FrameDirection.UPSTREAM)
#
# Camera out
#
async def send_image(self, frame: ImageRawFrame | SpriteFrame):
async def send_image(self, frame: OutputImageRawFrame | SpriteFrame):
await self.process_frame(frame, FrameDirection.DOWNSTREAM)
async def _draw_image(self, frame: ImageRawFrame):
async def _draw_image(self, frame: OutputImageRawFrame):
desired_size = (self._params.camera_out_width, self._params.camera_out_height)
if frame.size != desired_size:
image = Image.frombytes(frame.format, frame.size, frame.image)
resized_image = image.resize(desired_size)
logger.warning(
f"{frame} does not have the expected size {desired_size}, resizing")
frame = ImageRawFrame(resized_image.tobytes(), resized_image.size, resized_image.format)
logger.warning(f"{frame} does not have the expected size {desired_size}, resizing")
frame = OutputImageRawFrame(
resized_image.tobytes(), resized_image.size, resized_image.format
)
await self.write_frame_to_camera(frame)
async def _set_camera_image(self, image: ImageRawFrame):
async def _set_camera_image(self, image: OutputImageRawFrame):
self._camera_images = itertools.cycle([image])
async def _set_camera_images(self, images: List[ImageRawFrame]):
async def _set_camera_images(self, images: List[OutputImageRawFrame]):
self._camera_images = itertools.cycle(images)
async def _camera_out_task_handler(self):
@@ -311,9 +376,9 @@ class BaseOutputTransport(FrameProcessor):
elif self._camera_images:
image = next(self._camera_images)
await self._draw_image(image)
await asyncio.sleep(1.0 / self._params.camera_out_framerate)
await asyncio.sleep(self._camera_out_frame_duration)
else:
await asyncio.sleep(1.0 / self._params.camera_out_framerate)
await asyncio.sleep(self._camera_out_frame_duration)
except asyncio.CancelledError:
break
except Exception as e:
@@ -348,7 +413,7 @@ class BaseOutputTransport(FrameProcessor):
# Audio out
#
async def send_audio(self, frame: AudioRawFrame):
async def send_audio(self, frame: OutputAudioRawFrame):
await self.process_frame(frame, FrameDirection.DOWNSTREAM)
async def _audio_out_task_handler(self):
@@ -356,7 +421,7 @@ class BaseOutputTransport(FrameProcessor):
try:
frame = await self._audio_out_queue.get()
await self.write_raw_audio_frames(frame.audio)
await self._internal_push_frame(frame)
await self.push_frame(frame)
await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM)
except asyncio.CancelledError:
break

12
src/pipecat/transports/base_transport.py
View File

@@ -42,11 +42,12 @@ class TransportParams(BaseModel):
class BaseTransport(ABC):
def __init__(self,
input_name: str | None = None,
output_name: str | None = None,
loop: asyncio.AbstractEventLoop | None = None):
def __init__(
self,
input_name: str | None = None,
output_name: str | None = None,
loop: asyncio.AbstractEventLoop | None = None,
):
self._input_name = input_name
self._output_name = output_name
self._loop = loop or asyncio.get_running_loop()
@@ -64,6 +65,7 @@ class BaseTransport(ABC):
def decorator(handler):
self.add_event_handler(event_name, handler)
return handler
return decorator
def add_event_handler(self, event_name: str, handler):

22
src/pipecat/transports/local/audio.py
View File

@@ -8,7 +8,7 @@ import asyncio
from concurrent.futures import ThreadPoolExecutor
from pipecat.frames.frames import AudioRawFrame, StartFrame
from pipecat.frames.frames import InputAudioRawFrame, StartFrame
from pipecat.processors.frame_processor import FrameProcessor
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
@@ -21,12 +21,12 @@ try:
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use local audio, you need to `pip install pipecat-ai[local]`. On MacOS, you also need to `brew install portaudio`.")
"In order to use local audio, you need to `pip install pipecat-ai[local]`. On MacOS, you also need to `brew install portaudio`."
)
raise Exception(f"Missing module: {e}")
class LocalAudioInputTransport(BaseInputTransport):
def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams):
super().__init__(params)
@@ -39,7 +39,8 @@ class LocalAudioInputTransport(BaseInputTransport):
rate=params.audio_in_sample_rate,
frames_per_buffer=num_frames,
stream_callback=self._audio_in_callback,
input=True)
input=True,
)
async def start(self, frame: StartFrame):
await super().start(frame)
@@ -54,9 +55,11 @@ class LocalAudioInputTransport(BaseInputTransport):
self._in_stream.close()
def _audio_in_callback(self, in_data, frame_count, time_info, status):
frame = AudioRawFrame(audio=in_data,
sample_rate=self._params.audio_in_sample_rate,
num_channels=self._params.audio_in_channels)
frame = InputAudioRawFrame(
audio=in_data,
sample_rate=self._params.audio_in_sample_rate,
num_channels=self._params.audio_in_channels,
)
asyncio.run_coroutine_threadsafe(self.push_audio_frame(frame), self.get_event_loop())
@@ -64,7 +67,6 @@ class LocalAudioInputTransport(BaseInputTransport):
class LocalAudioOutputTransport(BaseOutputTransport):
def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams):
super().__init__(params)
@@ -74,7 +76,8 @@ class LocalAudioOutputTransport(BaseOutputTransport):
format=py_audio.get_format_from_width(2),
channels=params.audio_out_channels,
rate=params.audio_out_sample_rate,
output=True)
output=True,
)
async def start(self, frame: StartFrame):
await super().start(frame)
@@ -93,7 +96,6 @@ class LocalAudioOutputTransport(BaseOutputTransport):
class LocalAudioTransport(BaseTransport):
def __init__(self, params: TransportParams):
self._params = params
self._pyaudio = pyaudio.PyAudio()

37
src/pipecat/transports/local/tk.py
View File

@@ -11,8 +11,7 @@ from concurrent.futures import ThreadPoolExecutor
import numpy as np
import tkinter as tk
from pipecat.frames.frames import AudioRawFrame, ImageRawFrame, StartFrame
from pipecat.processors.frame_processor import FrameProcessor
from pipecat.frames.frames import InputAudioRawFrame, OutputImageRawFrame, StartFrame
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import BaseTransport, TransportParams
@@ -24,7 +23,8 @@ try:
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use local audio, you need to `pip install pipecat-ai[local]`. On MacOS, you also need to `brew install portaudio`.")
"In order to use local audio, you need to `pip install pipecat-ai[local]`. On MacOS, you also need to `brew install portaudio`."
)
raise Exception(f"Missing module: {e}")
try:
@@ -36,7 +36,6 @@ except ModuleNotFoundError as e:
class TkInputTransport(BaseInputTransport):
def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams):
super().__init__(params)
@@ -49,7 +48,8 @@ class TkInputTransport(BaseInputTransport):
rate=params.audio_in_sample_rate,
frames_per_buffer=num_frames,
stream_callback=self._audio_in_callback,
input=True)
input=True,
)
async def start(self, frame: StartFrame):
await super().start(frame)
@@ -64,9 +64,11 @@ class TkInputTransport(BaseInputTransport):
self._in_stream.close()
def _audio_in_callback(self, in_data, frame_count, time_info, status):
frame = AudioRawFrame(audio=in_data,
sample_rate=self._params.audio_in_sample_rate,
num_channels=self._params.audio_in_channels)
frame = InputAudioRawFrame(
audio=in_data,
sample_rate=self._params.audio_in_sample_rate,
num_channels=self._params.audio_in_channels,
)
asyncio.run_coroutine_threadsafe(self.push_audio_frame(frame), self.get_event_loop())
@@ -74,7 +76,6 @@ class TkInputTransport(BaseInputTransport):
class TkOutputTransport(BaseOutputTransport):
def __init__(self, tk_root: tk.Tk, py_audio: pyaudio.PyAudio, params: TransportParams):
super().__init__(params)
@@ -84,7 +85,8 @@ class TkOutputTransport(BaseOutputTransport):
format=py_audio.get_format_from_width(2),
channels=params.audio_out_channels,
rate=params.audio_out_sample_rate,
output=True)
output=True,
)
# Start with a neutral gray background.
array = np.ones((1024, 1024, 3)) * 128
@@ -108,18 +110,14 @@ class TkOutputTransport(BaseOutputTransport):
async def write_raw_audio_frames(self, frames: bytes):
await self.get_event_loop().run_in_executor(self._executor, self._out_stream.write, frames)
async def write_frame_to_camera(self, frame: ImageRawFrame):
async def write_frame_to_camera(self, frame: OutputImageRawFrame):
self.get_event_loop().call_soon(self._write_frame_to_tk, frame)
def _write_frame_to_tk(self, frame: ImageRawFrame):
def _write_frame_to_tk(self, frame: OutputImageRawFrame):
width = frame.size[0]
height = frame.size[1]
data = f"P6 {width} {height} 255 ".encode() + frame.image
photo = tk.PhotoImage(
width=width,
height=height,
data=data,
format="PPM")
photo = tk.PhotoImage(width=width, height=height, data=data, format="PPM")
self._image_label.config(image=photo)
# This holds a reference to the photo, preventing it from being garbage
@@ -128,7 +126,6 @@ class TkOutputTransport(BaseOutputTransport):
class TkLocalTransport(BaseTransport):
def __init__(self, tk_root: tk.Tk, params: TransportParams):
self._tk_root = tk_root
self._params = params
@@ -141,12 +138,12 @@ class TkLocalTransport(BaseTransport):
# BaseTransport
#
def input(self) -> FrameProcessor:
def input(self) -> TkInputTransport:
if not self._input:
self._input = TkInputTransport(self._pyaudio, self._params)
return self._input
def output(self) -> FrameProcessor:
def output(self) -> TkOutputTransport:
if not self._output:
self._output = TkOutputTransport(self._tk_root, self._pyaudio, self._params)
return self._output

77
src/pipecat/transports/network/fastapi_websocket.py
View File

@@ -12,8 +12,16 @@ import wave
from typing import Awaitable, Callable
from pydantic.main import BaseModel
from pipecat.frames.frames import AudioRawFrame, CancelFrame, EndFrame, Frame, StartFrame, StartInterruptionFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,
EndFrame,
Frame,
InputAudioRawFrame,
StartFrame,
StartInterruptionFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.serializers.base_serializer import FrameSerializer
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
@@ -27,7 +35,8 @@ try:
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use FastAPI websockets, you need to `pip install pipecat-ai[websocket]`.")
"In order to use FastAPI websockets, you need to `pip install pipecat-ai[websocket]`."
)
raise Exception(f"Missing module: {e}")
@@ -43,13 +52,13 @@ class FastAPIWebsocketCallbacks(BaseModel):
class FastAPIWebsocketInputTransport(BaseInputTransport):
def __init__(
self,
websocket: WebSocket,
params: FastAPIWebsocketParams,
callbacks: FastAPIWebsocketCallbacks,
**kwargs):
self,
websocket: WebSocket,
params: FastAPIWebsocketParams,
callbacks: FastAPIWebsocketCallbacks,
**kwargs,
):
super().__init__(params, **kwargs)
self._websocket = websocket
@@ -79,13 +88,18 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
continue
if isinstance(frame, AudioRawFrame):
await self.push_audio_frame(frame)
await self.push_audio_frame(
InputAudioRawFrame(
audio=frame.audio,
sample_rate=frame.sample_rate,
num_channels=frame.num_channels,
)
)
await self._callbacks.on_client_disconnected(self._websocket)
class FastAPIWebsocketOutputTransport(BaseOutputTransport):
def __init__(self, websocket: WebSocket, params: FastAPIWebsocketParams, **kwargs):
super().__init__(params, **kwargs)
@@ -101,12 +115,11 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
async def write_raw_audio_frames(self, frames: bytes):
self._websocket_audio_buffer += frames
while len(self._websocket_audio_buffer) >= self._params.audio_frame_size:
while len(self._websocket_audio_buffer):
frame = AudioRawFrame(
audio=self._websocket_audio_buffer[:
self._params.audio_frame_size],
audio=self._websocket_audio_buffer[: self._params.audio_frame_size],
sample_rate=self._params.audio_out_sample_rate,
num_channels=self._params.audio_out_channels
num_channels=self._params.audio_out_channels,
)
if self._params.add_wav_header:
@@ -119,9 +132,8 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
ww.close()
content.seek(0)
wav_frame = AudioRawFrame(
content.read(),
sample_rate=frame.sample_rate,
num_channels=frame.num_channels)
content.read(), sample_rate=frame.sample_rate, num_channels=frame.num_channels
)
frame = wav_frame
payload = self._params.serializer.serialize(frame)
@@ -129,7 +141,8 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
await self._websocket.send_text(payload)
self._websocket_audio_buffer = self._websocket_audio_buffer[
self._params.audio_frame_size:]
self._params.audio_frame_size :
]
async def _write_frame(self, frame: Frame):
payload = self._params.serializer.serialize(frame)
@@ -138,36 +151,38 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
class FastAPIWebsocketTransport(BaseTransport):
def __init__(
self,
websocket: WebSocket,
params: FastAPIWebsocketParams,
input_name: str | None = None,
output_name: str | None = None,
loop: asyncio.AbstractEventLoop | None = None):
self,
websocket: WebSocket,
params: FastAPIWebsocketParams,
input_name: str | None = None,
output_name: str | None = None,
loop: asyncio.AbstractEventLoop | None = None,
):
super().__init__(input_name=input_name, output_name=output_name, loop=loop)
self._params = params
self._callbacks = FastAPIWebsocketCallbacks(
on_client_connected=self._on_client_connected,
on_client_disconnected=self._on_client_disconnected
on_client_disconnected=self._on_client_disconnected,
)
self._input = FastAPIWebsocketInputTransport(
websocket, self._params, self._callbacks, name=self._input_name)
websocket, self._params, self._callbacks, name=self._input_name
)
self._output = FastAPIWebsocketOutputTransport(
websocket, self._params, name=self._output_name)
websocket, self._params, name=self._output_name
)
# Register supported handlers. The user will only be able to register
# these handlers.
self._register_event_handler("on_client_connected")
self._register_event_handler("on_client_disconnected")
def input(self) -> FrameProcessor:
def input(self) -> FastAPIWebsocketInputTransport:
return self._input
def output(self) -> FrameProcessor:
def output(self) -> FastAPIWebsocketOutputTransport:
return self._output
async def _on_client_connected(self, websocket):

72
src/pipecat/transports/network/websocket_server.py
View File

@@ -11,8 +11,13 @@ import wave
from typing import Awaitable, Callable
from pydantic.main import BaseModel
from pipecat.frames.frames import AudioRawFrame, CancelFrame, EndFrame, StartFrame
from pipecat.processors.frame_processor import FrameProcessor
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,
EndFrame,
InputAudioRawFrame,
StartFrame,
)
from pipecat.serializers.base_serializer import FrameSerializer
from pipecat.serializers.protobuf import ProtobufFrameSerializer
from pipecat.transports.base_input import BaseInputTransport
@@ -41,14 +46,14 @@ class WebsocketServerCallbacks(BaseModel):
class WebsocketServerInputTransport(BaseInputTransport):
def __init__(
self,
host: str,
port: int,
params: WebsocketServerParams,
callbacks: WebsocketServerCallbacks,
**kwargs):
self,
host: str,
port: int,
params: WebsocketServerParams,
callbacks: WebsocketServerCallbacks,
**kwargs,
):
super().__init__(params, **kwargs)
self._host = host
@@ -98,9 +103,15 @@ class WebsocketServerInputTransport(BaseInputTransport):
continue
if isinstance(frame, AudioRawFrame):
await self.push_audio_frame(frame)
await self.push_audio_frame(
InputAudioRawFrame(
audio=frame.audio,
sample_rate=frame.sample_rate,
num_channels=frame.num_channels,
)
)
else:
await self._internal_push_frame(frame)
await self.push_frame(frame)
# Notify disconnection
await self._callbacks.on_client_disconnected(websocket)
@@ -112,7 +123,6 @@ class WebsocketServerInputTransport(BaseInputTransport):
class WebsocketServerOutputTransport(BaseOutputTransport):
def __init__(self, params: WebsocketServerParams, **kwargs):
super().__init__(params, **kwargs)
@@ -135,9 +145,9 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
self._websocket_audio_buffer += frames
while len(self._websocket_audio_buffer) >= self._params.audio_frame_size:
frame = AudioRawFrame(
audio=self._websocket_audio_buffer[:self._params.audio_frame_size],
audio=self._websocket_audio_buffer[: self._params.audio_frame_size],
sample_rate=self._params.audio_out_sample_rate,
num_channels=self._params.audio_out_channels
num_channels=self._params.audio_out_channels,
)
if self._params.add_wav_header:
@@ -150,28 +160,29 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
ww.close()
content.seek(0)
wav_frame = AudioRawFrame(
content.read(),
sample_rate=frame.sample_rate,
num_channels=frame.num_channels)
content.read(), sample_rate=frame.sample_rate, num_channels=frame.num_channels
)
frame = wav_frame
proto = self._params.serializer.serialize(frame)
if proto:
await self._websocket.send(proto)
self._websocket_audio_buffer = self._websocket_audio_buffer[self._params.audio_frame_size:]
self._websocket_audio_buffer = self._websocket_audio_buffer[
self._params.audio_frame_size :
]
class WebsocketServerTransport(BaseTransport):
def __init__(
self,
host: str = "localhost",
port: int = 8765,
params: WebsocketServerParams = WebsocketServerParams(),
input_name: str | None = None,
output_name: str | None = None,
loop: asyncio.AbstractEventLoop | None = None):
self,
host: str = "localhost",
port: int = 8765,
params: WebsocketServerParams = WebsocketServerParams(),
input_name: str | None = None,
output_name: str | None = None,
loop: asyncio.AbstractEventLoop | None = None,
):
super().__init__(input_name=input_name, output_name=output_name, loop=loop)
self._host = host
self._port = port
@@ -179,7 +190,7 @@ class WebsocketServerTransport(BaseTransport):
self._callbacks = WebsocketServerCallbacks(
on_client_connected=self._on_client_connected,
on_client_disconnected=self._on_client_disconnected
on_client_disconnected=self._on_client_disconnected,
)
self._input: WebsocketServerInputTransport | None = None
self._output: WebsocketServerOutputTransport | None = None
@@ -190,13 +201,14 @@ class WebsocketServerTransport(BaseTransport):
self._register_event_handler("on_client_connected")
self._register_event_handler("on_client_disconnected")
def input(self) -> FrameProcessor:
def input(self) -> WebsocketServerInputTransport:
if not self._input:
self._input = WebsocketServerInputTransport(
self._host, self._port, self._params, self._callbacks, name=self._input_name)
self._host, self._port, self._params, self._callbacks, name=self._input_name
)
return self._input
def output(self) -> FrameProcessor:
def output(self) -> WebsocketServerOutputTransport:
if not self._output:
self._output = WebsocketServerOutputTransport(self._params, name=self._output_name)
return self._output

247
src/pipecat/transports/services/daily.py
View File

@@ -18,23 +18,32 @@ from daily import (
EventHandler,
VirtualCameraDevice,
VirtualMicrophoneDevice,
VirtualSpeakerDevice)
VirtualSpeakerDevice,
)
from pydantic.main import BaseModel
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,
EndFrame,
Frame,
ImageRawFrame,
InputAudioRawFrame,
InterimTranscriptionFrame,
MetricsFrame,
OutputAudioRawFrame,
OutputImageRawFrame,
SpriteFrame,
StartFrame,
TranscriptionFrame,
TransportMessageFrame,
UserImageRawFrame,
UserImageRequestFrame)
UserImageRequestFrame,
)
from pipecat.metrics.metrics import (
LLMUsageMetricsData,
ProcessingMetricsData,
TTFBMetricsData,
TTSUsageMetricsData,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.transcriptions.language import Language
from pipecat.transports.base_input import BaseInputTransport
@@ -45,11 +54,12 @@ from pipecat.vad.vad_analyzer import VADAnalyzer, VADParams
from loguru import logger
try:
from daily import (EventHandler, CallClient, Daily)
from daily import EventHandler, CallClient, Daily
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use the Daily transport, you need to `pip install pipecat-ai[daily]`.")
"In order to use the Daily transport, you need to `pip install pipecat-ai[daily]`."
)
raise Exception(f"Missing module: {e}")
VAD_RESET_PERIOD_MS = 2000
@@ -61,14 +71,11 @@ class DailyTransportMessageFrame(TransportMessageFrame):
class WebRTCVADAnalyzer(VADAnalyzer):
def __init__(self, *, sample_rate=16000, num_channels=1, params: VADParams = VADParams()):
super().__init__(sample_rate=sample_rate, num_channels=num_channels, params=params)
self._webrtc_vad = Daily.create_native_vad(
reset_period_ms=VAD_RESET_PERIOD_MS,
sample_rate=sample_rate,
channels=num_channels
reset_period_ms=VAD_RESET_PERIOD_MS, sample_rate=sample_rate, channels=num_channels
)
logger.debug("Loaded native WebRTC VAD")
@@ -96,9 +103,7 @@ class DailyTranscriptionSettings(BaseModel):
endpointing: bool = True
punctuate: bool = True
includeRawResponse: bool = True
extra: Mapping[str, Any] = {
"interim_results": True
}
extra: Mapping[str, Any] = {"interim_results": True}
class DailyParams(TransportParams):
@@ -137,12 +142,13 @@ def completion_callback(future):
future.set_result(*args)
except asyncio.InvalidStateError:
pass
future.get_loop().call_soon_threadsafe(set_result, future, *args)
return _callback
class DailyTransportClient(EventHandler):
_daily_initialized: bool = False
# This is necessary to override EventHandler's __new__ method.
@@ -150,13 +156,14 @@ class DailyTransportClient(EventHandler):
return super().__new__(cls)
def __init__(
self,
room_url: str,
token: str | None,
bot_name: str,
params: DailyParams,
callbacks: DailyCallbacks,
loop: asyncio.AbstractEventLoop):
self,
room_url: str,
token: str | None,
bot_name: str,
params: DailyParams,
callbacks: DailyCallbacks,
loop: asyncio.AbstractEventLoop,
):
super().__init__()
if not DailyTransportClient._daily_initialized:
@@ -189,7 +196,8 @@ class DailyTransportClient(EventHandler):
self._camera_name(),
width=self._params.camera_out_width,
height=self._params.camera_out_height,
color_format=self._params.camera_out_color_format)
color_format=self._params.camera_out_color_format,
)
self._mic: VirtualMicrophoneDevice | None = None
if self._params.audio_out_enabled:
@@ -197,7 +205,8 @@ class DailyTransportClient(EventHandler):
self._mic_name(),
sample_rate=self._params.audio_out_sample_rate,
channels=self._params.audio_out_channels,
non_blocking=True)
non_blocking=True,
)
self._speaker: VirtualSpeakerDevice | None = None
if self._params.audio_in_enabled or self._params.vad_enabled:
@@ -205,7 +214,8 @@ class DailyTransportClient(EventHandler):
self._speaker_name(),
sample_rate=self._params.audio_in_sample_rate,
channels=self._params.audio_in_channels,
non_blocking=True)
non_blocking=True,
)
Daily.select_speaker_device(self._speaker_name())
def _camera_name(self):
@@ -234,12 +244,11 @@ class DailyTransportClient(EventHandler):
future = self._loop.create_future()
self._client.send_app_message(
frame.message,
participant_id,
completion=completion_callback(future))
frame.message, participant_id, completion=completion_callback(future)
)
await future
async def read_next_audio_frame(self) -> AudioRawFrame | None:
async def read_next_audio_frame(self) -> InputAudioRawFrame | None:
if not self._speaker:
return None
@@ -252,7 +261,9 @@ class DailyTransportClient(EventHandler):
audio = await future
if len(audio) > 0:
return AudioRawFrame(audio=audio, sample_rate=sample_rate, num_channels=num_channels)
return InputAudioRawFrame(
audio=audio, sample_rate=sample_rate, num_channels=num_channels
)
else:
# If we don't read any audio it could be there's no participant
# connected. daily-python will return immediately if that's the
@@ -268,7 +279,7 @@ class DailyTransportClient(EventHandler):
self._mic.write_frames(frames, completion=completion_callback(future))
await future
async def write_frame_to_camera(self, frame: ImageRawFrame):
async def write_frame_to_camera(self, frame: OutputImageRawFrame):
if not self._camera:
return None
@@ -285,12 +296,9 @@ class DailyTransportClient(EventHandler):
# For performance reasons, never subscribe to video streams (unless a
# video renderer is registered).
self._client.update_subscription_profiles({
"base": {
"camera": "unsubscribed",
"screenVideo": "unsubscribed"
}
})
self._client.update_subscription_profiles(
{"base": {"camera": "unsubscribed", "screenVideo": "unsubscribed"}}
)
self._client.set_user_name(self._bot_name)
@@ -322,7 +330,7 @@ class DailyTransportClient(EventHandler):
future = self._loop.create_future()
self._client.start_transcription(
settings=self._params.transcription_settings.model_dump(exclude_none=True),
completion=completion_callback(future)
completion=completion_callback(future),
)
error = await future
if error:
@@ -369,12 +377,15 @@ class DailyTransportClient(EventHandler):
},
"microphone": {
"sendSettings": {
"channelConfig": "stereo" if self._params.audio_out_channels == 2 else "mono",
"channelConfig": "stereo"
if self._params.audio_out_channels == 2
else "mono",
"bitrate": self._params.audio_out_bitrate,
}
}
},
},
})
},
)
return await asyncio.wait_for(future, timeout=10)
@@ -451,18 +462,17 @@ class DailyTransportClient(EventHandler):
self._transcription_renderers[participant_id] = callback
def capture_participant_video(
self,
participant_id: str,
callback: Callable,
framerate: int = 30,
video_source: str = "camera",
color_format: str = "RGB"):
self,
participant_id: str,
callback: Callable,
framerate: int = 30,
video_source: str = "camera",
color_format: str = "RGB",
):
# Only enable camera subscription on this participant
self._client.update_subscriptions(participant_settings={
participant_id: {
"media": "subscribed"
}
})
self._client.update_subscriptions(
participant_settings={participant_id: {"media": "subscribed"}}
)
self._video_renderers[participant_id] = callback
@@ -470,7 +480,8 @@ class DailyTransportClient(EventHandler):
participant_id,
self._video_frame_received,
video_source=video_source,
color_format=color_format)
color_format=color_format,
)
#
#
@@ -548,9 +559,9 @@ class DailyTransportClient(EventHandler):
callback,
participant_id,
video_frame.buffer,
(video_frame.width,
video_frame.height),
video_frame.color_format)
(video_frame.width, video_frame.height),
video_frame.color_format,
)
def _call_async_callback(self, callback, *args):
future = asyncio.run_coroutine_threadsafe(callback(*args), self._loop)
@@ -558,20 +569,23 @@ class DailyTransportClient(EventHandler):
class DailyInputTransport(BaseInputTransport):
def __init__(self, client: DailyTransportClient, params: DailyParams, **kwargs):
super().__init__(params, **kwargs)
self._client = client
self._video_renderers = {}
# Task that gets audio data from a device or the network and queues it
# internally to be processed.
self._audio_in_task = None
self._vad_analyzer: VADAnalyzer | None = params.vad_analyzer
if params.vad_enabled and not params.vad_analyzer:
self._vad_analyzer = WebRTCVADAnalyzer(
sample_rate=self._params.audio_in_sample_rate,
num_channels=self._params.audio_in_channels)
num_channels=self._params.audio_in_channels,
)
async def start(self, frame: StartFrame):
# Parent start.
@@ -592,6 +606,7 @@ class DailyInputTransport(BaseInputTransport):
if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
self._audio_in_task.cancel()
await self._audio_in_task
self._audio_in_task = None
async def cancel(self, frame: CancelFrame):
# Parent stop.
@@ -602,6 +617,7 @@ class DailyInputTransport(BaseInputTransport):
if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
self._audio_in_task.cancel()
await self._audio_in_task
self._audio_in_task = None
async def cleanup(self):
await super().cleanup()
@@ -625,11 +641,11 @@ class DailyInputTransport(BaseInputTransport):
#
async def push_transcription_frame(self, frame: TranscriptionFrame | InterimTranscriptionFrame):
await self._internal_push_frame(frame)
await self.push_frame(frame)
async def push_app_message(self, message: Any, sender: str):
frame = DailyTransportMessageFrame(message=message, participant_id=sender)
await self._internal_push_frame(frame)
await self.push_frame(frame)
#
# Audio in
@@ -649,11 +665,12 @@ class DailyInputTransport(BaseInputTransport):
#
def capture_participant_video(
self,
participant_id: str,
framerate: int = 30,
video_source: str = "camera",
color_format: str = "RGB"):
self,
participant_id: str,
framerate: int = 30,
video_source: str = "camera",
color_format: str = "RGB",
):
self._video_renderers[participant_id] = {
"framerate": framerate,
"timestamp": 0,
@@ -661,11 +678,7 @@ class DailyInputTransport(BaseInputTransport):
}
self._client.capture_participant_video(
participant_id,
self._on_participant_video_frame,
framerate,
video_source,
color_format
participant_id, self._on_participant_video_frame, framerate, video_source, color_format
)
def request_participant_image(self, participant_id: str):
@@ -688,17 +701,14 @@ class DailyInputTransport(BaseInputTransport):
if render_frame:
frame = UserImageRawFrame(
user_id=participant_id,
image=buffer,
size=size,
format=format)
await self._internal_push_frame(frame)
user_id=participant_id, image=buffer, size=size, format=format
)
await self.push_frame(frame)
self._video_renderers[participant_id]["timestamp"] = curr_time
class DailyOutputTransport(BaseOutputTransport):
def __init__(self, client: DailyTransportClient, params: DailyParams, **kwargs):
super().__init__(params, **kwargs)
@@ -731,39 +741,47 @@ class DailyOutputTransport(BaseOutputTransport):
async def send_metrics(self, frame: MetricsFrame):
metrics = {}
if frame.ttfb:
metrics["ttfb"] = frame.ttfb
if frame.processing:
metrics["processing"] = frame.processing
if frame.tokens:
metrics["tokens"] = frame.tokens
if frame.characters:
metrics["characters"] = frame.characters
for d in frame.data:
if isinstance(d, TTFBMetricsData):
if "ttfb" not in metrics:
metrics["ttfb"] = []
metrics["ttfb"].append(d.model_dump(exclude_none=True))
elif isinstance(d, ProcessingMetricsData):
if "processing" not in metrics:
metrics["processing"] = []
metrics["processing"].append(d.model_dump(exclude_none=True))
elif isinstance(d, LLMUsageMetricsData):
if "tokens" not in metrics:
metrics["tokens"] = []
metrics["tokens"].append(d.value.model_dump(exclude_none=True))
elif isinstance(d, TTSUsageMetricsData):
if "characters" not in metrics:
metrics["characters"] = []
metrics["characters"].append(d.model_dump(exclude_none=True))
message = DailyTransportMessageFrame(message={
"type": "pipecat-metrics",
"metrics": metrics
})
message = DailyTransportMessageFrame(
message={"type": "pipecat-metrics", "metrics": metrics}
)
await self._client.send_message(message)
async def write_raw_audio_frames(self, frames: bytes):
await self._client.write_raw_audio_frames(frames)
async def write_frame_to_camera(self, frame: ImageRawFrame):
async def write_frame_to_camera(self, frame: OutputImageRawFrame):
await self._client.write_frame_to_camera(frame)
class DailyTransport(BaseTransport):
def __init__(
self,
room_url: str,
token: str | None,
bot_name: str,
params: DailyParams = DailyParams(),
input_name: str | None = None,
output_name: str | None = None,
loop: asyncio.AbstractEventLoop | None = None):
self,
room_url: str,
token: str | None,
bot_name: str,
params: DailyParams = DailyParams(),
input_name: str | None = None,
output_name: str | None = None,
loop: asyncio.AbstractEventLoop | None = None,
):
super().__init__(input_name=input_name, output_name=output_name, loop=loop)
callbacks = DailyCallbacks(
@@ -786,7 +804,8 @@ class DailyTransport(BaseTransport):
self._params = params
self._client = DailyTransportClient(
room_url, token, bot_name, params, callbacks, self._loop)
room_url, token, bot_name, params, callbacks, self._loop
)
self._input: DailyInputTransport | None = None
self._output: DailyOutputTransport | None = None
@@ -811,12 +830,12 @@ class DailyTransport(BaseTransport):
# BaseTransport
#
def input(self) -> FrameProcessor:
def input(self) -> DailyInputTransport:
if not self._input:
self._input = DailyInputTransport(self._client, self._params, name=self._input_name)
return self._input
def output(self) -> FrameProcessor:
def output(self) -> DailyOutputTransport:
if not self._output:
self._output = DailyOutputTransport(self._client, self._params, name=self._output_name)
return self._output
@@ -829,11 +848,11 @@ class DailyTransport(BaseTransport):
def participant_id(self) -> str:
return self._client.participant_id
async def send_image(self, frame: ImageRawFrame | SpriteFrame):
async def send_image(self, frame: OutputImageRawFrame | SpriteFrame):
if self._output:
await self._output.process_frame(frame, FrameDirection.DOWNSTREAM)
async def send_audio(self, frame: AudioRawFrame):
async def send_audio(self, frame: OutputAudioRawFrame):
if self._output:
await self._output.process_frame(frame, FrameDirection.DOWNSTREAM)
@@ -857,19 +876,20 @@ class DailyTransport(BaseTransport):
def capture_participant_transcription(self, participant_id: str):
self._client.capture_participant_transcription(
participant_id,
self._on_transcription_message
participant_id, self._on_transcription_message
)
def capture_participant_video(
self,
participant_id: str,
framerate: int = 30,
video_source: str = "camera",
color_format: str = "RGB"):
self,
participant_id: str,
framerate: int = 30,
video_source: str = "camera",
color_format: str = "RGB",
):
if self._input:
self._input.capture_participant_video(
participant_id, framerate, video_source, color_format)
participant_id, framerate, video_source, color_format
)
async def _on_joined(self, data):
await self._call_event_handler("on_joined", data)
@@ -897,12 +917,12 @@ class DailyTransport(BaseTransport):
async with aiohttp.ClientSession() as session:
headers = {
"Authorization": f"Bearer {self._params.api_key}",
"Content-Type": "application/json"
"Content-Type": "application/json",
}
data = {
"callId": self._params.dialin_settings.call_id,
"callDomain": self._params.dialin_settings.call_domain,
"sipUri": sip_endpoint
"sipUri": sip_endpoint,
}
url = f"{self._params.api_url}/dialin/pinlessCallUpdate"
@@ -912,7 +932,8 @@ class DailyTransport(BaseTransport):
if r.status != 200:
text = await r.text()
logger.error(
f"Unable to handle dialin-ready event (status: {r.status}, error: {text})")
f"Unable to handle dialin-ready event (status: {r.status}, error: {text})"
)
return
logger.debug("Event dialin-ready was handled successfully")

49
src/pipecat/transports/services/helpers/daily_rest.py
View File

@@ -41,12 +41,12 @@ class DailyRoomProperties(BaseModel, extra="allow"):
if not self.sip_uri:
return ""
else:
return "sip:%s" % self.sip_uri['endpoint']
return "sip:%s" % self.sip_uri["endpoint"]
class DailyRoomParams(BaseModel):
name: Optional[str] = None
privacy: Literal['private', 'public'] = "public"
privacy: Literal["private", "public"] = "public"
properties: DailyRoomProperties = Field(default_factory=DailyRoomProperties)
@@ -61,11 +61,13 @@ class DailyRoomObject(BaseModel):
class DailyRESTHelper:
def __init__(self,
*,
daily_api_key: str,
daily_api_url: str = "https://api.daily.co/v1",
aiohttp_session: aiohttp.ClientSession):
def __init__(
self,
*,
daily_api_key: str,
daily_api_url: str = "https://api.daily.co/v1",
aiohttp_session: aiohttp.ClientSession,
):
self.daily_api_key = daily_api_key
self.daily_api_url = daily_api_url
self.aiohttp_session = aiohttp_session
@@ -80,7 +82,9 @@ class DailyRESTHelper:
async def create_room(self, params: DailyRoomParams) -> DailyRoomObject:
headers = {"Authorization": f"Bearer {self.daily_api_key}"}
json = {**params.model_dump(exclude_none=True)}
async with self.aiohttp_session.post(f"{self.daily_api_url}/rooms", headers=headers, json=json) as r:
async with self.aiohttp_session.post(
f"{self.daily_api_url}/rooms", headers=headers, json=json
) as r:
if r.status != 200:
text = await r.text()
raise Exception(f"Unable to create room (status: {r.status}): {text}")
@@ -95,27 +99,22 @@ class DailyRESTHelper:
return room
async def get_token(
self,
room_url: str,
expiry_time: float = 60 * 60,
owner: bool = True) -> str:
self, room_url: str, expiry_time: float = 60 * 60, owner: bool = True
) -> str:
if not room_url:
raise Exception(
"No Daily room specified. You must specify a Daily room in order a token to be generated.")
"No Daily room specified. You must specify a Daily room in order a token to be generated."
)
expiration: float = time.time() + expiry_time
room_name = self.get_name_from_url(room_url)
headers = {"Authorization": f"Bearer {self.daily_api_key}"}
json = {
"properties": {
"room_name": room_name,
"is_owner": owner,
"exp": expiration
}
}
async with self.aiohttp_session.post(f"{self.daily_api_url}/meeting-tokens", headers=headers, json=json) as r:
json = {"properties": {"room_name": room_name, "is_owner": owner, "exp": expiration}}
async with self.aiohttp_session.post(
f"{self.daily_api_url}/meeting-tokens", headers=headers, json=json
) as r:
if r.status != 200:
text = await r.text()
raise Exception(f"Failed to create meeting token (status: {r.status}): {text}")
@@ -130,7 +129,9 @@ class DailyRESTHelper:
async def delete_room_by_name(self, room_name: str) -> bool:
headers = {"Authorization": f"Bearer {self.daily_api_key}"}
async with self.aiohttp_session.delete(f"{self.daily_api_url}/rooms/{room_name}", headers=headers) as r:
async with self.aiohttp_session.delete(
f"{self.daily_api_url}/rooms/{room_name}", headers=headers
) as r:
if r.status != 200 and r.status != 404:
text = await r.text()
raise Exception(f"Failed to delete room [{room_name}] (status: {r.status}): {text}")
@@ -139,7 +140,9 @@ class DailyRESTHelper:
async def _get_room_from_name(self, room_name: str) -> DailyRoomObject:
headers = {"Authorization": f"Bearer {self.daily_api_key}"}
async with self.aiohttp_session.get(f"{self.daily_api_url}/rooms/{room_name}", headers=headers) as r:
async with self.aiohttp_session.get(
f"{self.daily_api_url}/rooms/{room_name}", headers=headers
) as r:
if r.status != 200:
raise Exception(f"Room not found: {room_name}")

619
src/pipecat/transports/services/livekit.py Normal file
View File

@@ -0,0 +1,619 @@
#
# Copyright (c) 2024, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
from dataclasses import dataclass
from typing import Any, Awaitable, Callable, List
from pydantic import BaseModel
import numpy as np
from scipy import signal
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,
EndFrame,
Frame,
MetricsFrame,
StartFrame,
TransportMessageFrame,
)
from pipecat.metrics.metrics import (
LLMUsageMetricsData,
ProcessingMetricsData,
TTFBMetricsData,
TTSUsageMetricsData,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.vad.vad_analyzer import VADAnalyzer
from loguru import logger
try:
from livekit import rtc
from tenacity import retry, stop_after_attempt, wait_exponential
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use LiveKit, you need to `pip install pipecat-ai[livekit]`.")
raise Exception(f"Missing module: {e}")
@dataclass
class LiveKitTransportMessageFrame(TransportMessageFrame):
participant_id: str | None = None
class LiveKitParams(TransportParams):
audio_out_sample_rate: int = 48000
audio_out_channels: int = 1
vad_enabled: bool = True
vad_analyzer: VADAnalyzer | None = None
audio_in_sample_rate: int = 16000
class LiveKitCallbacks(BaseModel):
on_connected: Callable[[], Awaitable[None]]
on_disconnected: Callable[[], Awaitable[None]]
on_participant_connected: Callable[[str], Awaitable[None]]
on_participant_disconnected: Callable[[str], Awaitable[None]]
on_audio_track_subscribed: Callable[[str], Awaitable[None]]
on_audio_track_unsubscribed: Callable[[str], Awaitable[None]]
on_data_received: Callable[[bytes, str], Awaitable[None]]
class LiveKitTransportClient:
def __init__(
self,
url: str,
token: str,
room_name: str,
params: LiveKitParams,
callbacks: LiveKitCallbacks,
loop: asyncio.AbstractEventLoop,
):
self._url = url
self._token = token
self._room_name = room_name
self._params = params
self._callbacks = callbacks
self._loop = loop
self._room = rtc.Room(loop=loop)
self._participant_id: str = ""
self._connected = False
self._audio_source: rtc.AudioSource | None = None
self._audio_track: rtc.LocalAudioTrack | None = None
self._audio_tracks = {}
self._audio_queue = asyncio.Queue()
# Set up room event handlers
self._room.on("participant_connected")(self._on_participant_connected_wrapper)
self._room.on("participant_disconnected")(self._on_participant_disconnected_wrapper)
self._room.on("track_subscribed")(self._on_track_subscribed_wrapper)
self._room.on("track_unsubscribed")(self._on_track_unsubscribed_wrapper)
self._room.on("data_received")(self._on_data_received_wrapper)
self._room.on("connected")(self._on_connected_wrapper)
self._room.on("disconnected")(self._on_disconnected_wrapper)
@property
def participant_id(self) -> str:
return self._participant_id
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
async def connect(self):
if self._connected:
return
logger.info(f"Connecting to {self._room_name}")
try:
await self._room.connect(
self._url,
self._token,
options=rtc.RoomOptions(auto_subscribe=True),
)
self._connected = True
self._participant_id = self._room.local_participant.sid
logger.info(f"Connected to {self._room_name}")
# Set up audio source and track
self._audio_source = rtc.AudioSource(
self._params.audio_out_sample_rate, self._params.audio_out_channels
)
self._audio_track = rtc.LocalAudioTrack.create_audio_track(
"pipecat-audio", self._audio_source
)
options = rtc.TrackPublishOptions()
options.source = rtc.TrackSource.SOURCE_MICROPHONE
await self._room.local_participant.publish_track(self._audio_track, options)
await self._callbacks.on_connected()
except Exception as e:
logger.error(f"Error connecting to {self._room_name}: {e}")
raise
async def disconnect(self):
if not self._connected:
return
logger.info(f"Disconnecting from {self._room_name}")
await self._room.disconnect()
self._connected = False
logger.info(f"Disconnected from {self._room_name}")
await self._callbacks.on_disconnected()
async def send_data(self, data: bytes, participant_id: str | None = None):
if not self._connected:
return
try:
if participant_id:
await self._room.local_participant.publish_data(
data, reliable=True, destination_identities=[participant_id]
)
else:
await self._room.local_participant.publish_data(data, reliable=True)
except Exception as e:
logger.error(f"Error sending data: {e}")
async def publish_audio(self, audio_frame: rtc.AudioFrame):
if not self._connected or not self._audio_source:
return
try:
await self._audio_source.capture_frame(audio_frame)
except Exception as e:
logger.error(f"Error publishing audio: {e}")
def get_participants(self) -> List[str]:
return [p.sid for p in self._room.remote_participants.values()]
async def get_participant_metadata(self, participant_id: str) -> dict:
participant = self._room.remote_participants.get(participant_id)
if participant:
return {
"id": participant.sid,
"name": participant.name,
"metadata": participant.metadata,
"is_speaking": participant.is_speaking,
}
return {}
async def set_participant_metadata(self, metadata: str):
await self._room.local_participant.set_metadata(metadata)
async def mute_participant(self, participant_id: str):
participant = self._room.remote_participants.get(participant_id)
if participant:
for track in participant.tracks.values():
if track.kind == "audio":
await track.set_enabled(False)
async def unmute_participant(self, participant_id: str):
participant = self._room.remote_participants.get(participant_id)
if participant:
for track in participant.tracks.values():
if track.kind == "audio":
await track.set_enabled(True)
# Wrapper methods for event handlers
def _on_participant_connected_wrapper(self, participant: rtc.RemoteParticipant):
asyncio.create_task(self._async_on_participant_connected(participant))
def _on_participant_disconnected_wrapper(self, participant: rtc.RemoteParticipant):
asyncio.create_task(self._async_on_participant_disconnected(participant))
def _on_track_subscribed_wrapper(
self,
track: rtc.Track,
publication: rtc.RemoteTrackPublication,
participant: rtc.RemoteParticipant,
):
asyncio.create_task(self._async_on_track_subscribed(track, publication, participant))
def _on_track_unsubscribed_wrapper(
self,
track: rtc.Track,
publication: rtc.RemoteTrackPublication,
participant: rtc.RemoteParticipant,
):
asyncio.create_task(self._async_on_track_unsubscribed(track, publication, participant))
def _on_data_received_wrapper(self, data: rtc.DataPacket):
asyncio.create_task(self._async_on_data_received(data))
def _on_connected_wrapper(self):
asyncio.create_task(self._async_on_connected())
def _on_disconnected_wrapper(self):
asyncio.create_task(self._async_on_disconnected())
# Async methods for event handling
async def _async_on_participant_connected(self, participant: rtc.RemoteParticipant):
logger.info(f"Participant connected: {participant.identity}")
await self._callbacks.on_participant_connected(participant.sid)
async def _async_on_participant_disconnected(self, participant: rtc.RemoteParticipant):
logger.info(f"Participant disconnected: {participant.identity}")
await self._callbacks.on_participant_disconnected(participant.sid)
async def _async_on_track_subscribed(
self,
track: rtc.Track,
publication: rtc.RemoteTrackPublication,
participant: rtc.RemoteParticipant,
):
if track.kind == rtc.TrackKind.KIND_AUDIO:
logger.info(f"Audio track subscribed: {track.sid} from participant {participant.sid}")
self._audio_tracks[participant.sid] = track
audio_stream = rtc.AudioStream(track)
asyncio.create_task(self._process_audio_stream(audio_stream, participant.sid))
async def _async_on_track_unsubscribed(
self,
track: rtc.Track,
publication: rtc.RemoteTrackPublication,
participant: rtc.RemoteParticipant,
):
logger.info(f"Track unsubscribed: {publication.sid} from {participant.identity}")
if track.kind == rtc.TrackKind.KIND_AUDIO:
await self._callbacks.on_audio_track_unsubscribed(participant.sid)
async def _async_on_data_received(self, data: rtc.DataPacket):
await self._callbacks.on_data_received(data.data, data.participant.sid)
async def _async_on_connected(self):
await self._callbacks.on_connected()
async def _async_on_disconnected(self, reason=None):
self._connected = False
logger.info(f"Disconnected from {self._room_name}. Reason: {reason}")
await self._callbacks.on_disconnected()
async def _process_audio_stream(self, audio_stream: rtc.AudioStream, participant_id: str):
logger.info(f"Started processing audio stream for participant {participant_id}")
async for event in audio_stream:
if isinstance(event, rtc.AudioFrameEvent):
await self._audio_queue.put((event, participant_id))
else:
logger.warning(f"Received unexpected event type: {type(event)}")
async def cleanup(self):
await self.disconnect()
async def get_next_audio_frame(self):
frame, participant_id = await self._audio_queue.get()
return frame, participant_id
class LiveKitInputTransport(BaseInputTransport):
def __init__(self, client: LiveKitTransportClient, params: LiveKitParams, **kwargs):
super().__init__(params, **kwargs)
self._client = client
self._audio_in_task = None
self._vad_analyzer: VADAnalyzer | None = params.vad_analyzer
self._current_sample_rate: int = params.audio_in_sample_rate
if params.vad_enabled and not params.vad_analyzer:
self._vad_analyzer = VADAnalyzer(
sample_rate=self._current_sample_rate, num_channels=self._params.audio_in_channels
)
async def start(self, frame: StartFrame):
await super().start(frame)
await self._client.connect()
if self._params.audio_in_enabled or self._params.vad_enabled:
self._audio_in_task = asyncio.create_task(self._audio_in_task_handler())
logger.info("LiveKitInputTransport started")
async def stop(self, frame: EndFrame):
if self._audio_in_task:
self._audio_in_task.cancel()
try:
await self._audio_in_task
except asyncio.CancelledError:
pass
await super().stop(frame)
await self._client.disconnect()
logger.info("LiveKitInputTransport stopped")
async def process_frame(self, frame: Frame, direction: FrameDirection):
if isinstance(frame, EndFrame):
await self.stop(frame)
else:
await super().process_frame(frame, direction)
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._client.disconnect()
if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
self._audio_in_task.cancel()
await self._audio_in_task
def vad_analyzer(self) -> VADAnalyzer | None:
return self._vad_analyzer
async def push_app_message(self, message: Any, sender: str):
frame = LiveKitTransportMessageFrame(message=message, participant_id=sender)
await self.push_frame(frame)
async def _audio_in_task_handler(self):
logger.info("Audio input task started")
while True:
try:
audio_data = await self._client.get_next_audio_frame()
if audio_data:
audio_frame_event, participant_id = audio_data
pipecat_audio_frame = self._convert_livekit_audio_to_pipecat(audio_frame_event)
await self.push_audio_frame(pipecat_audio_frame)
await self.push_frame(
pipecat_audio_frame
) # TODO: ensure audio frames are pushed with the default BaseInputTransport.push_audio_frame()
except asyncio.CancelledError:
logger.info("Audio input task cancelled")
break
except Exception as e:
logger.error(f"Error in audio input task: {e}")
def _convert_livekit_audio_to_pipecat(
self, audio_frame_event: rtc.AudioFrameEvent
) -> AudioRawFrame:
audio_frame = audio_frame_event.frame
audio_data = np.frombuffer(audio_frame.data, dtype=np.int16)
original_sample_rate = audio_frame.sample_rate
# Allow 8kHz and 16kHz, convert anything else to 16kHz
if original_sample_rate not in [8000, 16000]:
audio_data = self._resample_audio(audio_data, original_sample_rate, 16000)
sample_rate = 16000
else:
sample_rate = original_sample_rate
if sample_rate != self._current_sample_rate:
self._current_sample_rate = sample_rate
self._vad_analyzer = VADAnalyzer(
sample_rate=self._current_sample_rate, num_channels=self._params.audio_in_channels
)
return AudioRawFrame(
audio=audio_data.tobytes(),
sample_rate=sample_rate,
num_channels=audio_frame.num_channels,
)
def _resample_audio(
self, audio_data: np.ndarray, original_rate: int, target_rate: int
) -> np.ndarray:
num_samples = int(len(audio_data) * target_rate / original_rate)
resampled_audio = signal.resample(audio_data, num_samples)
return resampled_audio.astype(np.int16)
class LiveKitOutputTransport(BaseOutputTransport):
def __init__(self, client: LiveKitTransportClient, params: LiveKitParams, **kwargs):
super().__init__(params, **kwargs)
self._client = client
async def start(self, frame: StartFrame):
await super().start(frame)
await self._client.connect()
logger.info("LiveKitOutputTransport started")
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._client.disconnect()
logger.info("LiveKitOutputTransport stopped")
async def process_frame(self, frame: Frame, direction: FrameDirection):
if isinstance(frame, EndFrame):
await self.stop(frame)
else:
await super().process_frame(frame, direction)
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._client.disconnect()
async def send_message(self, frame: TransportMessageFrame):
if isinstance(frame, LiveKitTransportMessageFrame):
await self._client.send_data(frame.message.encode(), frame.participant_id)
else:
await self._client.send_data(frame.message.encode())
async def send_metrics(self, frame: MetricsFrame):
metrics = {}
for d in frame.data:
if isinstance(d, TTFBMetricsData):
if "ttfb" not in metrics:
metrics["ttfb"] = []
metrics["ttfb"].append(d.model_dump())
elif isinstance(d, ProcessingMetricsData):
if "processing" not in metrics:
metrics["processing"] = []
metrics["processing"].append(d.model_dump())
elif isinstance(d, LLMUsageMetricsData):
if "tokens" not in metrics:
metrics["tokens"] = []
metrics["tokens"].append(d.value.model_dump(exclude_none=True))
elif isinstance(d, TTSUsageMetricsData):
if "characters" not in metrics:
metrics["characters"] = []
metrics["characters"].append(d.model_dump())
message = LiveKitTransportMessageFrame(
message={"type": "pipecat-metrics", "metrics": metrics}
)
await self._client.send_data(str(message.message).encode())
async def write_raw_audio_frames(self, frames: bytes):
livekit_audio = self._convert_pipecat_audio_to_livekit(frames)
await self._client.publish_audio(livekit_audio)
def _convert_pipecat_audio_to_livekit(self, pipecat_audio: bytes) -> rtc.AudioFrame:
bytes_per_sample = 2 # Assuming 16-bit audio
total_samples = len(pipecat_audio) // bytes_per_sample
samples_per_channel = total_samples // self._params.audio_out_channels
return rtc.AudioFrame(
data=pipecat_audio,
sample_rate=self._params.audio_out_sample_rate,
num_channels=self._params.audio_out_channels,
samples_per_channel=samples_per_channel,
)
class LiveKitTransport(BaseTransport):
def __init__(
self,
url: str,
token: str,
room_name: str,
params: LiveKitParams = LiveKitParams(),
input_name: str | None = None,
output_name: str | None = None,
loop: asyncio.AbstractEventLoop | None = None,
):
super().__init__(input_name=input_name, output_name=output_name, loop=loop)
self._url = url
self._token = token
self._room_name = room_name
self._params = params
self._client = LiveKitTransportClient(
url, token, room_name, self._params, self._create_callbacks(), self._loop
)
self._input: LiveKitInputTransport | None = None
self._output: LiveKitOutputTransport | None = None
self._register_event_handler("on_connected")
self._register_event_handler("on_disconnected")
self._register_event_handler("on_participant_connected")
self._register_event_handler("on_participant_disconnected")
self._register_event_handler("on_audio_track_subscribed")
self._register_event_handler("on_audio_track_unsubscribed")
self._register_event_handler("on_data_received")
self._register_event_handler("on_first_participant_joined")
self._register_event_handler("on_participant_left")
self._register_event_handler("on_call_state_updated")
def _create_callbacks(self) -> LiveKitCallbacks:
return LiveKitCallbacks(
on_connected=self._on_connected,
on_disconnected=self._on_disconnected,
on_participant_connected=self._on_participant_connected,
on_participant_disconnected=self._on_participant_disconnected,
on_audio_track_subscribed=self._on_audio_track_subscribed,
on_audio_track_unsubscribed=self._on_audio_track_unsubscribed,
on_data_received=self._on_data_received,
)
def input(self) -> FrameProcessor:
if not self._input:
self._input = LiveKitInputTransport(self._client, self._params, name=self._input_name)
return self._input
def output(self) -> FrameProcessor:
if not self._output:
self._output = LiveKitOutputTransport(
self._client, self._params, name=self._output_name
)
return self._output
@property
def participant_id(self) -> str:
return self._client.participant_id
async def send_audio(self, frame: AudioRawFrame):
if self._output:
await self._output.process_frame(frame, FrameDirection.DOWNSTREAM)
def get_participants(self) -> List[str]:
return self._client.get_participants()
async def get_participant_metadata(self, participant_id: str) -> dict:
return await self._client.get_participant_metadata(participant_id)
async def set_metadata(self, metadata: str):
await self._client.set_participant_metadata(metadata)
async def mute_participant(self, participant_id: str):
await self._client.mute_participant(participant_id)
async def unmute_participant(self, participant_id: str):
await self._client.unmute_participant(participant_id)
async def _on_connected(self):
await self._call_event_handler("on_connected")
async def _on_disconnected(self):
await self._call_event_handler("on_disconnected")
# Attempt to reconnect
try:
await self._client.connect()
await self._call_event_handler("on_connected")
except Exception as e:
logger.error(f"Failed to reconnect: {e}")
async def _on_participant_connected(self, participant_id: str):
await self._call_event_handler("on_participant_connected", participant_id)
if len(self.get_participants()) == 1:
await self._call_event_handler("on_first_participant_joined", participant_id)
async def _on_participant_disconnected(self, participant_id: str):
await self._call_event_handler("on_participant_disconnected", participant_id)
await self._call_event_handler("on_participant_left", participant_id, "disconnected")
if self._input:
await self._input.process_frame(EndFrame(), FrameDirection.DOWNSTREAM)
if self._output:
await self._output.process_frame(EndFrame(), FrameDirection.DOWNSTREAM)
async def _on_audio_track_subscribed(self, participant_id: str):
await self._call_event_handler("on_audio_track_subscribed", participant_id)
participant = self._client._room.remote_participants.get(participant_id)
if participant:
for publication in participant.audio_tracks.values():
self._client._on_track_subscribed_wrapper(
publication.track, publication, participant
)
async def _on_audio_track_unsubscribed(self, participant_id: str):
await self._call_event_handler("on_audio_track_unsubscribed", participant_id)
async def _on_data_received(self, data: bytes, participant_id: str):
if self._input:
await self._input.push_app_message(data.decode(), participant_id)
await self._call_event_handler("on_data_received", data, participant_id)
async def send_message(self, message: str, participant_id: str | None = None):
if self._output:
frame = LiveKitTransportMessageFrame(message=message, participant_id=participant_id)
await self._output.send_message(frame)
async def cleanup(self):
if self._input:
await self._input.cleanup()
if self._output:
await self._output.cleanup()
await self._client.disconnect()
async def on_room_event(self, event):
# Handle room events
pass
async def on_participant_event(self, event):
# Handle participant events
pass
async def on_track_event(self, event):
# Handle track events
pass
async def _on_call_state_updated(self, state: str):
await self._call_event_handler("on_call_state_updated", self, state)

9
src/pipecat/utils/string.py
View File

@@ -6,7 +6,6 @@
import re
ENDOFSENTENCE_PATTERN_STR = r"""
(?<![A-Z]) # Negative lookbehind: not preceded by an uppercase letter (e.g., "U.S.A.")
(?<!\d) # Negative lookbehind: not preceded by a digit (e.g., "1. Let's start")
@@ -14,11 +13,13 @@ ENDOFSENTENCE_PATTERN_STR = r"""
(?<!Mr|Ms|Dr) # Negative lookbehind: not preceded by Mr, Ms, Dr (combined bc. length is the same)
(?<!Mrs) # Negative lookbehind: not preceded by "Mrs"
(?<!Prof) # Negative lookbehind: not preceded by "Prof"
[\.\?\!:] # Match a period, question mark, exclamation point, or colon
[\.\?\!:;]| # Match a period, question mark, exclamation point, colon, or semicolon
[。？！：；] # the full-width version (mainly used in East Asian languages such as Chinese)
$ # End of string
"""
ENDOFSENTENCE_PATTERN = re.compile(ENDOFSENTENCE_PATTERN_STR, re.VERBOSE)
def match_endofsentence(text: str) -> bool:
return ENDOFSENTENCE_PATTERN.search(text.rstrip()) is not None
def match_endofsentence(text: str) -> int:
match = ENDOFSENTENCE_PATTERN.search(text.rstrip())
return match.end() if match else 0

4
src/pipecat/utils/test_frame_processor.py
View File

@@ -15,7 +15,9 @@ class TestFrameProcessor(FrameProcessor):
async def process_frame(self, frame, direction):
await super().process_frame(frame, direction)
if not self.test_frames[0]: # then we've run out of required frames but the generator is still going?
if not self.test_frames[
0
]: # then we've run out of required frames but the generator is still going?
raise TestException(f"Oops, got an extra frame, {frame}")
if isinstance(self.test_frames[0], List):
# We need to consume frames until we see the next frame type after this

17
src/pipecat/utils/time.py
View File

@@ -9,3 +9,20 @@ import datetime
def time_now_iso8601() -> str:
return datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="milliseconds")
def seconds_to_nanoseconds(seconds: float) -> int:
return int(seconds * 1_000_000_000)
def nanoseconds_to_seconds(nanoseconds: int) -> float:
return nanoseconds / 1_000_000_000
def nanoseconds_to_str(nanoseconds: int) -> str:
total_seconds = nanoseconds_to_seconds(nanoseconds)
hours = int(total_seconds // 3600)
minutes = int((total_seconds % 3600) // 60)
seconds = int(total_seconds % 60)
microseconds = int((total_seconds - int(total_seconds)) * 1_000_000)
return f"{hours}:{minutes:02}:{seconds:02}.{microseconds:06}"

0
src/pipecat/vad/data/__init__.py Normal file
View File

BIN
src/pipecat/vad/data/silero_vad.onnx Normal file
View File

Binary file not shown.

143
src/pipecat/vad/silero.py
View File

@@ -8,32 +8,111 @@ import time
import numpy as np
from pipecat.frames.frames import AudioRawFrame, Frame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame
from pipecat.frames.frames import (
AudioRawFrame,
Frame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.vad.vad_analyzer import VADAnalyzer, VADParams, VADState
from loguru import logger
# How often should we reset internal model state
_MODEL_RESET_STATES_TIME = 5.0
try:
from silero_vad import load_silero_vad
import torch
import onnxruntime
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use Silero VAD, you need to `pip install pipecat-ai[silero]`.")
raise Exception(f"Missing module(s): {e}")
# How often should we reset internal model state
_MODEL_RESET_STATES_TIME = 5.0
class SileroOnnxModel:
def __init__(self, path, force_onnx_cpu=True):
import numpy as np
global np
opts = onnxruntime.SessionOptions()
opts.inter_op_num_threads = 1
opts.intra_op_num_threads = 1
if force_onnx_cpu and "CPUExecutionProvider" in onnxruntime.get_available_providers():
self.session = onnxruntime.InferenceSession(
path, providers=["CPUExecutionProvider"], sess_options=opts
)
else:
self.session = onnxruntime.InferenceSession(path, sess_options=opts)
self.reset_states()
self.sample_rates = [8000, 16000]
def _validate_input(self, x, sr: int):
if np.ndim(x) == 1:
x = np.expand_dims(x, 0)
if np.ndim(x) > 2:
raise ValueError(f"Too many dimensions for input audio chunk {x.dim()}")
if sr not in self.sample_rates:
raise ValueError(
f"Supported sampling rates: {self.sample_rates} (or multiply of 16000)"
)
if sr / np.shape(x)[1] > 31.25:
raise ValueError("Input audio chunk is too short")
return x, sr
def reset_states(self, batch_size=1):
self._state = np.zeros((2, batch_size, 128), dtype="float32")
self._context = np.zeros((batch_size, 0), dtype="float32")
self._last_sr = 0
self._last_batch_size = 0
def __call__(self, x, sr: int):
x, sr = self._validate_input(x, sr)
num_samples = 512 if sr == 16000 else 256
if np.shape(x)[-1] != num_samples:
raise ValueError(
f"Provided number of samples is {np.shape(x)[-1]} (Supported values: 256 for 8000 sample rate, 512 for 16000)"
)
batch_size = np.shape(x)[0]
context_size = 64 if sr == 16000 else 32
if not self._last_batch_size:
self.reset_states(batch_size)
if (self._last_sr) and (self._last_sr != sr):
self.reset_states(batch_size)
if (self._last_batch_size) and (self._last_batch_size != batch_size):
self.reset_states(batch_size)
if not np.shape(self._context)[1]:
self._context = np.zeros((batch_size, context_size), dtype="float32")
x = np.concatenate((self._context, x), axis=1)
if sr in [8000, 16000]:
ort_inputs = {"input": x, "state": self._state, "sr": np.array(sr, dtype="int64")}
ort_outs = self.session.run(None, ort_inputs)
out, state = ort_outs
self._state = state
else:
raise ValueError()
self._context = x[..., -context_size:]
self._last_sr = sr
self._last_batch_size = batch_size
return out
class SileroVADAnalyzer(VADAnalyzer):
def __init__(
self,
*,
sample_rate: int = 16000,
params: VADParams = VADParams()):
def __init__(self, *, sample_rate: int = 16000, params: VADParams = VADParams()):
super().__init__(sample_rate=sample_rate, num_channels=1, params=params)
if sample_rate != 16000 and sample_rate != 8000:
@@ -41,7 +120,23 @@ class SileroVADAnalyzer(VADAnalyzer):
logger.debug("Loading Silero VAD model...")
self._model = load_silero_vad()
model_name = "silero_vad.onnx"
package_path = "pipecat.vad.data"
try:
import importlib_resources as impresources
model_file_path = str(impresources.files(package_path).joinpath(model_name))
except BaseException:
from importlib import resources as impresources
try:
with impresources.path(package_path, model_name) as f:
model_file_path = f
except BaseException:
model_file_path = str(impresources.files(package_path).joinpath(model_name))
self._model = SileroOnnxModel(model_file_path, force_onnx_cpu=True)
self._last_reset_time = 0
@@ -59,7 +154,7 @@ class SileroVADAnalyzer(VADAnalyzer):
audio_int16 = np.frombuffer(buffer, np.int16)
# Divide by 32768 because we have signed 16-bit data.
audio_float32 = np.frombuffer(audio_int16, dtype=np.int16).astype(np.float32) / 32768.0
new_confidence = self._model(torch.from_numpy(audio_float32), self.sample_rate).item()
new_confidence = self._model(audio_float32, self.sample_rate)[0]
# We need to reset the model from time to time because it doesn't
# really need all the data and memory will keep growing otherwise.
@@ -77,18 +172,16 @@ class SileroVADAnalyzer(VADAnalyzer):
class SileroVAD(FrameProcessor):
def __init__(
self,
*,
sample_rate: int = 16000,
vad_params: VADParams = VADParams(),
audio_passthrough: bool = False):
self,
*,
sample_rate: int = 16000,
vad_params: VADParams = VADParams(),
audio_passthrough: bool = False,
):
super().__init__()
self._vad_analyzer = SileroVADAnalyzer(
sample_rate=sample_rate,
params=vad_params)
self._vad_analyzer = SileroVADAnalyzer(sample_rate=sample_rate, params=vad_params)
self._audio_passthrough = audio_passthrough
self._processor_vad_state: VADState = VADState.QUIET
@@ -111,7 +204,11 @@ class SileroVAD(FrameProcessor):
# Check VAD and push event if necessary. We just care about changes
# from QUIET to SPEAKING and vice versa.
new_vad_state = self._vad_analyzer.analyze_audio(frame.audio)
if new_vad_state != self._processor_vad_state and new_vad_state != VADState.STARTING and new_vad_state != VADState.STOPPING:
if (
new_vad_state != self._processor_vad_state
and new_vad_state != VADState.STARTING
and new_vad_state != VADState.STOPPING
):
new_frame = None
if new_vad_state == VADState.SPEAKING:

1
src/pipecat/vad/vad_analyzer.py
View File

@@ -29,7 +29,6 @@ class VADParams(BaseModel):
class VADAnalyzer:
def __init__(self, *, sample_rate: int, num_channels: int, params: VADParams):
self._sample_rate = sample_rate
self._num_channels = num_channels