wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'pipecat-ai:main' into main

Browse Source
This commit is contained in:
fatwang2
2025-02-06 10:50:25 +08:00
committed by GitHub
parent fc90bdc638 99d3227ff5
commit 8cda4512ad
233 changed files with 9718 additions and 2265 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/pipecat/audio/filters/krisp_filter.py
View File

@@ -20,6 +20,22 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
class KrispProcessorManager:
"""
Ensures that only one KrispAudioProcessor instance exists for the entire program.
"""
_krisp_instance = None
@classmethod
def get_processor(cls, sample_rate: int, sample_type: str, channels: int, model_path: str):
if cls._krisp_instance is None:
cls._krisp_instance = KrispAudioProcessor(
sample_rate, sample_type, channels, model_path
)
return cls._krisp_instance
class KrispFilter(BaseAudioFilter):
def __init__(
self, sample_type: str = "PCM_16", channels: int = 1, model_path: str = None
@@ -48,7 +64,7 @@ class KrispFilter(BaseAudioFilter):
async def start(self, sample_rate: int):
self._sample_rate = sample_rate
self._krisp_processor = KrispAudioProcessor(
self._krisp_processor = KrispProcessorManager.get_processor(
self._sample_rate, self._sample_type, self._channels, self._model_path
)

1
src/pipecat/audio/resamplers/__init__.py Normal file
View File

@@ -0,0 +1 @@

30
src/pipecat/audio/resamplers/base_audio_resampler.py Normal file
View File

@@ -0,0 +1,30 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from abc import ABC, abstractmethod
class BaseAudioResampler(ABC):
"""Abstract base class for audio resampling. This class defines an
interface for audio resampling implementations.
"""
@abstractmethod
async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes:
"""
Resamples the given audio data to a different sample rate.
This is an abstract method that must be implemented in subclasses.
Parameters:
audio (bytes): The audio data to be resampled, represented as a byte string.
in_rate (int): The original sample rate of the audio data (in Hz).
out_rate (int): The desired sample rate for the resampled audio data (in Hz).
Returns:
bytes: The resampled audio data as a byte string.
"""
pass

25
src/pipecat/audio/resamplers/resampy_resampler.py Normal file
View File

@@ -0,0 +1,25 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import numpy as np
import resampy
from pipecat.audio.resamplers.base_audio_resampler import BaseAudioResampler
class ResampyResampler(BaseAudioResampler):
"""Audio resampler implementation using the resampy library."""
def __init__(self, **kwargs):
pass
async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes:
if in_rate == out_rate:
return audio
audio_data = np.frombuffer(audio, dtype=np.int16)
resampled_audio = resampy.resample(audio_data, in_rate, out_rate, filter="kaiser_fast")
result = resampled_audio.astype(np.int16).tobytes()
return result

25
src/pipecat/audio/resamplers/soxr_resampler.py Normal file
View File

@@ -0,0 +1,25 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import numpy as np
import soxr
from pipecat.audio.resamplers.base_audio_resampler import BaseAudioResampler
class SOXRAudioResampler(BaseAudioResampler):
"""Audio resampler implementation using the SoX resampler library."""
def __init__(self, **kwargs):
pass
async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes:
if in_rate == out_rate:
return audio
audio_data = np.frombuffer(audio, dtype=np.int16)
resampled_audio = soxr.resample(audio_data, in_rate, out_rate, quality="VHQ")
result = resampled_audio.astype(np.int16).tobytes()
return result

56
src/pipecat/audio/utils.py
View File

@@ -8,14 +8,30 @@ import audioop
import numpy as np
import pyloudnorm as pyln
import resampy
import soxr
from pipecat.audio.resamplers.base_audio_resampler import BaseAudioResampler
from pipecat.audio.resamplers.soxr_resampler import SOXRAudioResampler
def create_default_resampler(**kwargs) -> BaseAudioResampler:
return SOXRAudioResampler(**kwargs)
def resample_audio(audio: bytes, original_rate: int, target_rate: int) -> bytes:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("always")
warnings.warn(
"'resample_audio()' is deprecated, use 'create_default_resampler()' instead.",
DeprecationWarning,
)
if original_rate == target_rate:
return audio
audio_data = np.frombuffer(audio, dtype=np.int16)
resampled_audio = resampy.resample(audio_data, original_rate, target_rate)
resampled_audio = soxr.resample(audio_data, original_rate, target_rate)
return resampled_audio.astype(np.int16).tobytes()
@@ -75,21 +91,45 @@ def exp_smoothing(value: float, prev_value: float, factor: float) -> float:
return prev_value + factor * (value - prev_value)
def ulaw_to_pcm(ulaw_bytes: bytes, in_sample_rate: int, out_sample_rate: int):
async def ulaw_to_pcm(
ulaw_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler
):
# Convert μ-law to PCM
in_pcm_bytes = audioop.ulaw2lin(ulaw_bytes, 2)
# Resample
out_pcm_bytes = resample_audio(in_pcm_bytes, in_sample_rate, out_sample_rate)
out_pcm_bytes = await resampler.resample(in_pcm_bytes, in_rate, out_rate)
return out_pcm_bytes
def pcm_to_ulaw(pcm_bytes: bytes, in_sample_rate: int, out_sample_rate: int):
async def pcm_to_ulaw(pcm_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler):
# Resample
in_pcm_bytes = resample_audio(pcm_bytes, in_sample_rate, out_sample_rate)
in_pcm_bytes = await resampler.resample(pcm_bytes, in_rate, out_rate)
# Convert PCM to μ-law
ulaw_bytes = audioop.lin2ulaw(in_pcm_bytes, 2)
out_ulaw_bytes = audioop.lin2ulaw(in_pcm_bytes, 2)
return ulaw_bytes
return out_ulaw_bytes
async def alaw_to_pcm(
alaw_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler
) -> bytes:
# Convert a-law to PCM
in_pcm_bytes = audioop.alaw2lin(alaw_bytes, 2)
# Resample
out_pcm_bytes = await resampler.resample(in_pcm_bytes, in_rate, out_rate)
return out_pcm_bytes
async def pcm_to_alaw(pcm_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler):
# Resample
in_pcm_bytes = await resampler.resample(pcm_bytes, in_rate, out_rate)
# Convert PCM to μ-law
out_alaw_bytes = audioop.lin2alaw(in_pcm_bytes, 2)
return out_alaw_bytes

16
src/pipecat/audio/vad/silero.py
View File

@@ -5,6 +5,7 @@
#
import time
from typing import Optional
import numpy as np
from loguru import logger
@@ -104,11 +105,8 @@ class SileroOnnxModel:
class SileroVADAnalyzer(VADAnalyzer):
def __init__(self, *, sample_rate: int = 16000, params: VADParams = VADParams()):
super().__init__(sample_rate=sample_rate, num_channels=1, params=params)
if sample_rate != 16000 and sample_rate != 8000:
raise ValueError("Silero VAD sample rate needs to be 16000 or 8000")
def __init__(self, *, sample_rate: Optional[int] = None, params: VADParams = VADParams()):
super().__init__(sample_rate=sample_rate, params=params)
logger.debug("Loading Silero VAD model...")
@@ -138,6 +136,12 @@ class SileroVADAnalyzer(VADAnalyzer):
# VADAnalyzer
#
def set_sample_rate(self, sample_rate: int):
if sample_rate != 16000 and sample_rate != 8000:
raise ValueError("Silero VAD sample rate needs to be 16000 or 8000")
super().set_sample_rate(sample_rate)
def num_frames_required(self) -> int:
return 512 if self.sample_rate == 16000 else 256
@@ -159,5 +163,5 @@ class SileroVADAnalyzer(VADAnalyzer):
return new_confidence
except Exception as e:
# This comes from an empty audio array
logger.exception(f"Error analyzing audio with Silero VAD: {e}")
logger.error(f"Error analyzing audio with Silero VAD: {e}")
return 0

19
src/pipecat/audio/vad/vad_analyzer.py
View File

@@ -6,6 +6,7 @@
from abc import abstractmethod
from enum import Enum
from typing import Optional
from loguru import logger
from pydantic import BaseModel
@@ -33,11 +34,11 @@ class VADParams(BaseModel):
class VADAnalyzer:
def __init__(self, *, sample_rate: int, num_channels: int, params: VADParams):
self._sample_rate = sample_rate
self._num_channels = num_channels
self.set_params(params)
def __init__(self, *, sample_rate: Optional[int] = None, params: VADParams):
self._init_sample_rate = sample_rate
self._sample_rate = 0
self._params = params
self._num_channels = 1
self._vad_buffer = b""
@@ -65,13 +66,17 @@ class VADAnalyzer:
def voice_confidence(self, buffer) -> float:
pass
def set_sample_rate(self, sample_rate: int):
self._sample_rate = self._init_sample_rate or sample_rate
self.set_params(self._params)
def set_params(self, params: VADParams):
logger.info(f"Setting VAD params to: {params}")
self._params = params
self._vad_frames = self.num_frames_required()
self._vad_frames_num_bytes = self._vad_frames * self._num_channels * 2
vad_frames_per_sec = self._vad_frames / self._sample_rate
vad_frames_per_sec = self._vad_frames / self.sample_rate
self._vad_start_frames = round(self._params.start_secs / vad_frames_per_sec)
self._vad_stop_frames = round(self._params.stop_secs / vad_frames_per_sec)
@@ -80,7 +85,7 @@ class VADAnalyzer:
self._vad_state: VADState = VADState.QUIET
def _get_smoothed_volume(self, audio: bytes) -> float:
volume = calculate_audio_volume(audio, self._sample_rate)
volume = calculate_audio_volume(audio, self.sample_rate)
return exp_smoothing(volume, self._prev_volume, self._smoothing_factor)
def analyze_audio(self, buffer) -> VADState:

39
src/pipecat/frames/frames.py
View File

@@ -6,12 +6,24 @@
from dataclasses import dataclass, field
from enum import Enum
from typing import TYPE_CHECKING, Any, Awaitable, Callable, List, Literal, Mapping, Optional, Tuple
from typing import (
TYPE_CHECKING,
Any,
Awaitable,
Callable,
Dict,
List,
Literal,
Mapping,
Optional,
Tuple,
)
from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.clocks.base_clock import BaseClock
from pipecat.metrics.metrics import MetricsData
from pipecat.transcriptions.language import Language
from pipecat.utils.asyncio import TaskManager
from pipecat.utils.time import nanoseconds_to_str
from pipecat.utils.utils import obj_count, obj_id
@@ -47,11 +59,13 @@ class Frame:
id: int = field(init=False)
name: str = field(init=False)
pts: Optional[int] = field(init=False)
metadata: Dict[str, Any] = field(init=False)
def __post_init__(self):
self.id: int = obj_id()
self.name: str = f"{self.__class__.__name__}#{obj_count(self)}"
self.pts: Optional[int] = None
self.metadata: Dict[str, Any] = {}
def __str__(self):
return self.name
@@ -394,12 +408,26 @@ class TransportMessageFrame(DataFrame):
@dataclass
class InputDTMFFrame(DataFrame):
"""A DTMF button input"""
class DTMFFrame(DataFrame):
"""A DTMF button frame"""
button: KeypadEntry
@dataclass
class InputDTMFFrame(DTMFFrame):
"""A DTMF button input"""
pass
@dataclass
class OutputDTMFFrame(DTMFFrame):
"""A DTMF button output"""
pass
#
# System frames
#
@@ -410,11 +438,14 @@ class StartFrame(SystemFrame):
"""This is the first frame that should be pushed down a pipeline."""
clock: BaseClock
task_manager: TaskManager
audio_in_sample_rate: int = 16000
audio_out_sample_rate: int = 24000
allow_interruptions: bool = False
enable_metrics: bool = False
enable_usage_metrics: bool = False
report_only_initial_ttfb: bool = False
observer: Optional["BaseObserver"] = None
report_only_initial_ttfb: bool = False
@dataclass

66
src/pipecat/pipeline/base_task.py Normal file
View File

@@ -0,0 +1,66 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
from abc import ABC, abstractmethod
from typing import AsyncIterable, Iterable
from pipecat.frames.frames import Frame
class BaseTask(ABC):
@property
@abstractmethod
def id(self) -> int:
"""Returns the unique indetifier for this task."""
pass
@property
@abstractmethod
def name(self) -> str:
"""Returns the name of this task."""
pass
@abstractmethod
def set_event_loop(self, loop: asyncio.AbstractEventLoop):
"""Sets the event loop that this task will run on."""
pass
@abstractmethod
def has_finished(self) -> bool:
"""Indicates whether the tasks has finished. That is, all processors
have stopped.
"""
pass
@abstractmethod
async def stop_when_done(self):
"""This is a helper function that sends an EndFrame to the pipeline in
order to stop the task after everything in it has been processed.
"""
pass
@abstractmethod
async def cancel(self):
"""Stops the running pipeline immediately."""
pass
@abstractmethod
async def run(self):
"""Starts running the given pipeline."""
pass
@abstractmethod
async def queue_frame(self, frame: Frame):
"""Queue a frame to be pushed down the pipeline."""
pass
@abstractmethod
async def queue_frames(self, frames: Iterable[Frame] | AsyncIterable[Frame]):
"""Queues multiple frames to be pushed down the pipeline."""
pass

59
src/pipecat/pipeline/parallel_pipeline.py
View File

@@ -23,7 +23,7 @@ from pipecat.pipeline.pipeline import Pipeline
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class Source(FrameProcessor):
class ParallelPipelineSource(FrameProcessor):
def __init__(
self,
upstream_queue: asyncio.Queue,
@@ -46,7 +46,7 @@ class Source(FrameProcessor):
await self.push_frame(frame, direction)
class Sink(FrameProcessor):
class ParallelPipelineSink(FrameProcessor):
def __init__(
self,
downstream_queue: asyncio.Queue,
@@ -92,8 +92,8 @@ class ParallelPipeline(BasePipeline):
raise TypeError(f"ParallelPipeline argument {processors} is not a list")
# We will add a source before the pipeline and a sink after.
source = Source(self._up_queue, self._parallel_push_frame)
sink = Sink(self._down_queue, self._parallel_push_frame)
source = ParallelPipelineSource(self._up_queue, self._parallel_push_frame)
sink = ParallelPipelineSink(self._down_queue, self._parallel_push_frame)
self._sources.append(source)
self._sinks.append(sink)
@@ -117,6 +117,7 @@ class ParallelPipeline(BasePipeline):
#
async def cleanup(self):
await super().cleanup()
await asyncio.gather(*[s.cleanup() for s in self._sources])
await asyncio.gather(*[p.cleanup() for p in self._pipelines])
await asyncio.gather(*[s.cleanup() for s in self._sinks])
@@ -150,22 +151,18 @@ class ParallelPipeline(BasePipeline):
async def _stop(self):
# The up task doesn't receive an EndFrame, so we just cancel it.
self._up_task.cancel()
await self._up_task
# The down tasks waits for the last EndFrame send by the internal
await self.cancel_task(self._up_task)
# The down tasks waits for the last EndFrame sent by the internal
# pipelines.
await self._down_task
async def _cancel(self):
self._up_task.cancel()
await self._up_task
self._down_task.cancel()
await self._down_task
await self.cancel_task(self._up_task)
await self.cancel_task(self._down_task)
async def _create_tasks(self):
loop = self.get_event_loop()
self._up_task = loop.create_task(self._process_up_queue())
self._down_task = loop.create_task(self._process_down_queue())
self._up_task = self.create_task(self._process_up_queue())
self._down_task = self.create_task(self._process_down_queue())
async def _drain_queues(self):
while not self._up_queue.empty:
@@ -185,32 +182,26 @@ class ParallelPipeline(BasePipeline):
async def _process_up_queue(self):
while True:
try:
frame = await self._up_queue.get()
await self._parallel_push_frame(frame, FrameDirection.UPSTREAM)
self._up_queue.task_done()
except asyncio.CancelledError:
break
frame = await self._up_queue.get()
await self._parallel_push_frame(frame, FrameDirection.UPSTREAM)
self._up_queue.task_done()
async def _process_down_queue(self):
running = True
while running:
try:
frame = await self._down_queue.get()
frame = await self._down_queue.get()
endframe_counter = self._endframe_counter.get(frame.id, 0)
endframe_counter = self._endframe_counter.get(frame.id, 0)
# If we have a counter, decrement it.
if endframe_counter > 0:
self._endframe_counter[frame.id] -= 1
endframe_counter = self._endframe_counter[frame.id]
# If we have a counter, decrement it.
if endframe_counter > 0:
self._endframe_counter[frame.id] -= 1
endframe_counter = self._endframe_counter[frame.id]
# If we don't have a counter or we reached 0, push the frame.
if endframe_counter == 0:
await self._parallel_push_frame(frame, FrameDirection.DOWNSTREAM)
# If we don't have a counter or we reached 0, push the frame.
if endframe_counter == 0:
await self._parallel_push_frame(frame, FrameDirection.DOWNSTREAM)
running = not (endframe_counter == 0 and isinstance(frame, EndFrame))
running = not (endframe_counter == 0 and isinstance(frame, EndFrame))
self._down_queue.task_done()
except asyncio.CancelledError:
break
self._down_queue.task_done()

1
src/pipecat/pipeline/pipeline.py
View File

@@ -71,6 +71,7 @@ class Pipeline(BasePipeline):
#
async def cleanup(self):
await super().cleanup()
await self._cleanup_processors()
async def process_frame(self, frame: Frame, direction: FrameDirection):

40
src/pipecat/pipeline/runner.py
View File

@@ -5,7 +5,9 @@
#
import asyncio
import gc
import signal
from typing import Optional
from loguru import logger
@@ -14,11 +16,21 @@ from pipecat.utils.utils import obj_count, obj_id
class PipelineRunner:
def __init__(self, *, name: str | None = None, handle_sigint: bool = True):
def __init__(
self,
*,
name: str | None = None,
handle_sigint: bool = True,
force_gc: bool = False,
loop: Optional[asyncio.AbstractEventLoop] = None,
):
self.id: int = obj_id()
self.name: str = name or f"{self.__class__.__name__}#{obj_count(self)}"
self._tasks = {}
self._sig_task = None
self._force_gc = force_gc
self._loop = loop or asyncio.get_running_loop()
if handle_sigint:
self._setup_sigint()
@@ -26,8 +38,15 @@ class PipelineRunner:
async def run(self, task: PipelineTask):
logger.debug(f"Runner {self} started running {task}")
self._tasks[task.name] = task
task.set_event_loop(self._loop)
await task.run()
del self._tasks[task.name]
# If we are cancelling through a signal, make sure we wait for it so
# everything gets cleaned up nicely.
if self._sig_task:
await self._sig_task
if self._force_gc:
self._gc_collect()
logger.debug(f"Runner {self} finished running {task}")
async def stop_when_done(self):
@@ -40,16 +59,21 @@ class PipelineRunner:
def _setup_sigint(self):
loop = asyncio.get_running_loop()
loop.add_signal_handler(
signal.SIGINT, lambda *args: asyncio.create_task(self._sig_handler())
)
loop.add_signal_handler(
signal.SIGTERM, lambda *args: asyncio.create_task(self._sig_handler())
)
loop.add_signal_handler(signal.SIGINT, lambda *args: self._sig_handler())
loop.add_signal_handler(signal.SIGTERM, lambda *args: self._sig_handler())
async def _sig_handler(self):
def _sig_handler(self):
if not self._sig_task:
self._sig_task = asyncio.create_task(self._sig_cancel())
async def _sig_cancel(self):
logger.warning(f"Interruption detected. Canceling runner {self}")
await self.cancel()
def _gc_collect(self):
collected = gc.collect()
logger.debug(f"Garbage collector: collected {collected} objects.")
logger.debug(f"Garbage collector: uncollectable objects {gc.garbage}")
def __str__(self):
return self.name

24
src/pipecat/pipeline/sync_parallel_pipeline.py
View File

@@ -24,7 +24,7 @@ class SyncFrame(ControlFrame):
pass
class Source(FrameProcessor):
class SyncParallelPipelineSource(FrameProcessor):
def __init__(self, upstream_queue: asyncio.Queue):
super().__init__()
self._up_queue = upstream_queue
@@ -39,7 +39,7 @@ class Source(FrameProcessor):
await self.push_frame(frame, direction)
class Sink(FrameProcessor):
class SyncParallelPipelineSink(FrameProcessor):
def __init__(self, downstream_queue: asyncio.Queue):
super().__init__()
self._down_queue = downstream_queue
@@ -76,18 +76,20 @@ class SyncParallelPipeline(BasePipeline):
# We add a source at the beginning of the pipeline and a sink at the end.
up_queue = asyncio.Queue()
down_queue = asyncio.Queue()
source = Source(up_queue)
sink = Sink(down_queue)
processors: List[FrameProcessor] = [source] + processors + [sink]
source = SyncParallelPipelineSource(up_queue)
sink = SyncParallelPipelineSink(down_queue)
# Create pipeline
pipeline = Pipeline(processors)
source.link(pipeline)
pipeline.link(sink)
self._pipelines.append(pipeline)
# Keep track of sources and sinks. We also keep the output queue of
# the source and the sinks so we can use it later.
self._sources.append({"processor": source, "queue": down_queue})
self._sinks.append({"processor": sink, "queue": up_queue})
# Create pipeline
pipeline = Pipeline(processors)
self._pipelines.append(pipeline)
logger.debug(f"Finished creating {self} pipelines")
#
@@ -101,6 +103,12 @@ class SyncParallelPipeline(BasePipeline):
# Frame processor
#
async def cleanup(self):
await super().cleanup()
await asyncio.gather(*[s["processor"].cleanup() for s in self._sources])
await asyncio.gather(*[p.cleanup() for p in self._pipelines])
await asyncio.gather(*[s["processor"].cleanup() for s in self._sinks])
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)

230
src/pipecat/pipeline/task.py
View File

@@ -5,7 +5,7 @@
#
import asyncio
from typing import AsyncIterable, Iterable, List
from typing import Any, AsyncIterable, Dict, Iterable, List
from loguru import logger
from pydantic import BaseModel, ConfigDict
@@ -27,8 +27,10 @@ from pipecat.frames.frames import (
from pipecat.metrics.metrics import ProcessingMetricsData, TTFBMetricsData
from pipecat.observers.base_observer import BaseObserver
from pipecat.pipeline.base_pipeline import BasePipeline
from pipecat.pipeline.base_task import BaseTask
from pipecat.pipeline.task_observer import TaskObserver
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.asyncio import TaskManager
from pipecat.utils.utils import obj_count, obj_id
HEARTBEAT_SECONDS = 1.0
@@ -39,15 +41,19 @@ class PipelineParams(BaseModel):
model_config = ConfigDict(arbitrary_types_allowed=True)
allow_interruptions: bool = False
audio_in_sample_rate: int = 16000
audio_out_sample_rate: int = 24000
enable_heartbeats: bool = False
enable_metrics: bool = False
enable_usage_metrics: bool = False
send_initial_empty_metrics: bool = True
report_only_initial_ttfb: bool = False
heartbeats_period_secs: float = HEARTBEAT_SECONDS
observers: List[BaseObserver] = []
report_only_initial_ttfb: bool = False
send_initial_empty_metrics: bool = True
start_metadata: Dict[str, Any] = {}
class Source(FrameProcessor):
class PipelineTaskSource(FrameProcessor):
"""This is the source processor that is linked at the beginning of the
pipeline given to the pipeline task. It allows us to easily push frames
downstream to the pipeline and also receive upstream frames coming from the
@@ -55,8 +61,8 @@ class Source(FrameProcessor):
"""
def __init__(self, up_queue: asyncio.Queue):
super().__init__()
def __init__(self, up_queue: asyncio.Queue, **kwargs):
super().__init__(**kwargs)
self._up_queue = up_queue
async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -69,15 +75,15 @@ class Source(FrameProcessor):
await self.push_frame(frame, direction)
class Sink(FrameProcessor):
class PipelineTaskSink(FrameProcessor):
"""This is the sink processor that is linked at the end of the pipeline
given to the pipeline task. It allows us to receive downstream frames and
act on them, for example, waiting to receive an EndFrame.
"""
def __init__(self, down_queue: asyncio.Queue):
super().__init__()
def __init__(self, down_queue: asyncio.Queue, **kwargs):
super().__init__(**kwargs)
self._down_queue = down_queue
async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -85,15 +91,15 @@ class Sink(FrameProcessor):
await self._down_queue.put(frame)
class PipelineTask:
class PipelineTask(BaseTask):
def __init__(
self,
pipeline: BasePipeline,
params: PipelineParams = PipelineParams(),
clock: BaseClock = SystemClock(),
):
self.id: int = obj_id()
self.name: str = f"{self.__class__.__name__}#{obj_count(self)}"
self._id: int = obj_id()
self._name: str = f"{self.__class__.__name__}#{obj_count(self)}"
self._pipeline = pipeline
self._clock = clock
@@ -113,15 +119,35 @@ class PipelineTask:
# down queue.
self._endframe_event = asyncio.Event()
self._source = Source(self._up_queue)
self._source = PipelineTaskSource(self._up_queue)
self._source.link(pipeline)
self._sink = Sink(self._down_queue)
self._sink = PipelineTaskSink(self._down_queue)
pipeline.link(self._sink)
self._observer = TaskObserver(params.observers)
self._task_manager = TaskManager()
def has_finished(self):
self._observer = TaskObserver(observers=params.observers, task_manager=self._task_manager)
@property
def id(self) -> int:
"""Returns the unique indetifier for this task."""
return self._id
@property
def name(self) -> str:
"""Returns the name of this task."""
return self._name
@property
def params(self) -> PipelineParams:
"""Returns the pipeline parameters of this task."""
return self._params
def set_event_loop(self, loop: asyncio.AbstractEventLoop):
self._task_manager.set_event_loop(loop)
def has_finished(self) -> bool:
"""Indicates whether the tasks has finished. That is, all processors
have stopped.
@@ -145,14 +171,27 @@ class PipelineTask:
# out-of-band from the main streaming task which is what we want since
# we want to cancel right away.
await self._source.push_frame(CancelFrame())
await self._cancel_tasks(True)
# Only cancel the push task. Everything else will be cancelled in run().
await self._task_manager.cancel_task(self._process_push_task)
async def run(self):
"""
Starts running the given pipeline.
"""
tasks = self._create_tasks()
await asyncio.gather(*tasks)
if self.has_finished():
return
try:
push_task = await self._create_tasks()
await self._task_manager.wait_for_task(push_task)
except asyncio.CancelledError:
# We are awaiting on the push task and it might be cancelled
# (e.g. Ctrl-C). This means we will get a CancelledError here as
# well, because you get a CancelledError in every place you are
# awaiting a task.
pass
await self._cancel_tasks()
await self._cleanup()
self._print_dangling_tasks()
self._finished = True
async def queue_frame(self, frame: Frame):
@@ -172,42 +211,42 @@ class PipelineTask:
for frame in frames:
await self.queue_frame(frame)
def _create_tasks(self):
tasks = []
self._process_up_task = asyncio.create_task(self._process_up_queue())
self._process_down_task = asyncio.create_task(self._process_down_queue())
self._process_push_task = asyncio.create_task(self._process_push_queue())
async def _create_tasks(self):
self._process_up_task = self._task_manager.create_task(
self._process_up_queue(), f"{self}::_process_up_queue"
)
self._process_down_task = self._task_manager.create_task(
self._process_down_queue(), f"{self}::_process_down_queue"
)
self._process_push_task = self._task_manager.create_task(
self._process_push_queue(), f"{self}::_process_push_queue"
)
tasks = [self._process_up_task, self._process_down_task, self._process_push_task]
await self._observer.start()
return tasks
return self._process_push_task
def _maybe_start_heartbeat_tasks(self):
if self._params.enable_heartbeats:
self._heartbeat_push_task = asyncio.create_task(self._heartbeat_push_handler())
self._heartbeat_monitor_task = asyncio.create_task(self._heartbeat_monitor_handler())
self._heartbeat_push_task = self._task_manager.create_task(
self._heartbeat_push_handler(), f"{self}::_heartbeat_push_handler"
)
self._heartbeat_monitor_task = self._task_manager.create_task(
self._heartbeat_monitor_handler(), f"{self}::_heartbeat_monitor_handler"
)
async def _cancel_tasks(self, cancel_push: bool):
async def _cancel_tasks(self):
await self._maybe_cancel_heartbeat_tasks()
if cancel_push:
self._process_push_task.cancel()
await self._process_push_task
self._process_up_task.cancel()
await self._process_up_task
self._process_down_task.cancel()
await self._process_down_task
await self._task_manager.cancel_task(self._process_up_task)
await self._task_manager.cancel_task(self._process_down_task)
await self._observer.stop()
async def _maybe_cancel_heartbeat_tasks(self):
if self._params.enable_heartbeats:
self._heartbeat_push_task.cancel()
await self._heartbeat_push_task
self._heartbeat_monitor_task.cancel()
await self._heartbeat_monitor_task
await self._task_manager.cancel_task(self._heartbeat_push_task)
await self._task_manager.cancel_task(self._heartbeat_monitor_task)
def _initial_metrics_frame(self) -> MetricsFrame:
processors = self._pipeline.processors_with_metrics()
@@ -221,6 +260,11 @@ class PipelineTask:
await self._endframe_event.wait()
self._endframe_event.clear()
async def _cleanup(self):
await self._source.cleanup()
await self._pipeline.cleanup()
await self._sink.cleanup()
async def _process_push_queue(self):
"""This is the task that runs the pipeline for the first time by sending
a StartFrame and by pushing any other frames queued by the user. It runs
@@ -232,13 +276,17 @@ class PipelineTask:
self._maybe_start_heartbeat_tasks()
start_frame = StartFrame(
clock=self._clock,
task_manager=self._task_manager,
allow_interruptions=self._params.allow_interruptions,
audio_in_sample_rate=self._params.audio_in_sample_rate,
audio_out_sample_rate=self._params.audio_out_sample_rate,
enable_metrics=self._params.enable_metrics,
enable_usage_metrics=self._params.enable_usage_metrics,
report_only_initial_ttfb=self._params.report_only_initial_ttfb,
observer=self._observer,
clock=self._clock,
report_only_initial_ttfb=self._params.report_only_initial_ttfb,
)
start_frame.metadata = self._params.start_metadata
await self._source.queue_frame(start_frame, FrameDirection.DOWNSTREAM)
if self._params.enable_metrics and self._params.send_initial_empty_metrics:
@@ -247,24 +295,16 @@ class PipelineTask:
running = True
should_cleanup = True
while running:
try:
frame = await self._push_queue.get()
await self._source.queue_frame(frame, FrameDirection.DOWNSTREAM)
if isinstance(frame, EndFrame):
await self._wait_for_endframe()
running = not isinstance(frame, (StopTaskFrame, EndFrame))
should_cleanup = not isinstance(frame, StopTaskFrame)
self._push_queue.task_done()
except asyncio.CancelledError:
break
frame = await self._push_queue.get()
await self._source.queue_frame(frame, FrameDirection.DOWNSTREAM)
if isinstance(frame, EndFrame):
await self._wait_for_endframe()
running = not isinstance(frame, (CancelFrame, EndFrame, StopTaskFrame))
should_cleanup = not isinstance(frame, StopTaskFrame)
self._push_queue.task_done()
# Cleanup only if we need to.
if should_cleanup:
await self._source.cleanup()
await self._pipeline.cleanup()
await self._sink.cleanup()
# Finally, cancel internal tasks. We don't cancel the push tasks because
# that's us.
await self._cancel_tasks(False)
await self._cleanup()
async def _process_up_queue(self):
"""This is the task that processes frames coming upstream from the
@@ -274,26 +314,23 @@ class PipelineTask:
"""
while True:
try:
frame = await self._up_queue.get()
if isinstance(frame, EndTaskFrame):
# Tell the task we should end nicely.
await self.queue_frame(EndFrame())
elif isinstance(frame, CancelTaskFrame):
# Tell the task we should end right away.
frame = await self._up_queue.get()
if isinstance(frame, EndTaskFrame):
# Tell the task we should end nicely.
await self.queue_frame(EndFrame())
elif isinstance(frame, CancelTaskFrame):
# Tell the task we should end right away.
await self.queue_frame(CancelFrame())
elif isinstance(frame, StopTaskFrame):
await self.queue_frame(StopTaskFrame())
elif isinstance(frame, ErrorFrame):
logger.error(f"Error running app: {frame}")
if frame.fatal:
# Cancel all tasks downstream.
await self.queue_frame(CancelFrame())
elif isinstance(frame, StopTaskFrame):
# Tell the task we should stop.
await self.queue_frame(StopTaskFrame())
elif isinstance(frame, ErrorFrame):
logger.error(f"Error running app: {frame}")
if frame.fatal:
# Cancel all tasks downstream.
await self.queue_frame(CancelFrame())
# Tell the task we should stop.
await self.queue_frame(StopTaskFrame())
self._up_queue.task_done()
except asyncio.CancelledError:
break
self._up_queue.task_done()
async def _process_down_queue(self):
"""This tasks process frames coming downstream from the pipeline. For
@@ -303,29 +340,23 @@ class PipelineTask:
"""
while True:
try:
frame = await self._down_queue.get()
if isinstance(frame, EndFrame):
self._endframe_event.set()
elif isinstance(frame, HeartbeatFrame):
await self._heartbeat_queue.put(frame)
self._down_queue.task_done()
except asyncio.CancelledError:
break
frame = await self._down_queue.get()
if isinstance(frame, EndFrame):
self._endframe_event.set()
elif isinstance(frame, HeartbeatFrame):
await self._heartbeat_queue.put(frame)
self._down_queue.task_done()
async def _heartbeat_push_handler(self):
"""
This tasks pushes a heartbeat frame every HEARTBEAT_SECONDS.
This tasks pushes a heartbeat frame every heartbeat period.
"""
while True:
try:
# Don't use `queue_frame()` because if an EndFrame is queued the
# task will just stop waiting for the pipeline to finish not
# allowing more frames to be pushed.
await self._source.queue_frame(HeartbeatFrame(timestamp=self._clock.get_time()))
await asyncio.sleep(HEARTBEAT_SECONDS)
except asyncio.CancelledError:
break
# Don't use `queue_frame()` because if an EndFrame is queued the
# task will just stop waiting for the pipeline to finish not
# allowing more frames to be pushed.
await self._source.queue_frame(HeartbeatFrame(timestamp=self._clock.get_time()))
await asyncio.sleep(self._params.heartbeats_period_secs)
async def _heartbeat_monitor_handler(self):
"""This tasks monitors heartbeat frames. If a heartbeat frame has not
@@ -345,8 +376,11 @@ class PipelineTask:
logger.warning(
f"{self}: heartbeat frame not received for more than {wait_time} seconds"
)
except asyncio.CancelledError:
break
def _print_dangling_tasks(self):
tasks = [t.get_name() for t in self._task_manager.current_tasks()]
if tasks:
logger.warning(f"Dangling tasks detected: {tasks}")
def __str__(self):
return self.name

44
src/pipecat/pipeline/task_observer.py
View File

@@ -12,6 +12,8 @@ from attr import dataclass
from pipecat.frames.frames import Frame
from pipecat.observers.base_observer import BaseObserver
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.asyncio import TaskManager
from pipecat.utils.utils import obj_count, obj_id
@dataclass
@@ -53,14 +55,29 @@ class TaskObserver(BaseObserver):
"""
def __init__(self, observers: List[BaseObserver] = []):
self._proxies: List[Proxy] = self._create_proxies(observers)
def __init__(self, *, observers: List[BaseObserver] = [], task_manager: TaskManager):
self._id: int = obj_id()
self._name: str = f"{self.__class__.__name__}#{obj_count(self)}"
self._observers = observers
self._task_manager = task_manager
self._proxies: List[Proxy] = []
@property
def id(self) -> int:
return self._id
@property
def name(self) -> str:
return self._name
async def start(self):
"""Starts all proxy observer tasks."""
self._proxies = self._create_proxies(self._observers)
async def stop(self):
"""Stops all proxy observer tasks."""
for proxy in self._proxies:
proxy.task.cancel()
await proxy.task
await self._task_manager.cancel_task(proxy.task)
async def on_push_frame(
self,
@@ -81,17 +98,20 @@ class TaskObserver(BaseObserver):
proxies = []
for observer in observers:
queue = asyncio.Queue()
task = asyncio.create_task(self._proxy_task_handler(queue, observer))
task = self._task_manager.create_task(
self._proxy_task_handler(queue, observer),
f"{self}::{observer.__class__.__name__}::_proxy_task_handler",
)
proxy = Proxy(queue=queue, task=task, observer=observer)
proxies.append(proxy)
return proxies
async def _proxy_task_handler(self, queue: asyncio.Queue, observer: BaseObserver):
while True:
try:
data = await queue.get()
await observer.on_push_frame(
data.src, data.dst, data.frame, data.direction, data.timestamp
)
except asyncio.CancelledError:
break
data = await queue.get()
await observer.on_push_frame(
data.src, data.dst, data.frame, data.direction, data.timestamp
)
def __str__(self):
return self.name

27
src/pipecat/processors/aggregators/gated_openai_llm_context.py
View File

@@ -4,8 +4,6 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -18,9 +16,10 @@ class GatedOpenAILLMContextAggregator(FrameProcessor):
"""
def __init__(self, notifier: BaseNotifier, **kwargs):
def __init__(self, *, notifier: BaseNotifier, start_open: bool = False, **kwargs):
super().__init__(**kwargs)
self._notifier = notifier
self._start_open = start_open
self._last_context_frame = None
async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -33,23 +32,23 @@ class GatedOpenAILLMContextAggregator(FrameProcessor):
await self._stop()
await self.push_frame(frame)
elif isinstance(frame, OpenAILLMContextFrame):
self._last_context_frame = frame
if self._start_open:
self._start_open = False
await self.push_frame(frame, direction)
else:
self._last_context_frame = frame
else:
await self.push_frame(frame, direction)
async def _start(self):
self._gate_task = self.get_event_loop().create_task(self._gate_task_handler())
self._gate_task = self.create_task(self._gate_task_handler())
async def _stop(self):
self._gate_task.cancel()
await self._gate_task
await self.cancel_task(self._gate_task)
async def _gate_task_handler(self):
while True:
try:
await self._notifier.wait()
if self._last_context_frame:
await self.push_frame(self._last_context_frame)
self._last_context_frame = None
except asyncio.CancelledError:
break
await self._notifier.wait()
if self._last_context_frame:
await self.push_frame(self._last_context_frame)
self._last_context_frame = None

2
src/pipecat/processors/async_generator.py
View File

@@ -30,7 +30,7 @@ class AsyncGeneratorProcessor(FrameProcessor):
if isinstance(frame, (CancelFrame, EndFrame)):
await self._data_queue.put(None)
else:
data = self._serializer.serialize(frame)
data = await self._serializer.serialize(frame)
if data:
await self._data_queue.put(data)

99
src/pipecat/processors/audio/audio_buffer_processor.py
View File

@@ -4,11 +4,18 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
from pipecat.audio.utils import interleave_stereo_audio, mix_audio, resample_audio
import time
from typing import Optional
from pipecat.audio.utils import create_default_resampler, interleave_stereo_audio, mix_audio
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,
EndFrame,
Frame,
InputAudioRawFrame,
OutputAudioRawFrame,
StartFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -28,16 +35,29 @@ class AudioBufferProcessor(FrameProcessor):
"""
def __init__(
self, *, sample_rate: int = 24000, num_channels: int = 1, buffer_size: int = 0, **kwargs
self,
*,
sample_rate: Optional[int] = None,
num_channels: int = 1,
buffer_size: int = 0,
**kwargs,
):
super().__init__(**kwargs)
self._sample_rate = sample_rate
self._init_sample_rate = sample_rate
self._sample_rate = 0
self._num_channels = num_channels
self._buffer_size = buffer_size
self._user_audio_buffer = bytearray()
self._bot_audio_buffer = bytearray()
self._last_user_frame_at = 0
self._last_bot_frame_at = 0
self._recording = False
self._resampler = create_default_resampler()
self._register_event_handler("on_audio_data")
@property
@@ -63,40 +83,83 @@ class AudioBufferProcessor(FrameProcessor):
else:
return b""
def reset_audio_buffers(self):
self._user_audio_buffer = bytearray()
self._bot_audio_buffer = bytearray()
async def start_recording(self):
self._recording = True
self._reset_recording()
async def stop_recording(self):
await self._call_on_audio_data_handler()
self._recording = False
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
# Include all audio from the user.
if isinstance(frame, InputAudioRawFrame):
resampled = resample_audio(frame.audio, frame.sample_rate, self._sample_rate)
# Update output sample rate if necessary.
if isinstance(frame, StartFrame):
self._update_sample_rate(frame)
if self._recording and isinstance(frame, InputAudioRawFrame):
# Add silence if we need to.
silence = self._compute_silence(self._last_user_frame_at)
self._user_audio_buffer.extend(silence)
# Add user audio.
resampled = await self._resample_audio(frame)
self._user_audio_buffer.extend(resampled)
# Sync the bot's buffer to the user's buffer by adding silence if needed
if len(self._user_audio_buffer) > len(self._bot_audio_buffer):
silence = b"\x00" * len(resampled)
self._bot_audio_buffer.extend(silence)
# If the bot is speaking, include all audio from the bot.
elif isinstance(frame, OutputAudioRawFrame):
resampled = resample_audio(frame.audio, frame.sample_rate, self._sample_rate)
# Save time of frame so we can compute silence.
self._last_user_frame_at = time.time()
elif self._recording and isinstance(frame, OutputAudioRawFrame):
# Add silence if we need to.
silence = self._compute_silence(self._last_bot_frame_at)
self._bot_audio_buffer.extend(silence)
# Add bot audio.
resampled = await self._resample_audio(frame)
self._bot_audio_buffer.extend(resampled)
# Save time of frame so we can compute silence.
self._last_bot_frame_at = time.time()
if self._buffer_size > 0 and len(self._user_audio_buffer) > self._buffer_size:
await self._call_on_audio_data_handler()
if isinstance(frame, (CancelFrame, EndFrame)):
await self.stop_recording()
await self.push_frame(frame, direction)
def _update_sample_rate(self, frame: StartFrame):
self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate
async def _call_on_audio_data_handler(self):
if not self.has_audio():
if not self.has_audio() or not self._recording:
return
merged_audio = self.merge_audio_buffers()
await self._call_event_handler(
"on_audio_data", merged_audio, self._sample_rate, self._num_channels
)
self.reset_audio_buffers()
self._reset_audio_buffers()
def _buffer_has_audio(self, buffer: bytearray) -> bool:
return buffer is not None and len(buffer) > 0
def _reset_recording(self):
self._reset_audio_buffers()
self._last_user_frame_at = time.time()
self._last_bot_frame_at = time.time()
def _reset_audio_buffers(self):
self._user_audio_buffer = bytearray()
self._bot_audio_buffer = bytearray()
async def _resample_audio(self, frame: AudioRawFrame) -> bytes:
return await self._resampler.resample(frame.audio, frame.sample_rate, self._sample_rate)
def _compute_silence(self, from_time: float) -> bytes:
quiet_time = time.time() - from_time
# We should get audio frames very frequently. We pick 100ms because
# that's big enough, but it could be even a bit slower since we usually
# do 20ms audio frames.
if from_time == 0 or quiet_time < 0.1:
return b""
num_bytes = int(quiet_time * self._sample_rate) * 2
silence = b"\x00" * num_bytes
return silence

8
src/pipecat/processors/audio/vad/silero.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import Optional
from loguru import logger
from pipecat.audio.vad.silero import SileroVADAnalyzer
@@ -11,6 +13,7 @@ from pipecat.audio.vad.vad_analyzer import VADParams, VADState
from pipecat.frames.frames import (
AudioRawFrame,
Frame,
StartFrame,
StartInterruptionFrame,
StopInterruptionFrame,
UserStartedSpeakingFrame,
@@ -23,7 +26,7 @@ class SileroVAD(FrameProcessor):
def __init__(
self,
*,
sample_rate: int = 16000,
sample_rate: Optional[int] = None,
vad_params: VADParams = VADParams(),
audio_passthrough: bool = False,
):
@@ -41,6 +44,9 @@ class SileroVAD(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame):
self._vad_analyzer.set_sample_rate(frame.audio_in_sample_rate)
if isinstance(frame, AudioRawFrame):
await self._analyze_audio(frame)
if self._audio_passthrough:

2
src/pipecat/processors/filters/frame_filter.py
View File

@@ -11,7 +11,7 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class FrameFilter(FrameProcessor):
def __init__(self, types: Tuple[Type[Frame]]):
def __init__(self, types: Tuple[Type[Frame], ...]):
super().__init__()
self._types = types

2
src/pipecat/processors/filters/stt_mute_filter.py
View File

@@ -121,6 +121,8 @@ class STTMuteFilter(FrameProcessor):
return False
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
"""Processes incoming frames and manages muting state."""
# Handle function call state changes
if isinstance(frame, FunctionCallInProgressFrame):

165
src/pipecat/processors/frame_processor.py
View File

@@ -7,14 +7,13 @@
import asyncio
import inspect
from enum import Enum
from typing import Awaitable, Callable, Optional
from typing import Awaitable, Callable, Coroutine, Optional
from loguru import logger
from pipecat.clocks.base_clock import BaseClock
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
ErrorFrame,
Frame,
StartFrame,
@@ -24,6 +23,7 @@ from pipecat.frames.frames import (
)
from pipecat.metrics.metrics import LLMTokenUsage, MetricsData
from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMetrics
from pipecat.utils.asyncio import TaskManager
from pipecat.utils.utils import obj_count, obj_id
@@ -36,24 +36,25 @@ class FrameProcessor:
def __init__(
self,
*,
name: str | None = None,
metrics: FrameProcessorMetrics | None = None,
loop: asyncio.AbstractEventLoop | None = None,
name: Optional[str] = None,
metrics: Optional[FrameProcessorMetrics] = None,
**kwargs,
):
self.id: int = obj_id()
self.name = name or f"{self.__class__.__name__}#{obj_count(self)}"
self._parent: "FrameProcessor" | None = None
self._prev: "FrameProcessor" | None = None
self._next: "FrameProcessor" | None = None
self._loop: asyncio.AbstractEventLoop = loop or asyncio.get_running_loop()
self._id: int = obj_id()
self._name = name or f"{self.__class__.__name__}#{obj_count(self)}"
self._parent: Optional["FrameProcessor"] = None
self._prev: Optional["FrameProcessor"] = None
self._next: Optional["FrameProcessor"] = None
self._event_handlers: dict = {}
# Clock
self._clock: BaseClock | None = None
self._clock: Optional[BaseClock] = None
# Properties
# Task Manager
self._task_manager: Optional[TaskManager] = None
# Other properties
self._allow_interruptions = False
self._enable_metrics = False
self._enable_usage_metrics = False
@@ -72,16 +73,24 @@ class FrameProcessor:
self._metrics.set_processor_name(self.name)
# Processors have an input queue. The input queue will be processed
# immediately (default) or it will block if `pause_processing_frames()`
# is called. To resume processing frames we need to call
# immediately (default) or it will block if `pause_processing_frames()` is
# called. To resume processing frames we need to call
# `resume_processing_frames()`.
self.__should_block_frames = False
self.__create_input_task()
self.__input_frame_task: Optional[asyncio.Task] = None
# Every processor in Pipecat should only output frames from a single
# task. This avoid problems like audio overlapping. System frames are
# the exception to this rule. This create this task.
self.__create_push_task()
# task. This avoid problems like audio overlapping. System frames are the
# exception to this rule. This create this task.
self.__push_frame_task: Optional[asyncio.Task] = None
@property
def id(self) -> int:
return self._id
@property
def name(self) -> str:
return self._name
@property
def interruptions_allowed(self):
@@ -141,6 +150,22 @@ class FrameProcessor:
await self.stop_ttfb_metrics()
await self.stop_processing_metrics()
def create_task(self, coroutine: Coroutine) -> asyncio.Task:
if not self._task_manager:
raise Exception(f"{self} TaskManager is still not initialized.")
name = f"{self}::{coroutine.cr_code.co_name}"
return self._task_manager.create_task(coroutine, name)
async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None):
if not self._task_manager:
raise Exception(f"{self} TaskManager is still not initialized.")
await self._task_manager.cancel_task(task, timeout)
async def wait_for_task(self, task: asyncio.Task, timeout: Optional[float] = None):
if not self._task_manager:
raise Exception(f"{self} TaskManager is still not initialized.")
await self._task_manager.wait_for_task(task, timeout)
async def cleanup(self):
await self.__cancel_input_task()
await self.__cancel_push_task()
@@ -151,17 +176,26 @@ class FrameProcessor:
logger.debug(f"Linking {self} -> {self._next}")
def get_event_loop(self) -> asyncio.AbstractEventLoop:
return self._loop
if not self._task_manager:
raise Exception(f"{self} TaskManager is still not initialized.")
return self._task_manager.get_event_loop()
def set_parent(self, parent: "FrameProcessor"):
self._parent = parent
def get_parent(self) -> "FrameProcessor":
def get_parent(self) -> Optional["FrameProcessor"]:
return self._parent
def get_clock(self) -> BaseClock:
if not self._clock:
raise Exception(f"{self} Clock is still not initialized.")
return self._clock
def get_task_manager(self) -> TaskManager:
if not self._task_manager:
raise Exception(f"{self} TaskManager is still not initialized.")
return self._task_manager
async def queue_frame(
self,
frame: Frame,
@@ -186,18 +220,19 @@ class FrameProcessor:
self.__should_block_frames = True
async def resume_processing_frames(self):
logger.trace("f{self}: resuming frame processing")
logger.trace(f"{self}: resuming frame processing")
self.__input_event.set()
self.__should_block_frames = False
async def process_frame(self, frame: Frame, direction: FrameDirection):
if isinstance(frame, StartFrame):
self._clock = frame.clock
self._task_manager = frame.task_manager
self._allow_interruptions = frame.allow_interruptions
self._enable_metrics = frame.enable_metrics
self._enable_usage_metrics = frame.enable_usage_metrics
self._report_only_initial_ttfb = frame.report_only_initial_ttfb
self._observer = frame.observer
await self.__start(frame)
elif isinstance(frame, StartInterruptionFrame):
await self._start_interruption()
await self.stop_all_metrics()
@@ -232,6 +267,10 @@ class FrameProcessor:
raise Exception(f"Event handler {event_name} already registered")
self._event_handlers[event_name] = []
async def __start(self, frame: StartFrame):
self.__create_input_task()
self.__create_push_task()
#
# Handle interruptions
#
@@ -281,68 +320,52 @@ class FrameProcessor:
raise
def __create_input_task(self):
self.__should_block_frames = False
self.__input_queue = asyncio.Queue()
self.__input_frame_task = self.get_event_loop().create_task(
self.__input_frame_task_handler()
)
self.__input_event = asyncio.Event()
if not self.__input_frame_task:
self.__should_block_frames = False
self.__input_queue = asyncio.Queue()
self.__input_event = asyncio.Event()
self.__input_frame_task = self.create_task(self.__input_frame_task_handler())
async def __cancel_input_task(self):
self.__input_frame_task.cancel()
await self.__input_frame_task
if self.__input_frame_task:
await self.cancel_task(self.__input_frame_task)
self.__input_frame_task = None
async def __input_frame_task_handler(self):
running = True
while running:
try:
if self.__should_block_frames:
logger.trace(f"{self}: frame processing paused")
await self.__input_event.wait()
self.__input_event.clear()
logger.trace(f"{self}: frame processing resumed")
while True:
if self.__should_block_frames:
logger.trace(f"{self}: frame processing paused")
await self.__input_event.wait()
self.__input_event.clear()
self.__should_block_frames = False
logger.trace(f"{self}: frame processing resumed")
(frame, direction, callback) = await self.__input_queue.get()
(frame, direction, callback) = await self.__input_queue.get()
# Process the frame.
await self.process_frame(frame, direction)
# Process the frame.
await self.process_frame(frame, direction)
# If this frame has an associated callback, call it now.
if callback:
await callback(self, frame, direction)
# If this frame has an associated callback, call it now.
if callback:
await callback(self, frame, direction)
running = not isinstance(frame, EndFrame)
self.__input_queue.task_done()
except asyncio.CancelledError:
logger.trace(f"{self}: cancelled input task")
break
except Exception as e:
logger.exception(f"{self}: Uncaught exception {e}")
await self.push_error(ErrorFrame(str(e)))
self.__input_queue.task_done()
def __create_push_task(self):
self.__push_queue = asyncio.Queue()
self.__push_frame_task = self.get_event_loop().create_task(self.__push_frame_task_handler())
if not self.__push_frame_task:
self.__push_queue = asyncio.Queue()
self.__push_frame_task = self.create_task(self.__push_frame_task_handler())
async def __cancel_push_task(self):
self.__push_frame_task.cancel()
await self.__push_frame_task
if self.__push_frame_task:
await self.cancel_task(self.__push_frame_task)
self.__push_frame_task = None
async def __push_frame_task_handler(self):
running = True
while running:
try:
(frame, direction) = await self.__push_queue.get()
await self.__internal_push_frame(frame, direction)
running = not isinstance(frame, EndFrame)
self.__push_queue.task_done()
except asyncio.CancelledError:
logger.trace(f"{self}: cancelled push task")
break
except Exception as e:
logger.exception(f"{self}: Uncaught exception {e}")
await self.push_error(ErrorFrame(str(e)))
while True:
(frame, direction) = await self.__push_queue.get()
await self.__internal_push_frame(frame, direction)
self.__push_queue.task_done()
async def _call_event_handler(self, event_name: str, *args, **kwargs):
try:

139
src/pipecat/processors/frameworks/rtvi.py
View File

@@ -58,10 +58,14 @@ from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFrame
from pipecat.utils.string import match_endofsentence
RTVI_PROTOCOL_VERSION = "0.3.0"
RTVI_MESSAGE_LABEL = "rtvi-ai"
RTVIMessageLiteral = Literal["rtvi-ai"]
ActionResult = Union[bool, int, float, str, list, dict]
@@ -154,7 +158,7 @@ class RTVIActionFrame(DataFrame):
class RTVIMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: str
id: str
data: Optional[Dict[str, Any]] = None
@@ -170,7 +174,7 @@ class RTVIErrorResponseData(BaseModel):
class RTVIErrorResponse(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["error-response"] = "error-response"
id: str
data: RTVIErrorResponseData
@@ -182,7 +186,7 @@ class RTVIErrorData(BaseModel):
class RTVIError(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["error"] = "error"
data: RTVIErrorData
@@ -192,7 +196,7 @@ class RTVIDescribeConfigData(BaseModel):
class RTVIDescribeConfig(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["config-available"] = "config-available"
id: str
data: RTVIDescribeConfigData
@@ -203,14 +207,14 @@ class RTVIDescribeActionsData(BaseModel):
class RTVIDescribeActions(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["actions-available"] = "actions-available"
id: str
data: RTVIDescribeActionsData
class RTVIConfigResponse(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["config"] = "config"
id: str
data: RTVIConfig
@@ -221,7 +225,7 @@ class RTVIActionResponseData(BaseModel):
class RTVIActionResponse(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["action-response"] = "action-response"
id: str
data: RTVIActionResponseData
@@ -233,7 +237,7 @@ class RTVIBotReadyData(BaseModel):
class RTVIBotReady(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-ready"] = "bot-ready"
id: str
data: RTVIBotReadyData
@@ -246,7 +250,7 @@ class RTVILLMFunctionCallMessageData(BaseModel):
class RTVILLMFunctionCallMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["llm-function-call"] = "llm-function-call"
data: RTVILLMFunctionCallMessageData
@@ -256,7 +260,7 @@ class RTVILLMFunctionCallStartMessageData(BaseModel):
class RTVILLMFunctionCallStartMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["llm-function-call-start"] = "llm-function-call-start"
data: RTVILLMFunctionCallStartMessageData
@@ -269,22 +273,22 @@ class RTVILLMFunctionCallResultData(BaseModel):
class RTVIBotLLMStartedMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-llm-started"] = "bot-llm-started"
class RTVIBotLLMStoppedMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-llm-stopped"] = "bot-llm-stopped"
class RTVIBotTTSStartedMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-tts-started"] = "bot-tts-started"
class RTVIBotTTSStoppedMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-tts-stopped"] = "bot-tts-stopped"
@@ -292,20 +296,32 @@ class RTVITextMessageData(BaseModel):
text: str
class RTVISearchResponseMessageData(BaseModel):
search_result: Optional[str]
rendered_content: Optional[str]
origins: List[LLMSearchOrigin]
class RTVIBotTranscriptionMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-transcription"] = "bot-transcription"
data: RTVITextMessageData
class RTVIBotLLMTextMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-llm-text"] = "bot-llm-text"
data: RTVITextMessageData
class RTVIBotTTSTextMessage(BaseModel):
class RTVIBotLLMSearchResponseMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
type: Literal["bot-llm-search-response"] = "bot-llm-search-response"
data: RTVISearchResponseMessageData
class RTVIBotTTSTextMessage(BaseModel):
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-tts-text"] = "bot-tts-text"
data: RTVITextMessageData
@@ -317,7 +333,7 @@ class RTVIAudioMessageData(BaseModel):
class RTVIBotTTSAudioMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-tts-audio"] = "bot-tts-audio"
data: RTVIAudioMessageData
@@ -330,39 +346,39 @@ class RTVIUserTranscriptionMessageData(BaseModel):
class RTVIUserTranscriptionMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["user-transcription"] = "user-transcription"
data: RTVIUserTranscriptionMessageData
class RTVIUserLLMTextMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["user-llm-text"] = "user-llm-text"
data: RTVITextMessageData
class RTVIUserStartedSpeakingMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["user-started-speaking"] = "user-started-speaking"
class RTVIUserStoppedSpeakingMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["user-stopped-speaking"] = "user-stopped-speaking"
class RTVIBotStartedSpeakingMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-started-speaking"] = "bot-started-speaking"
class RTVIBotStoppedSpeakingMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-stopped-speaking"] = "bot-stopped-speaking"
class RTVIMetricsMessage(BaseModel):
label: Literal["rtvi-ai"] = "rtvi-ai"
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["metrics"] = "metrics"
data: Mapping[str, Any]
@@ -607,6 +623,8 @@ class RTVIObserver(BaseObserver):
await self._push_transport_message_urgent(RTVIBotLLMStoppedMessage())
elif isinstance(frame, LLMTextFrame):
await self._handle_llm_text_frame(frame)
elif isinstance(frame, LLMSearchResponseFrame):
await self._handle_llm_search_response_frame(frame)
elif isinstance(frame, TTSStartedFrame):
await self._push_transport_message_urgent(RTVIBotTTSStartedMessage())
elif isinstance(frame, TTSStoppedFrame):
@@ -657,6 +675,16 @@ class RTVIObserver(BaseObserver):
if match_endofsentence(self._bot_transcription):
await self._push_bot_transcription()
async def _handle_llm_search_response_frame(self, frame: LLMSearchResponseFrame):
message = RTVIBotLLMSearchResponseMessage(
data=RTVISearchResponseMessageData(
search_result=frame.search_result,
origins=frame.origins,
rendered_content=frame.rendered_content,
)
)
await self._push_transport_message_urgent(message)
async def _handle_user_transcriptions(self, frame: Frame):
message = None
if isinstance(frame, TranscriptionFrame):
@@ -676,17 +704,20 @@ class RTVIObserver(BaseObserver):
await self._push_transport_message_urgent(message)
async def _handle_context(self, frame: OpenAILLMContextFrame):
messages = frame.context.messages
if len(messages) > 0:
message = messages[-1]
if message["role"] == "user":
content = message["content"]
if isinstance(content, list):
text = " ".join(item["text"] for item in content if "text" in item)
else:
text = content
rtvi_message = RTVIUserLLMTextMessage(data=RTVITextMessageData(text=text))
await self._push_transport_message_urgent(rtvi_message)
try:
messages = frame.context.messages
if len(messages) > 0:
message = messages[-1]
if message["role"] == "user":
content = message["content"]
if isinstance(content, list):
text = " ".join(item["text"] for item in content if "text" in item)
else:
text = content
rtvi_message = RTVIUserLLMTextMessage(data=RTVITextMessageData(text=text))
await self._push_transport_message_urgent(rtvi_message)
except TypeError as e:
logger.warning(f"Caught an error while trying to handle context: {e}")
async def _handle_metrics(self, frame: MetricsFrame):
metrics = {}
@@ -733,11 +764,11 @@ class RTVIProcessor(FrameProcessor):
# A task to process incoming action frames.
self._action_queue = asyncio.Queue()
self._action_task = self.get_event_loop().create_task(self._action_task_handler())
self._action_task: Optional[asyncio.Task] = None
# A task to process incoming transport messages.
self._message_queue = asyncio.Queue()
self._message_task = self.get_event_loop().create_task(self._message_task_handler())
self._message_task: Optional[asyncio.Task] = None
self._register_event_handler("on_bot_started")
self._register_event_handler("on_client_ready")
@@ -832,6 +863,8 @@ class RTVIProcessor(FrameProcessor):
await self._pipeline.cleanup()
async def _start(self, frame: StartFrame):
self._action_task = self.create_task(self._action_task_handler())
self._message_task = self.create_task(self._message_task_handler())
await self._call_event_handler("on_bot_started")
async def _stop(self, frame: EndFrame):
@@ -842,13 +875,11 @@ class RTVIProcessor(FrameProcessor):
async def _cancel_tasks(self):
if self._action_task:
self._action_task.cancel()
await self._action_task
await self.cancel_task(self._action_task)
self._action_task = None
if self._message_task:
self._message_task.cancel()
await self._message_task
await self.cancel_task(self._message_task)
self._message_task = None
async def _push_transport_message(self, model: BaseModel, exclude_none: bool = True):
@@ -857,25 +888,23 @@ class RTVIProcessor(FrameProcessor):
async def _action_task_handler(self):
while True:
try:
frame = await self._action_queue.get()
await self._handle_action(frame.message_id, frame.rtvi_action_run)
self._action_queue.task_done()
except asyncio.CancelledError:
break
frame = await self._action_queue.get()
await self._handle_action(frame.message_id, frame.rtvi_action_run)
self._action_queue.task_done()
async def _message_task_handler(self):
while True:
try:
message = await self._message_queue.get()
await self._handle_message(message)
self._message_queue.task_done()
except asyncio.CancelledError:
break
message = await self._message_queue.get()
await self._handle_message(message)
self._message_queue.task_done()
async def _handle_transport_message(self, frame: TransportMessageUrgentFrame):
try:
message = RTVIMessage.model_validate(frame.message)
transport_message = frame.message
if transport_message.get("label") != RTVI_MESSAGE_LABEL:
logger.warning(f"Ignoring not RTVI message: {transport_message}")
return
message = RTVIMessage.model_validate(transport_message)
await self._message_queue.put(message)
except ValidationError as e:
await self.send_error(f"Invalid RTVI transport message: {e}")

9
src/pipecat/processors/gstreamer/pipeline_source.py
View File

@@ -5,6 +5,7 @@
#
import asyncio
from typing import Optional
from loguru import logger
from pydantic import BaseModel
@@ -38,7 +39,7 @@ class GStreamerPipelineSource(FrameProcessor):
class OutputParams(BaseModel):
video_width: int = 1280
video_height: int = 720
audio_sample_rate: int = 24000
audio_sample_rate: Optional[int] = None
audio_channels: int = 1
clock_sync: bool = True
@@ -46,6 +47,7 @@ class GStreamerPipelineSource(FrameProcessor):
super().__init__(**kwargs)
self._out_params = out_params
self._sample_rate = 0
Gst.init()
@@ -90,6 +92,7 @@ class GStreamerPipelineSource(FrameProcessor):
await self.push_frame(frame, direction)
async def _start(self, frame: StartFrame):
self._sample_rate = self._out_params.audio_sample_rate or frame.audio_out_sample_rate
self._player.set_state(Gst.State.PLAYING)
async def _stop(self, frame: EndFrame):
@@ -122,7 +125,7 @@ class GStreamerPipelineSource(FrameProcessor):
audioresample = Gst.ElementFactory.make("audioresample", None)
audiocapsfilter = Gst.ElementFactory.make("capsfilter", None)
audiocaps = Gst.Caps.from_string(
f"audio/x-raw,format=S16LE,rate={self._out_params.audio_sample_rate},channels={self._out_params.audio_channels},layout=interleaved"
f"audio/x-raw,format=S16LE,rate={self._sample_rate},channels={self._out_params.audio_channels},layout=interleaved"
)
audiocapsfilter.set_property("caps", audiocaps)
appsink_audio = Gst.ElementFactory.make("appsink", None)
@@ -188,7 +191,7 @@ class GStreamerPipelineSource(FrameProcessor):
(_, info) = buffer.map(Gst.MapFlags.READ)
frame = OutputAudioRawFrame(
audio=info.data,
sample_rate=self._out_params.audio_sample_rate,
sample_rate=self._sample_rate,
num_channels=self._out_params.audio_channels,
)
asyncio.run_coroutine_threadsafe(self.push_frame(frame), self.get_event_loop())

7
src/pipecat/processors/idle_frame_processor.py
View File

@@ -49,12 +49,11 @@ class IdleFrameProcessor(FrameProcessor):
self._idle_event.set()
async def cleanup(self):
self._idle_task.cancel()
await self._idle_task
await self.cancel_task(self._idle_task)
def _create_idle_task(self):
self._idle_event = asyncio.Event()
self._idle_task = self.get_event_loop().create_task(self._idle_task_handler())
self._idle_task = self.create_task(self._idle_task_handler())
async def _idle_task_handler(self):
while True:
@@ -62,7 +61,5 @@ class IdleFrameProcessor(FrameProcessor):
await asyncio.wait_for(self._idle_event.wait(), timeout=self._timeout)
except asyncio.TimeoutError:
await self._callback(self)
except asyncio.CancelledError:
break
finally:
self._idle_event.clear()

2
src/pipecat/processors/logger.py
View File

@@ -31,6 +31,8 @@ class FrameLogger(FrameProcessor):
self._ignored_frame_types = tuple(ignored_frame_types) if ignored_frame_types else None
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if self._ignored_frame_types and not isinstance(frame, self._ignored_frame_types):
dir = "<" if direction is FrameDirection.UPSTREAM else ">"
msg = f"{dir} {self._prefix}: {frame}"

131
src/pipecat/processors/transcript_processor.py
View File

@@ -4,17 +4,23 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import List
from typing import List, Optional
from loguru import logger
from pipecat.frames.frames import (
BotStoppedSpeakingFrame,
CancelFrame,
EndFrame,
Frame,
OpenAILLMContextAssistantTimestampFrame,
StartInterruptionFrame,
TranscriptionFrame,
TranscriptionMessage,
TranscriptionUpdateFrame,
TTSTextFrame,
)
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.time import time_now_iso8601
class BaseTranscriptProcessor(FrameProcessor):
@@ -64,89 +70,74 @@ class UserTranscriptProcessor(BaseTranscriptProcessor):
class AssistantTranscriptProcessor(BaseTranscriptProcessor):
"""Processes assistant LLM context frames into timestamped conversation messages."""
"""Processes assistant TTS text frames into timestamped conversation messages.
This processor aggregates TTS text frames into complete utterances and emits them as
transcript messages. Utterances are completed when:
- The bot stops speaking (BotStoppedSpeakingFrame)
- The bot is interrupted (StartInterruptionFrame)
- The pipeline ends (EndFrame)
Attributes:
_current_text_parts: List of text fragments being aggregated for current utterance
_aggregation_start_time: Timestamp when the current utterance began
"""
def __init__(self, **kwargs):
"""Initialize processor with empty message stores."""
"""Initialize processor with aggregation state."""
super().__init__(**kwargs)
self._pending_assistant_messages: List[TranscriptionMessage] = []
self._current_text_parts: List[str] = []
self._aggregation_start_time: Optional[str] | None = None
def _extract_messages(self, messages: List[dict]) -> List[TranscriptionMessage]:
"""Extract assistant messages from the OpenAI standard message format.
async def _emit_aggregated_text(self):
"""Emit aggregated text as a transcript message."""
if self._current_text_parts and self._aggregation_start_time:
content = " ".join(self._current_text_parts).strip()
if content:
logger.debug(f"Emitting aggregated assistant message: {content}")
message = TranscriptionMessage(
role="assistant",
content=content,
timestamp=self._aggregation_start_time,
)
await self._emit_update([message])
else:
logger.debug("No content to emit after stripping whitespace")
Args:
messages: List of messages in OpenAI format, which can be either:
- Simple format: {"role": "user", "content": "Hello"}
- Content list: {"role": "user", "content": [{"type": "text", "text": "Hello"}]}
Returns:
List[TranscriptionMessage]: Normalized conversation messages
"""
result = []
for msg in messages:
if msg["role"] != "assistant":
continue
content = msg.get("content")
if isinstance(content, str):
if content:
result.append(TranscriptionMessage(role="assistant", content=content))
elif isinstance(content, list):
text_parts = []
for part in content:
if isinstance(part, dict) and part.get("type") == "text":
text_parts.append(part["text"])
if text_parts:
result.append(
TranscriptionMessage(role="assistant", content=" ".join(text_parts))
)
return result
def _find_new_messages(self, current: List[TranscriptionMessage]) -> List[TranscriptionMessage]:
"""Find unprocessed messages from current list.
Args:
current: List of current messages
Returns:
List[TranscriptionMessage]: New messages not yet processed
"""
if not self._processed_messages:
return current
processed_len = len(self._processed_messages)
if len(current) <= processed_len:
return []
return current[processed_len:]
# Reset aggregation state
self._current_text_parts = []
self._aggregation_start_time = None
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames into assistant conversation messages.
Handles different frame types:
- TTSTextFrame: Aggregates text for current utterance
- BotStoppedSpeakingFrame: Completes current utterance
- StartInterruptionFrame: Completes current utterance due to interruption
- EndFrame: Completes current utterance at pipeline end
- CancelFrame: Completes current utterance due to cancellation
Args:
frame: Input frame to process
direction: Frame processing direction
"""
await super().process_frame(frame, direction)
if isinstance(frame, OpenAILLMContextFrame):
standard_messages = []
for msg in frame.context.messages:
converted = frame.context.to_standard_messages(msg)
standard_messages.extend(converted)
if isinstance(frame, TTSTextFrame):
# Start timestamp on first text part
if not self._aggregation_start_time:
self._aggregation_start_time = time_now_iso8601()
current_messages = self._extract_messages(standard_messages)
new_messages = self._find_new_messages(current_messages)
self._pending_assistant_messages.extend(new_messages)
self._current_text_parts.append(frame.text)
elif isinstance(frame, OpenAILLMContextAssistantTimestampFrame):
if self._pending_assistant_messages:
for msg in self._pending_assistant_messages:
msg.timestamp = frame.timestamp
await self._emit_update(self._pending_assistant_messages)
self._pending_assistant_messages = []
elif isinstance(frame, (BotStoppedSpeakingFrame, StartInterruptionFrame, CancelFrame)):
# Emit accumulated text when bot finishes speaking or is interrupted
await self._emit_aggregated_text()
elif isinstance(frame, EndFrame):
# Emit any remaining text when pipeline ends
await self._emit_aggregated_text()
await self.push_frame(frame, direction)
@@ -170,8 +161,8 @@ class TranscriptProcessor:
llm,
tts,
transport.output(),
transcript.assistant_tts(), # Assistant transcripts
context_aggregator.assistant(),
transcript.assistant(), # Assistant transcripts
]
)

93
src/pipecat/processors/user_idle_processor.py
View File

@@ -5,7 +5,8 @@
#
import asyncio
from typing import Awaitable, Callable
import inspect
from typing import Awaitable, Callable, Union
from pipecat.frames.frames import (
BotSpeakingFrame,
@@ -25,11 +26,23 @@ class UserIdleProcessor(FrameProcessor):
or BotSpeaking).
Args:
callback: Function to call when user is idle
callback: Function to call when user is idle. Can be either:
- Basic callback(processor) -> None
- Retry callback(processor, retry_count) -> bool
Return True to continue monitoring for idle events,
Return False to stop the idle monitoring task
timeout: Seconds to wait before considering user idle
**kwargs: Additional arguments passed to FrameProcessor
Example:
# Retry callback:
async def handle_idle(processor: "UserIdleProcessor", retry_count: int) -> bool:
if retry_count < 3:
await send_reminder("Are you still there?")
return True
return False
# Basic callback:
async def handle_idle(processor: "UserIdleProcessor") -> None:
await send_reminder("Are you still there?")
@@ -42,34 +55,68 @@ class UserIdleProcessor(FrameProcessor):
def __init__(
self,
*,
callback: Callable[["UserIdleProcessor"], Awaitable[None]],
callback: Union[
Callable[["UserIdleProcessor"], Awaitable[None]], # Basic
Callable[["UserIdleProcessor", int], Awaitable[bool]], # Retry
],
timeout: float,
**kwargs,
):
super().__init__(**kwargs)
self._callback = callback
self._callback = self._wrap_callback(callback)
self._timeout = timeout
self._retry_count = 0
self._interrupted = False
self._conversation_started = False
self._idle_task = None
self._idle_event = asyncio.Event()
def _create_idle_task(self):
"""Create the idle task if it hasn't been created yet."""
if self._idle_task is None:
self._idle_task = self.get_event_loop().create_task(self._idle_task_handler())
def _wrap_callback(
self,
callback: Union[
Callable[["UserIdleProcessor"], Awaitable[None]],
Callable[["UserIdleProcessor", int], Awaitable[bool]],
],
) -> Callable[["UserIdleProcessor", int], Awaitable[bool]]:
"""Wraps callback to support both basic and retry signatures.
async def _stop(self):
Args:
callback: The callback function to wrap.
Returns:
A wrapped callback that returns bool to indicate whether to continue monitoring.
"""
sig = inspect.signature(callback)
param_count = len(sig.parameters)
async def wrapper(processor: "UserIdleProcessor", retry_count: int) -> bool:
if param_count == 1:
# Basic callback
await callback(processor) # type: ignore
return True
else:
# Retry callback
return await callback(processor, retry_count) # type: ignore
return wrapper
def _create_idle_task(self) -> None:
"""Creates the idle task if it hasn't been created yet."""
if self._idle_task is None:
self._idle_task = self.create_task(self._idle_task_handler())
@property
def retry_count(self) -> int:
"""Returns the current retry count."""
return self._retry_count
async def _stop(self) -> None:
"""Stops and cleans up the idle monitoring task."""
if self._idle_task is not None:
self._idle_task.cancel()
try:
await self._idle_task
except asyncio.CancelledError:
pass # Expected when task is cancelled
await self.cancel_task(self._idle_task)
self._idle_task = None
async def process_frame(self, frame: Frame, direction: FrameDirection):
async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
"""Processes incoming frames and manages idle monitoring state.
Args:
@@ -98,6 +145,7 @@ class UserIdleProcessor(FrameProcessor):
if self._conversation_started:
# We shouldn't call the idle callback if the user or the bot are speaking
if isinstance(frame, UserStartedSpeakingFrame):
self._retry_count = 0 # Reset retry count when user speaks
self._interrupted = True
self._idle_event.set()
elif isinstance(frame, UserStoppedSpeakingFrame):
@@ -106,23 +154,26 @@ class UserIdleProcessor(FrameProcessor):
elif isinstance(frame, BotSpeakingFrame):
self._idle_event.set()
async def cleanup(self):
async def cleanup(self) -> None:
"""Cleans up resources when processor is shutting down."""
await super().cleanup()
if self._idle_task: # Only stop if task exists
await self._stop()
async def _idle_task_handler(self):
async def _idle_task_handler(self) -> None:
"""Monitors for idle timeout and triggers callbacks.
Runs in a loop until cancelled.
Runs in a loop until cancelled or callback indicates completion.
"""
while True:
try:
await asyncio.wait_for(self._idle_event.wait(), timeout=self._timeout)
except asyncio.TimeoutError:
if not self._interrupted:
await self._callback(self)
except asyncio.CancelledError:
break
self._retry_count += 1
should_continue = await self._callback(self, self._retry_count)
if not should_continue:
await self._stop()
break
finally:
self._idle_event.clear()

11
src/pipecat/serializers/base_serializer.py
View File

@@ -7,7 +7,7 @@
from abc import ABC, abstractmethod
from enum import Enum
from pipecat.frames.frames import Frame
from pipecat.frames.frames import Frame, StartFrame
class FrameSerializerType(Enum):
@@ -21,10 +21,13 @@ class FrameSerializer(ABC):
def type(self) -> FrameSerializerType:
pass
@abstractmethod
def serialize(self, frame: Frame) -> str | bytes | None:
async def setup(self, frame: StartFrame):
pass
@abstractmethod
def deserialize(self, data: str | bytes) -> Frame | None:
async def serialize(self, frame: Frame) -> str | bytes | None:
pass
@abstractmethod
async def deserialize(self, data: str | bytes) -> Frame | None:
pass

4
src/pipecat/serializers/livekit.py
View File

@@ -25,7 +25,7 @@ class LivekitFrameSerializer(FrameSerializer):
def type(self) -> FrameSerializerType:
return FrameSerializerType.BINARY
def serialize(self, frame: Frame) -> str | bytes | None:
async def serialize(self, frame: Frame) -> str | bytes | None:
if not isinstance(frame, OutputAudioRawFrame):
return None
audio_frame = AudioFrame(
@@ -36,7 +36,7 @@ class LivekitFrameSerializer(FrameSerializer):
)
return pickle.dumps(audio_frame)
def deserialize(self, data: str | bytes) -> Frame | None:
async def deserialize(self, data: str | bytes) -> Frame | None:
audio_frame: AudioFrame = pickle.loads(data)["frame"]
return InputAudioRawFrame(
audio=bytes(audio_frame.data),

35
src/pipecat/serializers/protobuf.py
View File

@@ -41,7 +41,7 @@ class ProtobufFrameSerializer(FrameSerializer):
def type(self) -> FrameSerializerType:
return FrameSerializerType.BINARY
def serialize(self, frame: Frame) -> str | bytes | None:
async def serialize(self, frame: Frame) -> str | bytes | None:
proto_frame = frame_protos.Frame()
if type(frame) not in self.SERIALIZABLE_TYPES:
logger.warning(f"Frame type {type(frame)} is not serializable")
@@ -57,26 +57,7 @@ class ProtobufFrameSerializer(FrameSerializer):
return proto_frame.SerializeToString()
def deserialize(self, data: str | bytes) -> Frame | None:
"""Returns a Frame object from a Frame protobuf.
Used to convert frames
passed over the wire as protobufs to Frame objects used in pipelines
and frame processors.
>>> serializer = ProtobufFrameSerializer()
>>> serializer.deserialize(
... serializer.serialize(OutputAudioFrame(data=b'1234567890')))
InputAudioFrame(data=b'1234567890')
>>> serializer.deserialize(
... serializer.serialize(TextFrame(text='hello world')))
TextFrame(text='hello world')
>>> serializer.deserialize(serializer.serialize(TranscriptionFrame(
... text="Hello there!", participantId="123", timestamp="2021-01-01")))
TranscriptionFrame(text='Hello there!', participantId='123', timestamp='2021-01-01')
"""
async def deserialize(self, data: str | bytes) -> Frame | None:
proto = frame_protos.Frame.FromString(data)
which = proto.WhichOneof("frame")
if which not in self.DESERIALIZABLE_FIELDS:
@@ -93,11 +74,11 @@ class ProtobufFrameSerializer(FrameSerializer):
id = getattr(args, "id", None)
name = getattr(args, "name", None)
pts = getattr(args, "pts", None)
if not id and "id" in args_dict:
if "id" in args_dict:
del args_dict["id"]
if not name and "name" in args_dict:
if "name" in args_dict:
del args_dict["name"]
if not pts and "pts" in args_dict:
if "pts" in args_dict:
del args_dict["pts"]
# Create the instance
@@ -105,10 +86,10 @@ class ProtobufFrameSerializer(FrameSerializer):
# Set special fields
if id:
setattr(instance, "id", getattr(args, "id", None))
setattr(instance, "id", id)
if name:
setattr(instance, "name", getattr(args, "name", None))
setattr(instance, "name", name)
if pts:
setattr(instance, "pts", getattr(args, "pts", None))
setattr(instance, "pts", pts)
return instance

125
src/pipecat/serializers/telnyx.py Normal file
View File

@@ -0,0 +1,125 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import base64
import json
from typing import Optional
from pydantic import BaseModel
from pipecat.audio.utils import (
alaw_to_pcm,
create_default_resampler,
pcm_to_alaw,
pcm_to_ulaw,
ulaw_to_pcm,
)
from pipecat.frames.frames import (
AudioRawFrame,
Frame,
InputAudioRawFrame,
InputDTMFFrame,
KeypadEntry,
StartFrame,
StartInterruptionFrame,
)
from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType
class TelnyxFrameSerializer(FrameSerializer):
class InputParams(BaseModel):
telnyx_sample_rate: Optional[int] = None
sample_rate: Optional[int] = None
inbound_encoding: str = "PCMU"
outbound_encoding: str = "PCMU"
def __init__(
self,
stream_id: str,
outbound_encoding: str,
inbound_encoding: str,
params: InputParams = InputParams(),
):
self._stream_id = stream_id
params.outbound_encoding = outbound_encoding
params.inbound_encoding = inbound_encoding
self._params = params
self._resampler = create_default_resampler()
@property
def type(self) -> FrameSerializerType:
return FrameSerializerType.TEXT
async def setup(self, frame: StartFrame):
self._telnyx_sample_rate = self._params.telnyx_sample_rate or frame.audio_in_sample_rate
self._sample_rate = self._params.sample_rate or frame.audio_out_sample_rate
async def serialize(self, frame: Frame) -> str | bytes | None:
if isinstance(frame, AudioRawFrame):
data = frame.audio
if self._params.inbound_encoding == "PCMU":
serialized_data = await pcm_to_ulaw(
data, frame.sample_rate, self._telnyx_sample_rate, self._resampler
)
elif self._params.inbound_encoding == "PCMA":
serialized_data = await pcm_to_alaw(
data, frame.sample_rate, self._telnyx_sample_rate, self._resampler
)
else:
raise ValueError(f"Unsupported encoding: {self._params.inbound_encoding}")
payload = base64.b64encode(serialized_data).decode("utf-8")
answer = {
"event": "media",
"media": {"payload": payload},
}
return json.dumps(answer)
if isinstance(frame, StartInterruptionFrame):
answer = {"event": "clear"}
return json.dumps(answer)
async def deserialize(self, data: str | bytes) -> Frame | None:
message = json.loads(data)
if message["event"] == "media":
payload_base64 = message["media"]["payload"]
payload = base64.b64decode(payload_base64)
if self._params.outbound_encoding == "PCMU":
deserialized_data = await ulaw_to_pcm(
payload,
self._telnyx_sample_rate,
self._sample_rate,
self._resampler,
)
elif self._params.outbound_encoding == "PCMA":
deserialized_data = await alaw_to_pcm(
payload,
self._telnyx_sample_rate,
self._sample_rate,
self._resampler,
)
else:
raise ValueError(f"Unsupported encoding: {self._params.outbound_encoding}")
audio_frame = InputAudioRawFrame(
audio=deserialized_data, num_channels=1, sample_rate=self._sample_rate
)
return audio_frame
elif message["event"] == "dtmf":
digit = message.get("dtmf", {}).get("digit")
try:
return InputDTMFFrame(KeypadEntry(digit))
except ValueError as e:
# Handle case where string doesn't match any enum value
return None
else:
return None

44
src/pipecat/serializers/twilio.py
View File

@@ -6,39 +6,57 @@
import base64
import json
from typing import Optional
from pydantic import BaseModel
from pipecat.audio.utils import pcm_to_ulaw, ulaw_to_pcm
from pipecat.audio.utils import create_default_resampler, pcm_to_ulaw, ulaw_to_pcm
from pipecat.frames.frames import (
AudioRawFrame,
Frame,
InputAudioRawFrame,
InputDTMFFrame,
KeypadEntry,
StartFrame,
StartInterruptionFrame,
TransportMessageFrame,
TransportMessageUrgentFrame,
)
from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType
class TwilioFrameSerializer(FrameSerializer):
class InputParams(BaseModel):
twilio_sample_rate: int = 8000
sample_rate: int = 16000
twilio_sample_rate: Optional[int] = None
sample_rate: Optional[int] = None
def __init__(self, stream_sid: str, params: InputParams = InputParams()):
self._stream_sid = stream_sid
self._params = params
self._twilio_sample_rate = 0
self._sample_rate = 0
self._resampler = create_default_resampler()
@property
def type(self) -> FrameSerializerType:
return FrameSerializerType.TEXT
def serialize(self, frame: Frame) -> str | bytes | None:
if isinstance(frame, AudioRawFrame):
async def setup(self, frame: StartFrame):
self._twilio_sample_rate = self._params.twilio_sample_rate or frame.audio_in_sample_rate
self._sample_rate = self._params.sample_rate or frame.audio_out_sample_rate
async def serialize(self, frame: Frame) -> str | bytes | None:
if isinstance(frame, StartInterruptionFrame):
answer = {"event": "clear", "streamSid": self._stream_sid}
return json.dumps(answer)
elif isinstance(frame, AudioRawFrame):
data = frame.audio
serialized_data = pcm_to_ulaw(data, frame.sample_rate, self._params.twilio_sample_rate)
serialized_data = await pcm_to_ulaw(
data, frame.sample_rate, self._twilio_sample_rate, self._resampler
)
payload = base64.b64encode(serialized_data).decode("utf-8")
answer = {
"event": "media",
@@ -47,23 +65,21 @@ class TwilioFrameSerializer(FrameSerializer):
}
return json.dumps(answer)
elif isinstance(frame, (TransportMessageFrame, TransportMessageUrgentFrame)):
return json.dumps(frame.message)
if isinstance(frame, StartInterruptionFrame):
answer = {"event": "clear", "streamSid": self._stream_sid}
return json.dumps(answer)
def deserialize(self, data: str | bytes) -> Frame | None:
async def deserialize(self, data: str | bytes) -> Frame | None:
message = json.loads(data)
if message["event"] == "media":
payload_base64 = message["media"]["payload"]
payload = base64.b64decode(payload_base64)
deserialized_data = ulaw_to_pcm(
payload, self._params.twilio_sample_rate, self._params.sample_rate
deserialized_data = await ulaw_to_pcm(
payload, self._twilio_sample_rate, self._sample_rate, self._resampler
)
audio_frame = InputAudioRawFrame(
audio=deserialized_data, num_channels=1, sample_rate=self._params.sample_rate
audio=deserialized_data, num_channels=1, sample_rate=self._sample_rate
)
return audio_frame
elif message["event"] == "dtmf":

171
src/pipecat/services/ai_services.py
View File

@@ -8,7 +8,7 @@ import asyncio
import io
import wave
from abc import abstractmethod
from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple
from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Tuple
from loguru import logger
@@ -69,7 +69,7 @@ class AIService(FrameProcessor):
async def cancel(self, frame: CancelFrame):
pass
async def _update_settings(self, settings: Dict[str, Any]):
async def _update_settings(self, settings: Mapping[str, Any]):
from pipecat.services.openai_realtime_beta.events import (
SessionProperties,
)
@@ -208,8 +208,12 @@ class TTSService(AIService):
push_stop_frames: bool = False,
# if push_stop_frames is True, wait for this idle period before pushing TTSStoppedFrame
stop_frame_timeout_s: float = 1.0,
# if True, TTSService will push silence audio frames after TTSStoppedFrame
push_silence_after_stop: bool = False,
# if push_silence_after_stop is True, send this amount of audio silence
silence_time_s: float = 2.0,
# TTS output sample rate
sample_rate: int = 24000,
sample_rate: Optional[int] = None,
text_filter: Optional[BaseTextFilter] = None,
**kwargs,
):
@@ -218,7 +222,10 @@ class TTSService(AIService):
self._push_text_frames: bool = push_text_frames
self._push_stop_frames: bool = push_stop_frames
self._stop_frame_timeout_s: float = stop_frame_timeout_s
self._sample_rate: int = sample_rate
self._push_silence_after_stop: bool = push_silence_after_stop
self._silence_time_s: float = silence_time_s
self._init_sample_rate = sample_rate
self._sample_rate = 0
self._voice_id: str = ""
self._settings: Dict[str, Any] = {}
self._text_filter: Optional[BaseTextFilter] = text_filter
@@ -242,34 +249,36 @@ class TTSService(AIService):
async def flush_audio(self):
pass
def language_to_service_language(self, language: Language) -> str | None:
return Language(language)
# Converts the text to audio.
@abstractmethod
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
pass
def language_to_service_language(self, language: Language) -> str | None:
return Language(language)
async def update_setting(self, key: str, value: Any):
pass
async def start(self, frame: StartFrame):
await super().start(frame)
self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate
if self._push_stop_frames:
self._stop_frame_task = self.get_event_loop().create_task(self._stop_frame_handler())
self._stop_frame_task = self.create_task(self._stop_frame_handler())
async def stop(self, frame: EndFrame):
await super().stop(frame)
if self._stop_frame_task:
self._stop_frame_task.cancel()
await self._stop_frame_task
await self.cancel_task(self._stop_frame_task)
self._stop_frame_task = None
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
if self._stop_frame_task:
self._stop_frame_task.cancel()
await self._stop_frame_task
await self.cancel_task(self._stop_frame_task)
self._stop_frame_task = None
async def _update_settings(self, settings: Dict[str, Any]):
async def _update_settings(self, settings: Mapping[str, Any]):
for key, value in settings.items():
if key in self._settings:
logger.info(f"Updating TTS setting {key} to: [{value}]")
@@ -316,6 +325,16 @@ class TTSService(AIService):
await self.push_frame(frame, direction)
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
if self._push_silence_after_stop and isinstance(frame, TTSStoppedFrame):
silence_num_bytes = int(self._silence_time_s * self.sample_rate * 2) # 16-bit
await self.push_frame(
TTSAudioRawFrame(
audio=b"\x00" * silence_num_bytes,
sample_rate=self.sample_rate,
num_channels=1,
)
)
await super().push_frame(frame, direction)
if self._push_stop_frames and (
@@ -364,23 +383,20 @@ class TTSService(AIService):
await self.push_frame(TTSTextFrame(text))
async def _stop_frame_handler(self):
try:
has_started = False
while True:
try:
frame = await asyncio.wait_for(
self._stop_frame_queue.get(), self._stop_frame_timeout_s
)
if isinstance(frame, TTSStartedFrame):
has_started = True
elif isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
has_started = False
except asyncio.TimeoutError:
if has_started:
await self.push_frame(TTSStoppedFrame())
has_started = False
except asyncio.CancelledError:
pass
has_started = False
while True:
try:
frame = await asyncio.wait_for(
self._stop_frame_queue.get(), self._stop_frame_timeout_s
)
if isinstance(frame, TTSStartedFrame):
has_started = True
elif isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
has_started = False
except asyncio.TimeoutError:
if has_started:
await self.push_frame(TTSStoppedFrame())
has_started = False
class WordTTSService(TTSService):
@@ -388,7 +404,7 @@ class WordTTSService(TTSService):
super().__init__(**kwargs)
self._initial_word_timestamp = -1
self._words_queue = asyncio.Queue()
self._words_task = self.get_event_loop().create_task(self._words_task_handler())
self._words_task = None
def start_word_timestamps(self):
if self._initial_word_timestamp == -1:
@@ -401,6 +417,10 @@ class WordTTSService(TTSService):
for word, timestamp in word_times:
await self._words_queue.put((word, seconds_to_nanoseconds(timestamp)))
async def start(self, frame: StartFrame):
await super().start(frame)
await self._create_words_task()
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._stop_words_task()
@@ -419,45 +439,50 @@ class WordTTSService(TTSService):
await super()._handle_interruption(frame, direction)
self.reset_word_timestamps()
async def _create_words_task(self):
self._words_task = self.create_task(self._words_task_handler())
async def _stop_words_task(self):
if self._words_task:
self._words_task.cancel()
await self._words_task
await self.cancel_task(self._words_task)
self._words_task = None
async def _words_task_handler(self):
last_pts = 0
while True:
try:
(word, timestamp) = await self._words_queue.get()
if word == "Reset" and timestamp == 0:
self.reset_word_timestamps()
frame = None
elif word == "LLMFullResponseEndFrame" and timestamp == 0:
frame = LLMFullResponseEndFrame()
frame.pts = last_pts
elif word == "TTSStoppedFrame" and timestamp == 0:
frame = TTSStoppedFrame()
frame.pts = last_pts
else:
frame = TTSTextFrame(word)
frame.pts = self._initial_word_timestamp + timestamp
if frame:
last_pts = frame.pts
await self.push_frame(frame)
self._words_queue.task_done()
except asyncio.CancelledError:
break
except Exception as e:
logger.exception(f"{self} exception: {e}")
(word, timestamp) = await self._words_queue.get()
if word == "Reset" and timestamp == 0:
self.reset_word_timestamps()
frame = None
elif word == "LLMFullResponseEndFrame" and timestamp == 0:
frame = LLMFullResponseEndFrame()
frame.pts = last_pts
elif word == "TTSStoppedFrame" and timestamp == 0:
frame = TTSStoppedFrame()
frame.pts = last_pts
else:
frame = TTSTextFrame(word)
frame.pts = self._initial_word_timestamp + timestamp
if frame:
last_pts = frame.pts
await self.push_frame(frame)
self._words_queue.task_done()
class STTService(AIService):
"""STTService is a base class for speech-to-text services."""
def __init__(self, audio_passthrough=False, **kwargs):
def __init__(
self,
audio_passthrough=False,
# STT input sample rate
sample_rate: Optional[int] = None,
**kwargs,
):
super().__init__(**kwargs)
self._audio_passthrough = audio_passthrough
self._init_sample_rate = sample_rate
self._sample_rate = 0
self._settings: Dict[str, Any] = {}
self._muted: bool = False
@@ -466,6 +491,10 @@ class STTService(AIService):
"""Returns whether the STT service is currently muted."""
return self._muted
@property
def sample_rate(self) -> int:
return self._sample_rate
@abstractmethod
async def set_model(self, model: str):
self.set_model_name(model)
@@ -479,7 +508,11 @@ class STTService(AIService):
"""Returns transcript as a string"""
pass
async def _update_settings(self, settings: Dict[str, Any]):
async def start(self, frame: StartFrame):
await super().start(frame)
self._sample_rate = self._init_sample_rate or frame.audio_in_sample_rate
async def _update_settings(self, settings: Mapping[str, Any]):
logger.info(f"Updating STT settings: {self._settings}")
for key, value in settings.items():
if key in self._settings:
@@ -528,17 +561,15 @@ class SegmentedSTTService(STTService):
min_volume: float = 0.6,
max_silence_secs: float = 0.3,
max_buffer_secs: float = 1.5,
sample_rate: int = 24000,
num_channels: int = 1,
sample_rate: Optional[int] = None,
**kwargs,
):
super().__init__(**kwargs)
super().__init__(sample_rate=sample_rate, **kwargs)
self._min_volume = min_volume
self._max_silence_secs = max_silence_secs
self._max_buffer_secs = max_buffer_secs
self._sample_rate = sample_rate
self._num_channels = num_channels
(self._content, self._wave) = self._new_wave()
self._content = None
self._wave = None
self._silence_num_frames = 0
# Volume exponential smoothing
self._smoothing_factor = 0.2
@@ -557,8 +588,8 @@ class SegmentedSTTService(STTService):
# If buffer is not empty and we have enough data or there's been a long
# silence, transcribe the audio gathered so far.
silence_secs = self._silence_num_frames / self._sample_rate
buffer_secs = self._wave.getnframes() / self._sample_rate
silence_secs = self._silence_num_frames / self.sample_rate
buffer_secs = self._wave.getnframes() / self.sample_rate
if self._content.tell() > 0 and (
buffer_secs > self._max_buffer_secs or silence_secs > self._max_silence_secs
):
@@ -568,18 +599,24 @@ class SegmentedSTTService(STTService):
await self.process_generator(self.run_stt(self._content.read()))
(self._content, self._wave) = self._new_wave()
async def start(self, frame: StartFrame):
await super().start(frame)
(self._content, self._wave) = self._new_wave()
async def stop(self, frame: EndFrame):
await super().stop(frame)
self._wave.close()
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
self._wave.close()
def _new_wave(self):
content = io.BytesIO()
ww = wave.open(content, "wb")
ww.setsampwidth(2)
ww.setnchannels(self._num_channels)
ww.setframerate(self._sample_rate)
ww.setnchannels(1)
ww.setframerate(self.sample_rate)
return (content, ww)
def _get_smoothed_volume(self, frame: AudioRawFrame) -> float:

17
src/pipecat/services/assemblyai.py
View File

@@ -5,7 +5,7 @@
#
import asyncio
from typing import AsyncGenerator
from typing import AsyncGenerator, Optional
from loguru import logger
@@ -38,20 +38,17 @@ class AssemblyAISTTService(STTService):
self,
*,
api_key: str,
sample_rate: int = 16000,
sample_rate: Optional[int] = None,
encoding: AudioEncoding = AudioEncoding("pcm_s16le"),
language=Language.EN, # Only English is supported for Realtime
**kwargs,
):
super().__init__(**kwargs)
super().__init__(sample_rate=sample_rate, **kwargs)
aai.settings.api_key = api_key
self._transcriber: aai.RealtimeTranscriber | None = None
# Store reference to the main event loop for use in callback functions
self._loop = asyncio.get_event_loop()
self._settings = {
"sample_rate": sample_rate,
"encoding": encoding,
"language": language,
}
@@ -121,7 +118,7 @@ class AssemblyAISTTService(STTService):
# Schedule the coroutine to run in the main event loop
# This is necessary because this callback runs in a different thread
asyncio.run_coroutine_threadsafe(self.push_frame(frame), self._loop)
asyncio.run_coroutine_threadsafe(self.push_frame(frame), self.get_event_loop())
def on_error(error: aai.RealtimeError):
"""Callback for handling errors from AssemblyAI.
@@ -131,14 +128,16 @@ class AssemblyAISTTService(STTService):
"""
logger.error(f"{self}: An error occurred: {error}")
# Schedule the coroutine to run in the main event loop
asyncio.run_coroutine_threadsafe(self.push_frame(ErrorFrame(str(error))), self._loop)
asyncio.run_coroutine_threadsafe(
self.push_frame(ErrorFrame(str(error))), self.get_event_loop()
)
def on_close():
"""Callback for when the connection to AssemblyAI is closed."""
logger.info(f"{self}: Disconnected from AssemblyAI")
self._transcriber = aai.RealtimeTranscriber(
sample_rate=self._settings["sample_rate"],
sample_rate=self.sample_rate,
encoding=self._settings["encoding"],
on_data=on_data,
on_error=on_error,

14
src/pipecat/services/aws.py
View File

@@ -10,7 +10,7 @@ from typing import AsyncGenerator, Optional
from loguru import logger
from pydantic import BaseModel
from pipecat.audio.utils import resample_audio
from pipecat.audio.utils import create_default_resampler
from pipecat.frames.frames import (
ErrorFrame,
Frame,
@@ -124,7 +124,7 @@ class PollyTTSService(TTSService):
aws_session_token: Optional[str] = None,
region: Optional[str] = None,
voice_id: str = "Joanna",
sample_rate: int = 24000,
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
@@ -138,7 +138,6 @@ class PollyTTSService(TTSService):
region_name=region,
)
self._settings = {
"sample_rate": sample_rate,
"engine": params.engine,
"language": self.language_to_service_language(params.language)
if params.language
@@ -148,6 +147,8 @@ class PollyTTSService(TTSService):
"volume": params.volume,
}
self._resampler = create_default_resampler()
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
@@ -193,8 +194,7 @@ class PollyTTSService(TTSService):
response = self._polly_client.synthesize_speech(**args)
if "AudioStream" in response:
audio_data = response["AudioStream"].read()
resampled = resample_audio(audio_data, 16000, self._settings["sample_rate"])
return resampled
return audio_data
return None
logger.debug(f"Generating TTS: [{text}]")
@@ -225,6 +225,8 @@ class PollyTTSService(TTSService):
yield None
return
audio_data = await self._resampler.resample(audio_data, 16000, self.sample_rate)
await self.start_tts_usage_metrics(text)
yield TTSStartedFrame()
@@ -234,7 +236,7 @@ class PollyTTSService(TTSService):
chunk = audio_data[i : i + chunk_size]
if len(chunk) > 0:
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
yield frame
yield TTSStoppedFrame()

71
src/pipecat/services/azure.py
View File

@@ -450,14 +450,13 @@ class AzureBaseTTSService(TTSService):
api_key: str,
region: str,
voice="en-US-SaraNeural",
sample_rate: int = 24000,
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
self._settings = {
"sample_rate": sample_rate,
"emphasis": params.emphasis,
"language": self.language_to_service_language(params.language)
if params.language
@@ -530,25 +529,32 @@ class AzureBaseTTSService(TTSService):
class AzureTTSService(AzureBaseTTSService):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._speech_config = None
self._speech_synthesizer = None
self._audio_queue = asyncio.Queue()
speech_config = SpeechConfig(
async def start(self, frame: StartFrame):
await super().start(frame)
# Now self.sample_rate is properly initialized
self._speech_config = SpeechConfig(
subscription=self._api_key,
region=self._region,
speech_recognition_language=self._settings["language"],
)
speech_config.set_speech_synthesis_output_format(
sample_rate_to_output_format(self._settings["sample_rate"])
self._speech_config.set_speech_synthesis_output_format(
sample_rate_to_output_format(self.sample_rate)
)
speech_config.set_service_property(
self._speech_config.set_service_property(
"synthesizer.synthesis.connection.synthesisConnectionImpl",
"websocket",
ServicePropertyChannel.UriQueryParameter,
)
self._speech_synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=None)
self._speech_synthesizer = SpeechSynthesizer(
speech_config=self._speech_config, audio_config=None
)
# Set up event handlers
self._audio_queue = asyncio.Queue()
self._speech_synthesizer.synthesizing.connect(self._handle_synthesizing)
self._speech_synthesizer.synthesis_completed.connect(self._handle_completed)
self._speech_synthesizer.synthesis_canceled.connect(self._handle_canceled)
@@ -591,7 +597,7 @@ class AzureTTSService(AzureBaseTTSService):
yield TTSAudioRawFrame(
audio=chunk,
sample_rate=self._settings["sample_rate"],
sample_rate=self.sample_rate,
num_channels=1,
)
@@ -605,17 +611,22 @@ class AzureTTSService(AzureBaseTTSService):
class AzureHttpTTSService(AzureBaseTTSService):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._speech_config = None
self._speech_synthesizer = None
speech_config = SpeechConfig(
async def start(self, frame: StartFrame):
await super().start(frame)
self._speech_config = SpeechConfig(
subscription=self._api_key,
region=self._region,
speech_recognition_language=self._settings["language"],
)
speech_config.set_speech_synthesis_output_format(
sample_rate_to_output_format(self._settings["sample_rate"])
self._speech_config.set_speech_synthesis_output_format(
sample_rate_to_output_format(self.sample_rate)
)
self._speech_synthesizer = SpeechSynthesizer(
speech_config=self._speech_config, audio_config=None
)
self._speech_synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=None)
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
@@ -633,7 +644,7 @@ class AzureHttpTTSService(AzureBaseTTSService):
# Azure always sends a 44-byte header. Strip it off.
yield TTSAudioRawFrame(
audio=result.audio_data[44:],
sample_rate=self._settings["sample_rate"],
sample_rate=self.sample_rate,
num_channels=1,
)
yield TTSStoppedFrame()
@@ -650,24 +661,14 @@ class AzureSTTService(STTService):
*,
api_key: str,
region: str,
language=Language.EN_US,
sample_rate=24000,
channels=1,
language: Language = Language.EN_US,
sample_rate: Optional[int] = None,
**kwargs,
):
super().__init__(**kwargs)
super().__init__(sample_rate=sample_rate, **kwargs)
speech_config = SpeechConfig(subscription=api_key, region=region)
speech_config.speech_recognition_language = language
stream_format = AudioStreamFormat(samples_per_second=sample_rate, channels=channels)
self._audio_stream = PushAudioInputStream(stream_format)
audio_config = AudioConfig(stream=self._audio_stream)
self._speech_recognizer = SpeechRecognizer(
speech_config=speech_config, audio_config=audio_config
)
self._speech_recognizer.recognized.connect(self._on_handle_recognized)
self._speech_config = SpeechConfig(subscription=api_key, region=region)
self._speech_config.speech_recognition_language = language
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
await self.start_processing_metrics()
@@ -677,6 +678,16 @@ class AzureSTTService(STTService):
async def start(self, frame: StartFrame):
await super().start(frame)
stream_format = AudioStreamFormat(samples_per_second=self.sample_rate, channels=1)
self._audio_stream = PushAudioInputStream(stream_format)
audio_config = AudioConfig(stream=self._audio_stream)
self._speech_recognizer = SpeechRecognizer(
speech_config=self._speech_config, audio_config=audio_config
)
self._speech_recognizer.recognized.connect(self._on_handle_recognized)
self._speech_recognizer.start_continuous_recognition_async()
async def stop(self, frame: EndFrame):

2
src/pipecat/services/canonical.py
View File

@@ -18,6 +18,7 @@ from pipecat.frames.frames import CancelFrame, EndFrame, Frame
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import AIService
try:
import aiofiles
@@ -116,7 +117,6 @@ class CanonicalMetricsService(AIService):
try:
await self._multipart_upload(filename)
await aiofiles.os.remove(filename)
audio_buffer_processor.reset_audio_buffers()
except FileNotFoundError:
pass
except Exception as e:

30
src/pipecat/services/cartesia.py
View File

@@ -88,8 +88,8 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
voice_id: str,
cartesia_version: str = "2024-06-10",
url: str = "wss://api.cartesia.ai/tts/websocket",
model: str = "sonic-english",
sample_rate: int = 24000,
model: str = "sonic",
sample_rate: Optional[int] = None,
encoding: str = "pcm_s16le",
container: str = "raw",
params: InputParams = InputParams(),
@@ -121,7 +121,7 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
"output_format": {
"container": container,
"encoding": encoding,
"sample_rate": sample_rate,
"sample_rate": 0,
},
"language": self.language_to_service_language(params.language)
if params.language
@@ -174,6 +174,7 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
async def start(self, frame: StartFrame):
await super().start(frame)
self._settings["output_format"]["sample_rate"] = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -187,16 +188,13 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
async def _connect(self):
await self._connect_websocket()
self._receive_task = self.get_event_loop().create_task(
self._receive_task_handler(self.push_error)
)
self._receive_task = self.create_task(self._receive_task_handler(self.push_error))
async def _disconnect(self):
await self._disconnect_websocket()
if self._receive_task:
self._receive_task.cancel()
await self._receive_task
await self.cancel_task(self._receive_task)
self._receive_task = None
async def _connect_websocket(self):
@@ -265,7 +263,7 @@ class CartesiaTTSService(WordTTSService, WebsocketService):
self.start_word_timestamps()
frame = TTSAudioRawFrame(
audio=base64.b64decode(msg["data"]),
sample_rate=self._settings["output_format"]["sample_rate"],
sample_rate=self.sample_rate,
num_channels=1,
)
await self.push_frame(frame)
@@ -329,9 +327,9 @@ class CartesiaHttpTTSService(TTSService):
*,
api_key: str,
voice_id: str,
model: str = "sonic-english",
model: str = "sonic",
base_url: str = "https://api.cartesia.ai",
sample_rate: int = 24000,
sample_rate: Optional[int] = None,
encoding: str = "pcm_s16le",
container: str = "raw",
params: InputParams = InputParams(),
@@ -344,7 +342,7 @@ class CartesiaHttpTTSService(TTSService):
"output_format": {
"container": container,
"encoding": encoding,
"sample_rate": sample_rate,
"sample_rate": 0,
},
"language": self.language_to_service_language(params.language)
if params.language
@@ -363,6 +361,10 @@ class CartesiaHttpTTSService(TTSService):
def language_to_service_language(self, language: Language) -> str | None:
return language_to_cartesia_language(language)
async def start(self, frame: StartFrame):
await super().start(frame)
self._settings["output_format"]["sample_rate"] = self.sample_rate
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._client.close()
@@ -397,9 +399,7 @@ class CartesiaHttpTTSService(TTSService):
)
frame = TTSAudioRawFrame(
audio=output["audio"],
sample_rate=self._settings["output_format"]["sample_rate"],
num_channels=1,
audio=output["audio"], sample_rate=self.sample_rate, num_channels=1
)
yield frame
except Exception as e:

19
src/pipecat/services/deepgram.py
View File

@@ -5,7 +5,7 @@
#
import asyncio
from typing import AsyncGenerator
from typing import AsyncGenerator, Optional
from loguru import logger
@@ -53,14 +53,13 @@ class DeepgramTTSService(TTSService):
*,
api_key: str,
voice: str = "aura-helios-en",
sample_rate: int = 24000,
sample_rate: Optional[int] = None,
encoding: str = "linear16",
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
self._settings = {
"sample_rate": sample_rate,
"encoding": encoding,
}
self.set_voice(voice)
@@ -75,7 +74,7 @@ class DeepgramTTSService(TTSService):
options = SpeakOptions(
model=self._voice_id,
encoding=self._settings["encoding"],
sample_rate=self._settings["sample_rate"],
sample_rate=self.sample_rate,
container="none",
)
@@ -103,9 +102,7 @@ class DeepgramTTSService(TTSService):
chunk = audio_buffer.read(chunk_size)
if not chunk:
break
frame = TTSAudioRawFrame(
audio=chunk, sample_rate=self._settings["sample_rate"], num_channels=1
)
frame = TTSAudioRawFrame(audio=chunk, sample_rate=self.sample_rate, num_channels=1)
yield frame
yield TTSStoppedFrame()
@@ -121,15 +118,16 @@ class DeepgramSTTService(STTService):
*,
api_key: str,
url: str = "",
live_options: LiveOptions = None,
sample_rate: Optional[int] = None,
live_options: Optional[LiveOptions] = None,
**kwargs,
):
super().__init__(**kwargs)
super().__init__(sample_rate=sample_rate, **kwargs)
default_options = LiveOptions(
encoding="linear16",
language=Language.EN,
model="nova-2-general",
sample_rate=16000,
channels=1,
interim_results=True,
smart_format=True,
@@ -187,6 +185,7 @@ class DeepgramSTTService(STTService):
async def start(self, frame: StartFrame):
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):

114
src/pipecat/services/elevenlabs.py
View File

@@ -44,10 +44,14 @@ except ModuleNotFoundError as e:
ElevenLabsOutputFormat = Literal["pcm_16000", "pcm_22050", "pcm_24000", "pcm_44100"]
# Models that support language codes
# The following models are excluded as they don't support language codes:
# - eleven_flash_v2
# - eleven_turbo_v2
# - eleven_multilingual_v2
ELEVENLABS_MULTILINGUAL_MODELS = {
"eleven_turbo_v2_5",
"eleven_multilingual_v2",
"eleven_flash_v2_5",
"eleven_turbo_v2_5",
}
@@ -100,17 +104,20 @@ def language_to_elevenlabs_language(language: Language) -> str | None:
return result
def sample_rate_from_output_format(output_format: str) -> int:
match output_format:
case "pcm_16000":
return 16000
case "pcm_22050":
return 22050
case "pcm_24000":
return 24000
case "pcm_44100":
return 44100
return 16000
def output_format_from_sample_rate(sample_rate: int) -> str:
match sample_rate:
case 16000:
return "pcm_16000"
case 22050:
return "pcm_22050"
case 24000:
return "pcm_24000"
case 44100:
return "pcm_44100"
logger.warning(
f"ElevenLabsTTSService: No output format available for {sample_rate} sample rate"
)
return "pcm_16000"
def calculate_word_times(
@@ -136,7 +143,7 @@ def calculate_word_times(
class ElevenLabsTTSService(WordTTSService, WebsocketService):
class InputParams(BaseModel):
language: Optional[Language] = Language.EN
language: Optional[Language] = None
optimize_streaming_latency: Optional[str] = None
stability: Optional[float] = None
similarity_boost: Optional[float] = None
@@ -161,7 +168,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
voice_id: str,
model: str = "eleven_flash_v2_5",
url: str = "wss://api.elevenlabs.io",
output_format: ElevenLabsOutputFormat = "pcm_24000",
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
@@ -185,7 +192,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
push_text_frames=False,
push_stop_frames=True,
stop_frame_timeout_s=2.0,
sample_rate=sample_rate_from_output_format(output_format),
sample_rate=sample_rate,
**kwargs,
)
WebsocketService.__init__(self)
@@ -193,11 +200,9 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
self._api_key = api_key
self._url = url
self._settings = {
"sample_rate": sample_rate_from_output_format(output_format),
"language": self.language_to_service_language(params.language)
if params.language
else "en",
"output_format": output_format,
else None,
"optimize_streaming_latency": params.optimize_streaming_latency,
"stability": params.stability,
"similarity_boost": params.similarity_boost,
@@ -207,6 +212,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
}
self.set_model_name(model)
self.set_voice(voice_id)
self._output_format = "" # initialized in start()
self._voice_settings = self._set_voice_settings()
# Indicates if we have sent TTSStartedFrame. It will reset to False when
@@ -250,7 +256,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
await self._disconnect()
await self._connect()
async def _update_settings(self, settings: Dict[str, Any]):
async def _update_settings(self, settings: Mapping[str, Any]):
prev_voice = self._voice_id
await super()._update_settings(settings)
if not prev_voice == self._voice_id:
@@ -260,6 +266,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
async def start(self, frame: StartFrame):
await super().start(frame)
self._output_format = output_format_from_sample_rate(self.sample_rate)
await self._connect()
async def stop(self, frame: EndFrame):
@@ -298,20 +305,16 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
async def _connect(self):
await self._connect_websocket()
self._receive_task = self.get_event_loop().create_task(
self._receive_task_handler(self.push_error)
)
self._keepalive_task = self.get_event_loop().create_task(self._keepalive_task_handler())
self._receive_task = self.create_task(self._receive_task_handler(self.push_error))
self._keepalive_task = self.create_task(self._keepalive_task_handler())
async def _disconnect(self):
if self._receive_task:
self._receive_task.cancel()
await self._receive_task
await self.cancel_task(self._receive_task)
self._receive_task = None
if self._keepalive_task:
self._keepalive_task.cancel()
await self._keepalive_task
await self.cancel_task(self._keepalive_task)
self._keepalive_task = None
await self._disconnect_websocket()
@@ -322,7 +325,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
voice_id = self._voice_id
model = self.model_name
output_format = self._settings["output_format"]
output_format = self._output_format
url = f"{self._url}/v1/text-to-speech/{voice_id}/stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings['auto_mode']}"
if self._settings["optimize_streaming_latency"]:
@@ -330,14 +333,16 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
# Language can only be used with the ELEVENLABS_MULTILINGUAL_MODELS
language = self._settings["language"]
if model in ELEVENLABS_MULTILINGUAL_MODELS:
if model in ELEVENLABS_MULTILINGUAL_MODELS and language is not None:
url += f"&language_code={language}"
else:
logger.debug(f"Using language code: {language}")
elif language is not None:
logger.warning(
f"Language code [{language}] not applied. Language codes can only be used with multilingual models: {', '.join(sorted(ELEVENLABS_MULTILINGUAL_MODELS))}"
)
self._websocket = await websockets.connect(url)
# Set max websocket message size to 16MB for large audio responses
self._websocket = await websockets.connect(url, max_size=16 * 1024 * 1024)
# According to ElevenLabs, we should always start with a single space.
msg: Dict[str, Any] = {
@@ -373,7 +378,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
self.start_word_timestamps()
audio = base64.b64decode(msg["audio"])
frame = TTSAudioRawFrame(audio, self._settings["sample_rate"], 1)
frame = TTSAudioRawFrame(audio, self.sample_rate, 1)
await self.push_frame(frame)
if msg.get("alignment"):
word_times = calculate_word_times(msg["alignment"], self._cumulative_time)
@@ -382,13 +387,8 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService):
async def _keepalive_task_handler(self):
while True:
try:
await asyncio.sleep(10)
await self._send_text("")
except asyncio.CancelledError:
break
except Exception as e:
logger.error(f"{self} exception: {e}")
await asyncio.sleep(10)
await self._send_text("")
async def _send_text(self, text: str):
if self._websocket:
@@ -431,12 +431,12 @@ class ElevenLabsHttpTTSService(TTSService):
aiohttp_session: aiohttp ClientSession
model: Model ID (default: "eleven_flash_v2_5" for low latency)
base_url: API base URL
output_format: Audio output format (PCM)
sample_rate: Output sample rate
params: Additional parameters for voice configuration
"""
class InputParams(BaseModel):
language: Optional[Language] = Language.EN
language: Optional[Language] = None
optimize_streaming_latency: Optional[int] = None
stability: Optional[float] = None
similarity_boost: Optional[float] = None
@@ -451,24 +451,21 @@ class ElevenLabsHttpTTSService(TTSService):
aiohttp_session: aiohttp.ClientSession,
model: str = "eleven_flash_v2_5",
base_url: str = "https://api.elevenlabs.io",
output_format: ElevenLabsOutputFormat = "pcm_24000",
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(sample_rate=sample_rate_from_output_format(output_format), **kwargs)
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._base_url = base_url
self._output_format = output_format
self._params = params
self._session = aiohttp_session
self._settings = {
"sample_rate": sample_rate_from_output_format(output_format),
"language": self.language_to_service_language(params.language)
if params.language
else "en",
"output_format": output_format,
else None,
"optimize_streaming_latency": params.optimize_streaming_latency,
"stability": params.stability,
"similarity_boost": params.similarity_boost,
@@ -477,6 +474,7 @@ class ElevenLabsHttpTTSService(TTSService):
}
self.set_model_name(model)
self.set_voice(voice_id)
self._output_format = "" # initialized in start()
self._voice_settings = self._set_voice_settings()
def can_generate_metrics(self) -> bool:
@@ -511,6 +509,10 @@ class ElevenLabsHttpTTSService(TTSService):
return voice_settings or None
async def start(self, frame: StartFrame):
await super().start(frame)
self._output_format = output_format_from_sample_rate(self.sample_rate)
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using ElevenLabs streaming API.
@@ -524,16 +526,22 @@ class ElevenLabsHttpTTSService(TTSService):
url = f"{self._base_url}/v1/text-to-speech/{self._voice_id}/stream"
payload = {
payload: Dict[str, Union[str, Dict[str, Union[float, bool]]]] = {
"text": text,
"model_id": self._model_name,
}
if self._voice_settings:
payload["voice_settings"] = json.dumps(self._voice_settings)
payload["voice_settings"] = self._voice_settings
if self._settings["language"]:
payload["language_code"] = self._settings["language"]
language = self._settings["language"]
if self._model_name in ELEVENLABS_MULTILINGUAL_MODELS and language:
payload["language_code"] = language
logger.debug(f"Using language code: {language}")
elif language:
logger.warning(
f"Language code [{language}] not applied. Language codes can only be used with multilingual models: {', '.join(sorted(ELEVENLABS_MULTILINGUAL_MODELS))}"
)
headers = {
"xi-api-key": self._api_key,
@@ -567,7 +575,7 @@ class ElevenLabsHttpTTSService(TTSService):
async for chunk in response.content:
if chunk:
await self.stop_ttfb_metrics()
yield TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
yield TTSAudioRawFrame(chunk, self.sample_rate, 1)
yield TTSStoppedFrame()

16
src/pipecat/services/fish.py
View File

@@ -56,7 +56,7 @@ class FishAudioTTSService(TTSService, WebsocketService):
api_key: str,
model: str, # This is the reference_id
output_format: FishAudioOutputFormat = "pcm",
sample_rate: int = 24000,
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
@@ -70,7 +70,7 @@ class FishAudioTTSService(TTSService, WebsocketService):
self._started = False
self._settings = {
"sample_rate": sample_rate,
"sample_rate": 0,
"latency": params.latency,
"format": output_format,
"prosody": {
@@ -92,6 +92,7 @@ class FishAudioTTSService(TTSService, WebsocketService):
async def start(self, frame: StartFrame):
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
await self._connect()
async def stop(self, frame: EndFrame):
@@ -104,15 +105,12 @@ class FishAudioTTSService(TTSService, WebsocketService):
async def _connect(self):
await self._connect_websocket()
self._receive_task = self.get_event_loop().create_task(
self._receive_task_handler(self.push_error)
)
self._receive_task = self.create_task(self._receive_task_handler(self.push_error))
async def _disconnect(self):
await self._disconnect_websocket()
if self._receive_task:
self._receive_task.cancel()
await self._receive_task
await self.cancel_task(self._receive_task)
self._receive_task = None
async def _connect_websocket(self):
@@ -160,9 +158,7 @@ class FishAudioTTSService(TTSService, WebsocketService):
audio_data = msg.get("audio")
# Only process larger chunks to remove msgpack overhead
if audio_data and len(audio_data) > 1024:
frame = TTSAudioRawFrame(
audio_data, self._settings["sample_rate"], 1
)
frame = TTSAudioRawFrame(audio_data, self.sample_rate, 1)
await self.push_frame(frame)
await self.stop_ttfb_metrics()
continue

2
src/pipecat/services/gemini_multimodal_live/events.py
View File

@@ -48,7 +48,7 @@ class AudioInputMessage(BaseModel):
realtimeInput: RealtimeInput
@classmethod
def from_raw_audio(cls, raw_audio: bytes, sample_rate=16000) -> "AudioInputMessage":
def from_raw_audio(cls, raw_audio: bytes, sample_rate: int) -> "AudioInputMessage":
data = base64.b64encode(raw_audio).decode("utf-8")
return cls(
realtimeInput=RealtimeInput(

105
src/pipecat/services/gemini_multimodal_live/gemini.py
View File

@@ -182,9 +182,13 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._audio_input_paused = start_audio_paused
self._video_input_paused = start_video_paused
self._context = None
self._websocket = None
self._receive_task = None
self._context = None
self._transcribe_audio_task = None
self._transcribe_model_audio_task = None
self._transcribe_audio_queue = asyncio.Queue()
self._transcribe_model_audio_queue = asyncio.Queue()
self._disconnecting = False
self._api_session_ready = False
@@ -199,6 +203,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._bot_audio_buffer = bytearray()
self._bot_text_buffer = ""
self._sample_rate = 24000
self._settings = {
"frequency_penalty": params.frequency_penalty,
"max_tokens": params.max_tokens,
@@ -244,6 +250,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
async def start(self, frame: StartFrame):
await super().start(frame)
await self._connect()
async def stop(self, frame: EndFrame):
await super().stop(frame)
@@ -275,7 +282,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
)
await self.send_client_event(evt)
if self._transcribe_user_audio and self._context:
asyncio.create_task(self._handle_transcribe_user_audio(audio, self._context))
await self._transcribe_audio_queue.put(audio)
async def _handle_transcribe_user_audio(self, audio, context):
text = await self._transcribe_audio(audio, context)
@@ -288,6 +295,10 @@ class GeminiMultimodalLiveLLMService(LLMService):
)
async def _handle_transcribe_model_audio(self, audio, context):
# Early return if modalities are not set to audio.
if self._settings["modalities"] != GeminiMultimodalModalities.AUDIO:
return
text = await self._transcribe_audio(audio, context)
logger.debug(f"[Transcription:model] {text}")
# We add user messages directly to the context. We don't do that for assistant messages,
@@ -377,17 +388,21 @@ class GeminiMultimodalLiveLLMService(LLMService):
await self._ws_send(event.model_dump(exclude_none=True))
async def _connect(self):
if self._websocket:
# Here we assume that if we have a websocket, we are connected. We
# handle disconnections in the send/recv code paths.
return
logger.info("Connecting to Gemini service")
try:
if self._websocket:
# Here we assume that if we have a websocket, we are connected. We
# handle disconnections in the send/recv code paths.
return
uri = f"wss://{self.base_url}/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent?key={self.api_key}"
logger.info(f"Connecting to {uri}")
self._websocket = await websockets.connect(uri=uri)
self._receive_task = self.get_event_loop().create_task(self._receive_task_handler())
self._receive_task = self.create_task(self._receive_task_handler())
self._transcribe_audio_task = self.create_task(self._transcribe_audio_handler())
self._transcribe_model_audio_task = self.create_task(
self._transcribe_model_audio_handler()
)
config = events.Config.model_validate(
{
"setup": {
@@ -437,12 +452,14 @@ class GeminiMultimodalLiveLLMService(LLMService):
await self._websocket.close()
self._websocket = None
if self._receive_task:
self._receive_task.cancel()
try:
await asyncio.wait_for(self._receive_task, timeout=1.0)
except asyncio.TimeoutError:
logger.warning("Timed out waiting for receive task to finish")
await self.cancel_task(self._receive_task, timeout=1.0)
self._receive_task = None
if self._transcribe_audio_task:
await self.cancel_task(self._transcribe_audio_task)
self._transcribe_audio_task = None
if self._transcribe_model_audio_task:
await self.cancel_task(self._transcribe_model_audio_task)
self._transcribe_model_audio_task = None
self._disconnecting = False
except Exception as e:
logger.error(f"{self} error disconnecting: {e}")
@@ -450,9 +467,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
async def _ws_send(self, message):
# logger.debug(f"Sending message to websocket: {message}")
try:
if not self._websocket:
await self._connect()
await self._websocket.send(json.dumps(message))
if self._websocket:
await self._websocket.send(json.dumps(message))
except Exception as e:
if self._disconnecting:
return
@@ -469,32 +485,35 @@ class GeminiMultimodalLiveLLMService(LLMService):
#
async def _receive_task_handler(self):
try:
async for message in self._websocket:
evt = events.parse_server_event(message)
# logger.debug(f"Received event: {message[:500]}")
# logger.debug(f"Received event: {evt}")
async for message in self._websocket:
evt = events.parse_server_event(message)
# logger.debug(f"Received event: {message[:500]}")
# logger.debug(f"Received event: {evt}")
if evt.setupComplete:
await self._handle_evt_setup_complete(evt)
elif evt.serverContent and evt.serverContent.modelTurn:
await self._handle_evt_model_turn(evt)
elif evt.serverContent and evt.serverContent.turnComplete:
await self._handle_evt_turn_complete(evt)
elif evt.toolCall:
await self._handle_evt_tool_call(evt)
if evt.setupComplete:
await self._handle_evt_setup_complete(evt)
elif evt.serverContent and evt.serverContent.modelTurn:
await self._handle_evt_model_turn(evt)
elif evt.serverContent and evt.serverContent.turnComplete:
await self._handle_evt_turn_complete(evt)
elif evt.toolCall:
await self._handle_evt_tool_call(evt)
elif False: # !!! todo: error events?
await self._handle_evt_error(evt)
# errors are fatal, so exit the receive loop
return
else:
pass
elif False: # !!! todo: error events?
await self._handle_evt_error(evt)
# errors are fatal, so exit the receive loop
return
async def _transcribe_audio_handler(self):
while True:
audio = await self._transcribe_audio_queue.get()
await self._handle_transcribe_user_audio(audio, self._context)
else:
pass
except asyncio.CancelledError:
logger.debug("websocket receive task cancelled")
except Exception as e:
logger.error(f"{self} exception: {e}")
async def _transcribe_model_audio_handler(self):
while True:
audio = await self._transcribe_model_audio_queue.get()
await self._handle_transcribe_model_audio(audio, self._context)
#
#
@@ -504,7 +523,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
if self._audio_input_paused:
return
# Send all audio to Gemini
evt = events.AudioInputMessage.from_raw_audio(frame.audio)
evt = events.AudioInputMessage.from_raw_audio(frame.audio, frame.sample_rate)
await self.send_client_event(evt)
# Manage a buffer of audio to use for transcription
audio = frame.audio
@@ -633,7 +652,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
inline_data = part.inlineData
if not inline_data:
return
if inline_data.mimeType != "audio/pcm;rate=24000":
if inline_data.mimeType != f"audio/pcm;rate={self._sample_rate}":
logger.warning(f"Unrecognized server_content format {inline_data.mimeType}")
return
@@ -648,7 +667,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._bot_audio_buffer.extend(audio)
frame = TTSAudioRawFrame(
audio=audio,
sample_rate=24000,
sample_rate=self._sample_rate,
num_channels=1,
)
await self.push_frame(frame)
@@ -675,7 +694,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._bot_text_buffer = ""
if audio and self._transcribe_model_audio and self._context:
asyncio.create_task(self._handle_transcribe_model_audio(audio, self._context))
await self._transcribe_model_audio_queue.put(audio)
elif text:
await self.push_frame(LLMFullResponseEndFrame())

11
src/pipecat/services/gladia.py
View File

@@ -131,7 +131,6 @@ def language_to_gladia_language(language: Language) -> str | None:
class GladiaSTTService(STTService):
class InputParams(BaseModel):
sample_rate: Optional[int] = 16000
language: Optional[Language] = Language.EN
endpointing: Optional[float] = 0.2
maximum_duration_without_endpointing: Optional[int] = 10
@@ -144,17 +143,18 @@ class GladiaSTTService(STTService):
api_key: str,
url: str = "https://api.gladia.io/v2/live",
confidence: float = 0.5,
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(**kwargs)
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._url = url
self._settings = {
"encoding": "wav/pcm",
"bit_depth": 16,
"sample_rate": params.sample_rate,
"sample_rate": 0,
"channels": 1,
"language_config": {
"languages": [self.language_to_service_language(params.language)]
@@ -178,18 +178,21 @@ class GladiaSTTService(STTService):
async def start(self, frame: StartFrame):
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
response = await self._setup_gladia()
self._websocket = await websockets.connect(response["url"])
self._receive_task = self.get_event_loop().create_task(self._receive_task_handler())
self._receive_task = self.create_task(self._receive_task_handler())
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._send_stop_recording()
await self._websocket.close()
await self.wait_for_task(self._receive_task)
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._websocket.close()
await self.cancel_task(self._receive_task)
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
await self.start_processing_metrics()

2
src/pipecat/services/google/__init__.py Normal file
View File

@@ -0,0 +1,2 @@
from .frames import LLMSearchResponseFrame
from .google import *

33
src/pipecat/services/google/frames.py Normal file
View File

@@ -0,0 +1,33 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
from dataclasses import dataclass, field
from typing import List, Optional
from pipecat.frames.frames import DataFrame
@dataclass
class LLMSearchResult:
text: str
confidence: List[float] = field(default_factory=list)
@dataclass
class LLMSearchOrigin:
site_uri: Optional[str] = None
site_title: Optional[str] = None
results: List[LLMSearchResult] = field(default_factory=list)
@dataclass
class LLMSearchResponseFrame(DataFrame):
search_result: Optional[str] = None
rendered_content: Optional[str] = None
origins: List[LLMSearchOrigin] = field(default_factory=list)
def __str__(self):
return f"LLMSearchResponseFrame(search_result={self.search_result}, origins={self.origins})"

217
src/pipecat/services/google.py → src/pipecat/services/google/google.py
View File

@@ -29,6 +29,7 @@ from pipecat.frames.frames import (
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
URLImageRawFrame,
VisionImageRawFrame,
)
from pipecat.metrics.metrics import LLMTokenUsage
@@ -37,7 +38,8 @@ from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import LLMService, TTSService
from pipecat.services.ai_services import ImageGenService, LLMService, TTSService
from pipecat.services.google.frames import LLMSearchResponseFrame
from pipecat.services.openai import (
OpenAIAssistantContextAggregator,
OpenAIUserContextAggregator,
@@ -48,7 +50,9 @@ from pipecat.utils.time import time_now_iso8601
try:
import google.ai.generativelanguage as glm
import google.generativeai as gai
from google import genai
from google.cloud import texttospeech_v1
from google.genai import types
from google.generativeai.types import GenerationConfig
from google.oauth2 import service_account
except ModuleNotFoundError as e:
@@ -563,23 +567,56 @@ class GoogleLLMContext(OpenAILLMContext):
return [msg]
def _restructure_from_openai_messages(self):
"""Restructures messages to ensure proper Google format and message ordering.
This method handles conversion of OpenAI-formatted messages to Google format,
with special handling for function calls, function responses, and system messages.
System messages are added back to the context as user messages when needed.
The final message order is preserved as:
1. Function calls (from model)
2. Function responses (from user)
3. Text messages (converted from system messages)
Note:
System messages are only added back when there are no regular text
messages in the context, ensuring proper conversation continuity
after function calls.
"""
self.system_message = None
# first, map across self._messages calling self.from_standard_message(m) to modify messages in place
try:
self._messages[:] = [
msg
for msg in (self.from_standard_message(m) for m in self._messages)
if msg is not None
]
# We might have been given a messages list with only a system message. If so, let's put that back in
# the messages list as a user message.
if self.system_message and not self._messages:
self.add_message(
glm.Content(role="user", parts=[glm.Part(text=self.system_message)])
)
except Exception as e:
logger.error(f"Error mapping messages: {e}")
# iterate over messages and remove any messages that have an empty content list
converted_messages = []
# Process each message, preserving Google-formatted messages and converting others
for message in self._messages:
if isinstance(message, glm.Content):
# Keep existing Google-formatted messages (e.g., function calls/responses)
converted_messages.append(message)
continue
# Convert OpenAI format to Google format, system messages return None
converted = self.from_standard_message(message)
if converted is not None:
converted_messages.append(converted)
# Update message list
self._messages[:] = converted_messages
# Check if we only have function-related messages (no regular text)
has_regular_messages = any(
len(msg.parts) == 1
and hasattr(msg.parts[0], "text")
and not hasattr(msg.parts[0], "function_call")
and not hasattr(msg.parts[0], "function_response")
for msg in self._messages
)
# Add system message back as a user message if we only have function messages
if self.system_message and not has_regular_messages:
self._messages.append(
glm.Content(role="user", parts=[glm.Part(text=self.system_message)])
)
# Remove any empty messages
self._messages = [m for m in self._messages if m.parts]
@@ -639,6 +676,9 @@ class GoogleLLMService(LLMService):
completion_tokens = 0
total_tokens = 0
grounding_metadata = None
search_result = ""
try:
logger.debug(
# f"Generating chat: {self._system_instruction} | {context.get_messages_for_logging()}"
@@ -674,7 +714,6 @@ class GoogleLLMService(LLMService):
tool_config = None
if self._tool_config:
tool_config = self._tool_config
response = await self._client.generate_content_async(
contents=messages,
tools=tools,
@@ -698,9 +737,10 @@ class GoogleLLMService(LLMService):
try:
for c in chunk.parts:
if c.text:
search_result += c.text
await self.push_frame(LLMTextFrame(c.text))
elif c.function_call:
logger.debug(f"!!! Function call: {c.function_call}")
logger.debug(f"Function call: {c.function_call}")
args = type(c.function_call).to_dict(c.function_call).get("args", {})
await self.call_function(
context=context,
@@ -708,6 +748,63 @@ class GoogleLLMService(LLMService):
function_name=c.function_call.name,
arguments=args,
)
# Handle grounding metadata
# It seems only the last chunk that we receive may contain this information
# If the response doesn't include groundingMetadata, this means the response wasn't grounded.
if chunk.candidates:
for candidate in chunk.candidates:
# logger.debug(f"candidate received: {candidate}")
# Extract grounding metadata
grounding_metadata = (
{
"rendered_content": getattr(
getattr(candidate, "grounding_metadata", None),
"search_entry_point",
None,
).rendered_content
if hasattr(
getattr(candidate, "grounding_metadata", None),
"search_entry_point",
)
else None,
"origins": [
{
"site_uri": getattr(grounding_chunk.web, "uri", None),
"site_title": getattr(
grounding_chunk.web, "title", None
),
"results": [
{
"text": getattr(
grounding_support.segment, "text", ""
),
"confidence": getattr(
grounding_support, "confidence_scores", None
),
}
for grounding_support in getattr(
getattr(candidate, "grounding_metadata", None),
"grounding_supports",
[],
)
if index
in getattr(
grounding_support, "grounding_chunk_indices", []
)
],
}
for index, grounding_chunk in enumerate(
getattr(
getattr(candidate, "grounding_metadata", None),
"grounding_chunks",
[],
)
)
],
}
if getattr(candidate, "grounding_metadata", None)
else None
)
except Exception as e:
# Google LLMs seem to flag safety issues a lot!
if chunk.candidates[0].finish_reason == 3:
@@ -720,6 +817,14 @@ class GoogleLLMService(LLMService):
except Exception as e:
logger.exception(f"{self} exception: {e}")
finally:
if grounding_metadata is not None and isinstance(grounding_metadata, dict):
llm_search_frame = LLMSearchResponseFrame(
search_result=search_result,
origins=grounding_metadata["origins"],
rendered_content=grounding_metadata["rendered_content"],
)
await self.push_frame(llm_search_frame)
await self.start_llm_usage_metrics(
LLMTokenUsage(
prompt_tokens=prompt_tokens,
@@ -778,14 +883,13 @@ class GoogleTTSService(TTSService):
credentials: Optional[str] = None,
credentials_path: Optional[str] = None,
voice_id: str = "en-US-Neural2-A",
sample_rate: int = 24000,
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
self._settings = {
"sample_rate": sample_rate,
"pitch": params.pitch,
"rate": params.rate,
"volume": params.volume,
@@ -891,7 +995,7 @@ class GoogleTTSService(TTSService):
)
audio_config = texttospeech_v1.AudioConfig(
audio_encoding=texttospeech_v1.AudioEncoding.LINEAR16,
sample_rate_hertz=self._settings["sample_rate"],
sample_rate_hertz=self.sample_rate,
)
request = texttospeech_v1.SynthesizeSpeechRequest(
@@ -914,7 +1018,7 @@ class GoogleTTSService(TTSService):
if not chunk:
break
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
yield frame
await asyncio.sleep(0) # Allow other tasks to run
@@ -926,3 +1030,70 @@ class GoogleTTSService(TTSService):
yield ErrorFrame(error=error_message)
finally:
yield TTSStoppedFrame()
class GoogleImageGenService(ImageGenService):
class InputParams(BaseModel):
number_of_images: int = Field(default=1, ge=1, le=8)
model: str = Field(default="imagen-3.0-generate-002")
negative_prompt: str = Field(default=None)
def __init__(
self,
*,
params: InputParams = InputParams(),
api_key: str,
**kwargs,
):
super().__init__(**kwargs)
self.set_model_name(params.model)
self._params = params
self._client = genai.Client(api_key=api_key)
def can_generate_metrics(self) -> bool:
return True
async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]:
"""Generate images from a text prompt using Google's Imagen model.
Args:
prompt (str): The text description to generate images from.
Yields:
Frame: Generated image frames or error frames.
"""
logger.debug(f"Generating image from prompt: {prompt}")
await self.start_ttfb_metrics()
try:
response = await self._client.aio.models.generate_images(
model=self._params.model,
prompt=prompt,
config=types.GenerateImagesConfig(
number_of_images=self._params.number_of_images,
negative_prompt=self._params.negative_prompt,
),
)
await self.stop_ttfb_metrics()
if not response or not response.generated_images:
logger.error(f"{self} error: image generation failed")
yield ErrorFrame("Image generation failed")
return
for img_response in response.generated_images:
# Google returns the image data directly
image_bytes = img_response.image.image_bytes
image = Image.open(io.BytesIO(image_bytes))
frame = URLImageRawFrame(
url=None, # Google doesn't provide URLs, only image data
image=image.tobytes(),
size=image.size,
format=image.format,
)
yield frame
except Exception as e:
logger.error(f"{self} error generating image: {e}")
yield ErrorFrame(f"Image generation error: {str(e)}")

16
src/pipecat/services/lmnt.py
View File

@@ -5,7 +5,7 @@
#
import json
from typing import AsyncGenerator
from typing import AsyncGenerator, Optional
from loguru import logger
@@ -66,7 +66,7 @@ class LmntTTSService(TTSService, WebsocketService):
*,
api_key: str,
voice_id: str,
sample_rate: int = 24000,
sample_rate: Optional[int] = None,
language: Language = Language.EN,
**kwargs,
):
@@ -81,7 +81,6 @@ class LmntTTSService(TTSService, WebsocketService):
self._api_key = api_key
self._voice_id = voice_id
self._settings = {
"sample_rate": sample_rate,
"language": self.language_to_service_language(language),
"format": "raw", # Use raw format for direct PCM data
}
@@ -113,16 +112,13 @@ class LmntTTSService(TTSService, WebsocketService):
async def _connect(self):
await self._connect_websocket()
self._receive_task = self.get_event_loop().create_task(
self._receive_task_handler(self.push_error)
)
self._receive_task = self.create_task(self._receive_task_handler(self.push_error))
async def _disconnect(self):
await self._disconnect_websocket()
if self._receive_task:
self._receive_task.cancel()
await self._receive_task
await self.cancel_task(self._receive_task)
self._receive_task = None
async def _connect_websocket(self):
@@ -135,7 +131,7 @@ class LmntTTSService(TTSService, WebsocketService):
"X-API-Key": self._api_key,
"voice": self._voice_id,
"format": self._settings["format"],
"sample_rate": self._settings["sample_rate"],
"sample_rate": self.sample_rate,
"language": self._settings["language"],
}
@@ -178,7 +174,7 @@ class LmntTTSService(TTSService, WebsocketService):
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(
audio=message,
sample_rate=self._settings["sample_rate"],
sample_rate=self.sample_rate,
num_channels=1,
)
await self.push_frame(frame)

37
src/pipecat/services/openai.py
View File

@@ -28,6 +28,7 @@ from pipecat.frames.frames import (
LLMTextFrame,
LLMUpdateSettingsFrame,
OpenAILLMContextAssistantTimestampFrame,
StartFrame,
StartInterruptionFrame,
TTSAudioRawFrame,
TTSStartedFrame,
@@ -116,6 +117,8 @@ class BaseOpenAILLMService(LLMService):
model: str,
api_key=None,
base_url=None,
organization=None,
project=None,
params: InputParams = InputParams(),
**kwargs,
):
@@ -131,12 +134,16 @@ class BaseOpenAILLMService(LLMService):
"extra": params.extra if isinstance(params.extra, dict) else {},
}
self.set_model_name(model)
self._client = self.create_client(api_key=api_key, base_url=base_url, **kwargs)
self._client = self.create_client(
api_key=api_key, base_url=base_url, organization=organization, project=project, **kwargs
)
def create_client(self, api_key=None, base_url=None, **kwargs):
def create_client(self, api_key=None, base_url=None, organization=None, project=None, **kwargs):
return AsyncOpenAI(
api_key=api_key,
base_url=base_url,
organization=organization,
project=project,
http_client=DefaultAsyncHttpxClient(
limits=httpx.Limits(
max_keepalive_connections=100, max_connections=1000, keepalive_expiry=None
@@ -221,7 +228,7 @@ class BaseOpenAILLMService(LLMService):
)
await self.start_llm_usage_metrics(tokens)
if len(chunk.choices) == 0:
if chunk.choices is None or len(chunk.choices) == 0:
continue
await self.stop_ttfb_metrics()
@@ -406,20 +413,24 @@ class OpenAITTSService(TTSService):
The service returns PCM-encoded audio at the specified sample rate.
"""
OPENAI_SAMPLE_RATE = 24000 # OpenAI TTS always outputs at 24kHz
def __init__(
self,
*,
api_key: str | None = None,
api_key: Optional[str] = None,
voice: str = "alloy",
model: Literal["tts-1", "tts-1-hd"] = "tts-1",
sample_rate: int = 24000,
sample_rate: Optional[int] = None,
**kwargs,
):
if sample_rate and sample_rate != self.OPENAI_SAMPLE_RATE:
logger.warning(
f"OpenAI TTS only supports {self.OPENAI_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {self.sample_rate}Hz may cause issues."
)
super().__init__(sample_rate=sample_rate, **kwargs)
self._settings = {
"sample_rate": sample_rate,
}
self.set_model_name(model)
self.set_voice(voice)
@@ -432,6 +443,14 @@ class OpenAITTSService(TTSService):
logger.info(f"Switching TTS model to: [{model}]")
self.set_model_name(model)
async def start(self, frame: StartFrame):
await super().start(frame)
if self.sample_rate != self.OPENAI_SAMPLE_RATE:
logger.warning(
f"OpenAI TTS requires {self.OPENAI_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {self.sample_rate}Hz may cause issues."
)
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
try:
@@ -459,7 +478,7 @@ class OpenAITTSService(TTSService):
async for chunk in r.iter_bytes(8192):
if len(chunk) > 0:
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
yield frame
yield TTSStoppedFrame()
except BadRequestError as e:

22
src/pipecat/services/openai_realtime_beta/context.py
View File

@@ -6,10 +6,16 @@
import copy
import json
from typing import Optional
from loguru import logger
from pipecat.frames.frames import Frame, LLMMessagesUpdateFrame, LLMSetToolsFrame
from pipecat.frames.frames import (
Frame,
FunctionCallResultProperties,
LLMMessagesUpdateFrame,
LLMSetToolsFrame,
)
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
OpenAILLMContextFrame,
@@ -174,10 +180,13 @@ class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator)
if not self._function_call_result:
return
properties: Optional[FunctionCallResultProperties] = None
self._reset()
try:
run_llm = True
frame = self._function_call_result
properties = frame.properties
self._function_call_result = None
if frame.result:
# The "tool_call" message from the LLM that triggered the function call
@@ -211,11 +220,20 @@ class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator)
await self._user_context_aggregator.push_frame(
RealtimeFunctionCallResultFrame(result_frame=frame)
)
run_llm = frame.run_llm
if properties and properties.run_llm is not None:
# If the tool call result has a run_llm property, use it
run_llm = properties.run_llm
else:
# Default behavior is to run the LLM if there are no function calls in progress
run_llm = not bool(self._function_calls_in_progress)
if run_llm:
await self._user_context_aggregator.push_context_frame()
# Emit the on_context_updated callback once the function call result is added to the context
if properties and properties.on_context_updated is not None:
await properties.on_context_updated()
frame = OpenAILLMContextFrame(self._context)
await self.push_frame(frame)

81
src/pipecat/services/openai_realtime_beta/openai.py
View File

@@ -167,8 +167,11 @@ class OpenAIRealtimeBetaLLMService(LLMService):
either the wall clock time or the actual audio duration to prevent invalid truncation
requests.
"""
if not self._current_audio_response:
return
# if the bot is still speaking, truncate the last message
if self._current_audio_response:
try:
current = self._current_audio_response
self._current_audio_response = None
@@ -179,6 +182,11 @@ class OpenAIRealtimeBetaLLMService(LLMService):
elapsed_ms = int(time.time() * 1000 - current.start_time_ms)
truncate_ms = min(elapsed_ms, audio_duration_ms)
logger.trace(
f"Truncating audio: duration={audio_duration_ms}ms, "
f"elapsed={elapsed_ms}ms, truncate={truncate_ms}ms"
)
await self.send_client_event(
events.ConversationItemTruncateEvent(
item_id=current.item_id,
@@ -186,6 +194,9 @@ class OpenAIRealtimeBetaLLMService(LLMService):
audio_end_ms=truncate_ms,
)
)
except Exception as e:
# Log warning and don't re-raise - allow session to continue
logger.warning(f"Audio truncation failed (non-fatal): {e}")
#
# frame processing
@@ -266,7 +277,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
"OpenAI-Beta": "realtime=v1",
},
)
self._receive_task = self.get_event_loop().create_task(self._receive_task_handler())
self._receive_task = self.create_task(self._receive_task_handler())
except Exception as e:
logger.error(f"{self} initialization error: {e}")
self._websocket = None
@@ -280,11 +291,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
await self._websocket.close()
self._websocket = None
if self._receive_task:
self._receive_task.cancel()
try:
await asyncio.wait_for(self._receive_task, timeout=1.0)
except asyncio.TimeoutError:
logger.warning("Timed out waiting for receive task to finish")
await self.cancel_task(self._receive_task, timeout=1.0)
self._receive_task = None
self._disconnecting = False
except Exception as e:
@@ -321,40 +328,32 @@ class OpenAIRealtimeBetaLLMService(LLMService):
#
async def _receive_task_handler(self):
try:
async for message in self._websocket:
evt = events.parse_server_event(message)
if evt.type == "session.created":
await self._handle_evt_session_created(evt)
elif evt.type == "session.updated":
await self._handle_evt_session_updated(evt)
elif evt.type == "response.audio.delta":
await self._handle_evt_audio_delta(evt)
elif evt.type == "response.audio.done":
await self._handle_evt_audio_done(evt)
elif evt.type == "conversation.item.created":
await self._handle_evt_conversation_item_created(evt)
elif evt.type == "conversation.item.input_audio_transcription.completed":
await self.handle_evt_input_audio_transcription_completed(evt)
elif evt.type == "response.done":
await self._handle_evt_response_done(evt)
elif evt.type == "input_audio_buffer.speech_started":
await self._handle_evt_speech_started(evt)
elif evt.type == "input_audio_buffer.speech_stopped":
await self._handle_evt_speech_stopped(evt)
elif evt.type == "response.audio_transcript.delta":
await self._handle_evt_audio_transcript_delta(evt)
elif evt.type == "error":
await self._handle_evt_error(evt)
# errors are fatal, so exit the receive loop
return
else:
pass
except asyncio.CancelledError:
logger.debug("websocket receive task cancelled")
except Exception as e:
logger.error(f"{self} exception: {e}")
async for message in self._websocket:
evt = events.parse_server_event(message)
if evt.type == "session.created":
await self._handle_evt_session_created(evt)
elif evt.type == "session.updated":
await self._handle_evt_session_updated(evt)
elif evt.type == "response.audio.delta":
await self._handle_evt_audio_delta(evt)
elif evt.type == "response.audio.done":
await self._handle_evt_audio_done(evt)
elif evt.type == "conversation.item.created":
await self._handle_evt_conversation_item_created(evt)
elif evt.type == "conversation.item.input_audio_transcription.completed":
await self.handle_evt_input_audio_transcription_completed(evt)
elif evt.type == "response.done":
await self._handle_evt_response_done(evt)
elif evt.type == "input_audio_buffer.speech_started":
await self._handle_evt_speech_started(evt)
elif evt.type == "input_audio_buffer.speech_stopped":
await self._handle_evt_speech_stopped(evt)
elif evt.type == "response.audio_transcript.delta":
await self._handle_evt_audio_transcript_delta(evt)
elif evt.type == "error":
await self._handle_evt_error(evt)
# errors are fatal, so exit the receive loop
return
async def _handle_evt_session_created(self, evt):
# session.created is received right after connecting. Send a message

31
src/pipecat/services/playht.py
View File

@@ -113,7 +113,7 @@ class PlayHTTTSService(TTSService, WebsocketService):
user_id: str,
voice_url: str,
voice_engine: str = "Play3.0-mini",
sample_rate: int = 24000,
sample_rate: Optional[int] = None,
output_format: str = "wav",
params: InputParams = InputParams(),
**kwargs,
@@ -132,7 +132,6 @@ class PlayHTTTSService(TTSService, WebsocketService):
self._request_id = None
self._settings = {
"sample_rate": sample_rate,
"language": self.language_to_service_language(params.language)
if params.language
else "english",
@@ -165,16 +164,13 @@ class PlayHTTTSService(TTSService, WebsocketService):
async def _connect(self):
await self._connect_websocket()
self._receive_task = self.get_event_loop().create_task(
self._receive_task_handler(self.push_error)
)
self._receive_task = self.create_task(self._receive_task_handler(self.push_error))
async def _disconnect(self):
await self._disconnect_websocket()
if self._receive_task:
self._receive_task.cancel()
await self._receive_task
await self.cancel_task(self._receive_task)
self._receive_task = None
async def _connect_websocket(self):
@@ -253,7 +249,7 @@ class PlayHTTTSService(TTSService, WebsocketService):
if message.startswith(b"RIFF"):
continue
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(message, self._settings["sample_rate"], 1)
frame = TTSAudioRawFrame(message, self.sample_rate, 1)
await self.push_frame(frame)
else:
logger.debug(f"Received text message: {message}")
@@ -304,7 +300,7 @@ class PlayHTTTSService(TTSService, WebsocketService):
"voice": self._voice_id,
"voice_engine": self._settings["voice_engine"],
"output_format": self._settings["output_format"],
"sample_rate": self._settings["sample_rate"],
"sample_rate": self.sample_rate,
"language": self._settings["language"],
"speed": self._settings["speed"],
"seed": self._settings["seed"],
@@ -342,7 +338,7 @@ class PlayHTHttpTTSService(TTSService):
user_id: str,
voice_url: str,
voice_engine: str = "Play3.0-mini-http", # Options: Play3.0-mini-http, Play3.0-mini-ws
sample_rate: int = 24000,
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
@@ -356,7 +352,6 @@ class PlayHTHttpTTSService(TTSService):
api_key=self._api_key,
)
self._settings = {
"sample_rate": sample_rate,
"language": self.language_to_service_language(params.language)
if params.language
else "english",
@@ -368,6 +363,11 @@ class PlayHTHttpTTSService(TTSService):
self.set_model_name(voice_engine)
self.set_voice(voice_url)
async def start(self, frame: StartFrame):
await super().start(frame)
self._settings["sample_rate"] = self.sample_rate
def _create_options(self) -> TTSOptions:
language_str = self._settings["language"]
playht_language = None
if language_str:
@@ -377,10 +377,10 @@ class PlayHTHttpTTSService(TTSService):
playht_language = lang
break
self._options = TTSOptions(
return TTSOptions(
voice=self._voice_id,
language=playht_language,
sample_rate=self._settings["sample_rate"],
sample_rate=self.sample_rate,
format=self._settings["format"],
speed=self._settings["speed"],
seed=self._settings["seed"],
@@ -396,13 +396,14 @@ class PlayHTHttpTTSService(TTSService):
logger.debug(f"Generating TTS: [{text}]")
try:
options = self._create_options()
b = bytearray()
in_header = True
await self.start_ttfb_metrics()
playht_gen = self._client.tts(
text, voice_engine=self._settings["voice_engine"], options=self._options
text, voice_engine=self._settings["voice_engine"], options=options
)
await self.start_tts_usage_metrics(text)
@@ -425,7 +426,7 @@ class PlayHTHttpTTSService(TTSService):
else:
if len(chunk):
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
yield frame
yield TTSStoppedFrame()
except Exception as e:

6
src/pipecat/services/rime.py
View File

@@ -34,7 +34,7 @@ class RimeHttpTTSService(TTSService):
api_key: str,
voice_id: str = "eva",
model: str = "mist",
sample_rate: int = 24000,
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
@@ -43,7 +43,6 @@ class RimeHttpTTSService(TTSService):
self._api_key = api_key
self._base_url = "https://users.rime.ai/v1/rime-tts"
self._settings = {
"samplingRate": sample_rate,
"speedAlpha": params.speed_alpha,
"reduceLatency": params.reduce_latency,
"pauseBetweenBrackets": params.pause_between_brackets,
@@ -71,6 +70,7 @@ class RimeHttpTTSService(TTSService):
payload["text"] = text
payload["speaker"] = self._voice_id
payload["modelId"] = self._model_name
payload["samplingRate"] = self.sample_rate
try:
await self.start_ttfb_metrics()
@@ -96,7 +96,7 @@ class RimeHttpTTSService(TTSService):
first_chunk = False
if chunk:
frame = TTSAudioRawFrame(chunk, self._settings["samplingRate"], 1)
frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
yield frame
yield TTSStoppedFrame()

50
src/pipecat/services/riva.py
View File

@@ -49,7 +49,7 @@ class FastPitchTTSService(TTSService):
api_key: str,
server: str = "grpc.nvcf.nvidia.com:443",
voice_id: str = "English-US.Female-1",
sample_rate: int = 24000,
sample_rate: Optional[int] = None,
function_id: str = "0149dedb-2be8-4195-b9a0-e57e0e14f972",
params: InputParams = InputParams(),
**kwargs,
@@ -57,7 +57,6 @@ class FastPitchTTSService(TTSService):
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._voice_id = voice_id
self._sample_rate = sample_rate
self._language_code = params.language
self._quality = params.quality
@@ -87,7 +86,7 @@ class FastPitchTTSService(TTSService):
text,
self._voice_id,
self._language_code,
sample_rate_hz=self._sample_rate,
sample_rate_hz=self.sample_rate,
audio_prompt_file=None,
quality=self._quality,
custom_dictionary={},
@@ -114,7 +113,7 @@ class FastPitchTTSService(TTSService):
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(
audio=resp.audio,
sample_rate=self._sample_rate,
sample_rate=self.sample_rate,
num_channels=1,
)
yield frame
@@ -136,10 +135,11 @@ class ParakeetSTTService(STTService):
api_key: str,
server: str = "grpc.nvcf.nvidia.com:443",
function_id: str = "1598d209-5e27-4d3c-8079-4751568b1081",
sample_rate: Optional[int] = None,
params: InputParams = InputParams(),
**kwargs,
):
super().__init__(**kwargs)
super().__init__(sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._profanity_filter = False
self._automatic_punctuation = False
@@ -154,7 +154,6 @@ class ParakeetSTTService(STTService):
self._stop_history_eou = -1
self._stop_threshold_eou = -1.0
self._custom_configuration = ""
self._sample_rate: int = 16000
self.set_model_name("parakeet-ctc-1.1b-asr")
@@ -166,6 +165,14 @@ class ParakeetSTTService(STTService):
self._asr_service = riva.client.ASRService(auth)
self._queue = asyncio.Queue()
def can_generate_metrics(self) -> bool:
return False
async def start(self, frame: StartFrame):
await super().start(frame)
config = riva.client.StreamingRecognitionConfig(
config=riva.client.RecognitionConfig(
encoding=riva.client.AudioEncoding.LINEAR_PCM,
@@ -175,14 +182,16 @@ class ParakeetSTTService(STTService):
profanity_filter=self._profanity_filter,
enable_automatic_punctuation=self._automatic_punctuation,
verbatim_transcripts=not self._no_verbatim_transcripts,
sample_rate_hertz=self._sample_rate,
sample_rate_hertz=self.sample_rate,
audio_channel_count=1,
),
interim_results=True,
)
riva.client.add_word_boosting_to_config(
config, self._boosted_lm_words, self._boosted_lm_score
)
riva.client.add_endpoint_parameters_to_config(
config,
self._start_history,
@@ -193,17 +202,11 @@ class ParakeetSTTService(STTService):
self._stop_threshold_eou,
)
riva.client.add_custom_configuration_to_config(config, self._custom_configuration)
self._config = config
self._queue = asyncio.Queue()
def can_generate_metrics(self) -> bool:
return False
async def start(self, frame: StartFrame):
await super().start(frame)
self._thread_task = self.get_event_loop().create_task(self._thread_task_handler())
self._response_task = self.get_event_loop().create_task(self._response_task_handler())
self._thread_task = self.create_task(self._thread_task_handler())
self._response_task = self.create_task(self._response_task_handler())
self._response_queue = asyncio.Queue()
async def stop(self, frame: EndFrame):
@@ -215,10 +218,8 @@ class ParakeetSTTService(STTService):
await self._stop_tasks()
async def _stop_tasks(self):
self._thread_task.cancel()
await self._thread_task
self._response_task.cancel()
await self._response_task
await self.cancel_task(self._thread_task)
await self.cancel_task(self._response_task)
def _response_handler(self):
responses = self._asr_service.streaming_response_generator(
@@ -238,7 +239,7 @@ class ParakeetSTTService(STTService):
await asyncio.to_thread(self._response_handler)
except asyncio.CancelledError:
self._thread_running = False
pass
raise
async def _handle_response(self, response):
for result in response.results:
@@ -260,11 +261,8 @@ class ParakeetSTTService(STTService):
async def _response_task_handler(self):
while True:
try:
response = await self._response_queue.get()
await self._handle_response(response)
except asyncio.CancelledError:
break
response = await self._response_queue.get()
await self._handle_response(response)
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
await self._queue.put(audio)

64
src/pipecat/services/simli.py
View File

@@ -49,45 +49,33 @@ class SimliVideoService(FrameProcessor):
async def _start_connection(self):
await self._simli_client.Initialize()
# Create task to consume and process audio and video
self._audio_task = asyncio.create_task(self._consume_and_process_audio())
self._video_task = asyncio.create_task(self._consume_and_process_video())
self._audio_task = self.create_task(self._consume_and_process_audio())
self._video_task = self.create_task(self._consume_and_process_video())
async def _consume_and_process_audio(self):
try:
await self._pipecat_resampler_event.wait()
async for audio_frame in self._simli_client.getAudioStreamIterator():
resampled_frames = self._pipecat_resampler.resample(audio_frame)
for resampled_frame in resampled_frames:
await self.push_frame(
TTSAudioRawFrame(
audio=resampled_frame.to_ndarray().tobytes(),
sample_rate=self._pipecat_resampler.rate,
num_channels=1,
),
)
except Exception as e:
logger.exception(f"{self} exception: {e}")
except asyncio.CancelledError:
pass
await self._pipecat_resampler_event.wait()
async for audio_frame in self._simli_client.getAudioStreamIterator():
resampled_frames = self._pipecat_resampler.resample(audio_frame)
for resampled_frame in resampled_frames:
await self.push_frame(
TTSAudioRawFrame(
audio=resampled_frame.to_ndarray().tobytes(),
sample_rate=self._pipecat_resampler.rate,
num_channels=1,
),
)
async def _consume_and_process_video(self):
try:
await self._pipecat_resampler_event.wait()
async for video_frame in self._simli_client.getVideoStreamIterator(
targetFormat="rgb24"
):
# Process the video frame
convertedFrame: OutputImageRawFrame = OutputImageRawFrame(
image=video_frame.to_rgb().to_image().tobytes(),
size=(video_frame.width, video_frame.height),
format="RGB",
)
convertedFrame.pts = video_frame.pts
await self.push_frame(convertedFrame)
except Exception as e:
logger.exception(f"{self} exception: {e}")
except asyncio.CancelledError:
pass
await self._pipecat_resampler_event.wait()
async for video_frame in self._simli_client.getVideoStreamIterator(targetFormat="rgb24"):
# Process the video frame
convertedFrame: OutputImageRawFrame = OutputImageRawFrame(
image=video_frame.to_rgb().to_image().tobytes(),
size=(video_frame.width, video_frame.height),
format="RGB",
)
convertedFrame.pts = video_frame.pts
await self.push_frame(convertedFrame)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -128,8 +116,6 @@ class SimliVideoService(FrameProcessor):
async def _stop(self):
await self._simli_client.stop()
if self._audio_task:
self._audio_task.cancel()
await self._audio_task
await self.cancel_task(self._audio_task)
if self._video_task:
self._video_task.cancel()
await self._video_task
await self.cancel_task(self._video_task)

20
src/pipecat/services/tavus.py
View File

@@ -12,7 +12,7 @@ import base64
import aiohttp
from loguru import logger
from pipecat.audio.utils import resample_audio
from pipecat.audio.utils import create_default_resampler
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
@@ -47,6 +47,8 @@ class TavusVideoService(AIService):
self._conversation_id: str
self._resampler = create_default_resampler()
async def initialize(self) -> str:
url = "https://tavusapi.com/v2/conversations"
headers = {"Content-Type": "application/json", "x-api-key": self._api_key}
@@ -75,18 +77,24 @@ class TavusVideoService(AIService):
logger.debug(f"TavusVideoService persona grabbed {response_json}")
return response_json["persona_name"]
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._end_conversation()
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._end_conversation()
async def _end_conversation(self) -> None:
url = f"https://tavusapi.com/v2/conversations/{self._conversation_id}/end"
headers = {"Content-Type": "application/json", "x-api-key": self._api_key}
async with self._session.post(url, headers=headers) as r:
r.raise_for_status()
async def _encode_audio_and_send(
self, audio: bytes, original_sample_rate: int, done: bool
) -> None:
async def _encode_audio_and_send(self, audio: bytes, in_rate: int, done: bool) -> None:
"""Encodes audio to base64 and sends it to Tavus"""
if not done:
audio = resample_audio(audio, original_sample_rate, 16000)
audio = await self._resampler.resample(audio, in_rate, 16000)
audio_base64 = base64.b64encode(audio).decode("utf-8")
logger.trace(f"{self}: sending {len(audio)} bytes")
await self._send_audio_message(audio_base64, done=done)
@@ -105,8 +113,6 @@ class TavusVideoService(AIService):
await self.stop_processing_metrics()
elif isinstance(frame, StartInterruptionFrame):
await self._send_interrupt_message()
elif isinstance(frame, (EndFrame, CancelFrame)):
await self._end_conversation()
else:
await self.push_frame(frame, direction)

4
src/pipecat/services/websocket_service.py
View File

@@ -85,10 +85,6 @@ class WebsocketService(ABC):
await self._receive_messages()
logger.debug(f"{self} connection established successfully")
retry_count = 0 # Reset counter on successful message receive
except asyncio.CancelledError:
break
except Exception as e:
retry_count += 1
if retry_count >= MAX_RETRIES:

20
src/pipecat/services/xtts.py
View File

@@ -4,12 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import Any, AsyncGenerator, Dict
from typing import Any, AsyncGenerator, Dict, Optional
import aiohttp
from loguru import logger
from pipecat.audio.utils import resample_audio
from pipecat.audio.utils import create_default_resampler
from pipecat.frames.frames import (
ErrorFrame,
Frame,
@@ -76,7 +76,7 @@ class XTTSService(TTSService):
base_url: str,
aiohttp_session: aiohttp.ClientSession,
language: Language = Language.EN,
sample_rate: int = 24000,
sample_rate: Optional[int] = None,
**kwargs,
):
super().__init__(sample_rate=sample_rate, **kwargs)
@@ -89,6 +89,8 @@ class XTTSService(TTSService):
self._studio_speakers: Dict[str, Any] | None = None
self._aiohttp_session = aiohttp_session
self._resampler = create_default_resampler()
def can_generate_metrics(self) -> bool:
return True
@@ -161,17 +163,19 @@ class XTTSService(TTSService):
buffer = buffer[48000:]
# XTTS uses 24000 so we need to resample to our desired rate.
resampled_audio = resample_audio(
bytes(process_data), 24000, self._sample_rate
resampled_audio = await self._resampler.resample(
bytes(process_data), 24000, self.sample_rate
)
# Create the frame with the resampled audio
frame = TTSAudioRawFrame(resampled_audio, self._sample_rate, 1)
frame = TTSAudioRawFrame(resampled_audio, self.sample_rate, 1)
yield frame
# Process any remaining data in the buffer.
if len(buffer) > 0:
resampled_audio = resample_audio(bytes(buffer), 24000, self._sample_rate)
frame = TTSAudioRawFrame(resampled_audio, self._sample_rate, 1)
resampled_audio = await self._resampler.resample(
bytes(buffer), 24000, self.sample_rate
)
frame = TTSAudioRawFrame(resampled_audio, self.sample_rate, 1)
yield frame
yield TTSStoppedFrame()

0
src/pipecat/tests/__init__.py Normal file
View File

128
src/pipecat/tests/utils.py Normal file
View File

@@ -0,0 +1,128 @@
#
# Copyright (c) 2024-2025 Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
import sys
from typing import Any, Awaitable, Callable, Dict, Sequence, Tuple
from loguru import logger
from pipecat.frames.frames import (
EndFrame,
Frame,
HeartbeatFrame,
StartFrame,
)
from pipecat.observers.base_observer import BaseObserver
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
logger.remove(0)
logger.add(sys.stderr, level="TRACE")
class HeartbeatsObserver(BaseObserver):
def __init__(
self,
*,
target: FrameProcessor,
heartbeat_callback: Callable[[FrameProcessor, HeartbeatFrame], Awaitable[None]],
):
self._target = target
self._callback = heartbeat_callback
async def on_push_frame(
self,
src: FrameProcessor,
dst: FrameProcessor,
frame: Frame,
direction: FrameDirection,
timestamp: int,
):
if src == self._target and isinstance(frame, HeartbeatFrame):
await self._callback(self._target, frame)
class QueuedFrameProcessor(FrameProcessor):
def __init__(
self, queue: asyncio.Queue, queue_direction: FrameDirection, ignore_start: bool = True
):
super().__init__()
self._queue = queue
self._queue_direction = queue_direction
self._ignore_start = ignore_start
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if direction == self._queue_direction:
if not isinstance(frame, StartFrame) or not self._ignore_start:
await self._queue.put(frame)
await self.push_frame(frame, direction)
async def run_test(
processor: FrameProcessor,
*,
frames_to_send: Sequence[Frame],
expected_down_frames: Sequence[type],
expected_up_frames: Sequence[type] = [],
ignore_start: bool = True,
start_metadata: Dict[str, Any] = {},
send_end_frame: bool = True,
) -> Tuple[Sequence[Frame], Sequence[Frame]]:
received_up = asyncio.Queue()
received_down = asyncio.Queue()
source = QueuedFrameProcessor(received_up, FrameDirection.UPSTREAM, ignore_start)
sink = QueuedFrameProcessor(received_down, FrameDirection.DOWNSTREAM, ignore_start)
pipeline = Pipeline([source, processor, sink])
task = PipelineTask(pipeline, params=PipelineParams(start_metadata=start_metadata))
for frame in frames_to_send:
await task.queue_frame(frame)
if send_end_frame:
await task.queue_frame(EndFrame())
runner = PipelineRunner()
await runner.run(task)
#
# Down frames
#
received_down_frames: Sequence[Frame] = []
while not received_down.empty():
frame = await received_down.get()
if not isinstance(frame, EndFrame) or not send_end_frame:
received_down_frames.append(frame)
print("received DOWN frames =", received_down_frames)
assert len(received_down_frames) == len(expected_down_frames)
for real, expected in zip(received_down_frames, expected_down_frames):
assert isinstance(real, expected)
#
# Up frames
#
received_up_frames: Sequence[Frame] = []
while not received_up.empty():
frame = await received_up.get()
received_up_frames.append(frame)
print("received UP frames =", received_up_frames)
assert len(received_up_frames) == len(expected_up_frames)
for real, expected in zip(received_up_frames, expected_up_frames):
assert isinstance(real, expected)
return (received_down_frames, received_up_frames)

72
src/pipecat/transports/base_input.py
View File

@@ -35,6 +35,9 @@ class BaseInputTransport(FrameProcessor):
self._params = params
# Input sample rate. It will be initialized on StartFrame.
self._sample_rate = 0
# We read audio from a single queue one at a time and we then run VAD in
# a thread. Therefore, only one thread should be necessary.
self._executor = ThreadPoolExecutor(max_workers=1)
@@ -43,20 +46,32 @@ class BaseInputTransport(FrameProcessor):
# if passthrough is enabled.
self._audio_task = None
@property
def sample_rate(self) -> int:
return self._sample_rate
@property
def vad_analyzer(self) -> VADAnalyzer | None:
return self._params.vad_analyzer
async def start(self, frame: StartFrame):
self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
# Configure VAD analyzer.
if self._params.vad_enabled and self._params.vad_analyzer:
self._params.vad_analyzer.set_sample_rate(self._sample_rate)
# Start audio filter.
if self._params.audio_in_filter:
await self._params.audio_in_filter.start(self._params.audio_in_sample_rate)
await self._params.audio_in_filter.start(self._sample_rate)
# Create audio input queue and task if needed.
if self._params.audio_in_enabled or self._params.vad_enabled:
self._audio_in_queue = asyncio.Queue()
self._audio_task = self.get_event_loop().create_task(self._audio_task_handler())
self._audio_task = self.create_task(self._audio_task_handler())
async def stop(self, frame: EndFrame):
# Cancel and wait for the audio input task to finish.
if self._audio_task and (self._params.audio_in_enabled or self._params.vad_enabled):
self._audio_task.cancel()
await self._audio_task
await self.cancel_task(self._audio_task)
self._audio_task = None
# Stop audio filter.
if self._params.audio_in_filter:
@@ -65,13 +80,9 @@ class BaseInputTransport(FrameProcessor):
async def cancel(self, frame: CancelFrame):
# Cancel and wait for the audio input task to finish.
if self._audio_task and (self._params.audio_in_enabled or self._params.vad_enabled):
self._audio_task.cancel()
await self._audio_task
await self.cancel_task(self._audio_task)
self._audio_task = None
def vad_analyzer(self) -> VADAnalyzer | None:
return self._params.vad_analyzer
async def push_audio_frame(self, frame: InputAudioRawFrame):
if self._params.audio_in_enabled or self._params.vad_enabled:
await self._audio_in_queue.put(frame)
@@ -106,9 +117,8 @@ class BaseInputTransport(FrameProcessor):
await self.push_frame(frame, direction)
await self.stop(frame)
elif isinstance(frame, VADParamsUpdateFrame):
vad_analyzer = self.vad_analyzer()
if vad_analyzer:
vad_analyzer.set_params(frame.params)
if self.vad_analyzer:
self.vad_analyzer.set_params(frame.params)
elif isinstance(frame, FilterUpdateSettingsFrame) and self._params.audio_in_filter:
await self._params.audio_in_filter.process_frame(frame)
# Other frames
@@ -142,11 +152,10 @@ class BaseInputTransport(FrameProcessor):
async def _vad_analyze(self, audio_frame: InputAudioRawFrame) -> VADState:
state = VADState.QUIET
vad_analyzer = self.vad_analyzer()
if vad_analyzer:
if self.vad_analyzer:
logger.trace(f"{self}: analyzing VAD on {audio_frame}")
state = await self.get_event_loop().run_in_executor(
self._executor, vad_analyzer.analyze_audio, audio_frame.audio
self._executor, self.vad_analyzer.analyze_audio, audio_frame.audio
)
logger.trace(f"{self}: done analyzing VAD on {audio_frame}")
return state
@@ -173,27 +182,22 @@ class BaseInputTransport(FrameProcessor):
async def _audio_task_handler(self):
vad_state: VADState = VADState.QUIET
while True:
try:
frame: InputAudioRawFrame = await self._audio_in_queue.get()
frame: InputAudioRawFrame = await self._audio_in_queue.get()
audio_passthrough = True
audio_passthrough = True
# If an audio filter is available, run it before VAD.
if self._params.audio_in_filter:
frame.audio = await self._params.audio_in_filter.filter(frame.audio)
# If an audio filter is available, run it before VAD.
if self._params.audio_in_filter:
frame.audio = await self._params.audio_in_filter.filter(frame.audio)
# Check VAD and push event if necessary. We just care about
# changes from QUIET to SPEAKING and vice versa.
if self._params.vad_enabled:
vad_state = await self._handle_vad(frame, vad_state)
audio_passthrough = self._params.vad_audio_passthrough
# Check VAD and push event if necessary. We just care about
# changes from QUIET to SPEAKING and vice versa.
if self._params.vad_enabled:
vad_state = await self._handle_vad(frame, vad_state)
audio_passthrough = self._params.vad_audio_passthrough
# Push audio downstream if passthrough.
if audio_passthrough:
await self.push_frame(frame)
# Push audio downstream if passthrough.
if audio_passthrough:
await self.push_frame(frame)
self._audio_in_queue.task_done()
except asyncio.CancelledError:
break
except Exception as e:
logger.exception(f"{self} error reading audio frames: {e}")
self._audio_in_queue.task_done()

109
src/pipecat/transports/base_output.py
View File

@@ -57,12 +57,11 @@ class BaseOutputTransport(FrameProcessor):
# framerate.
self._camera_images = None
# We will write 20ms audio at a time. If we receive long audio frames we
# will chunk them. This will help with interruption handling.
audio_bytes_10ms = (
int(self._params.audio_out_sample_rate / 100) * self._params.audio_out_channels * 2
)
self._audio_chunk_size = audio_bytes_10ms * 2
# Output sample rate. It will be initialized on StartFrame.
self._sample_rate = 0
# Chunk size that will be written. It will be computed on StartFrame
self._audio_chunk_size = 0
self._audio_buffer = bytearray()
self._stopped_event = asyncio.Event()
@@ -70,10 +69,21 @@ class BaseOutputTransport(FrameProcessor):
# Indicates if the bot is currently speaking.
self._bot_speaking = False
@property
def sample_rate(self) -> int:
return self._sample_rate
async def start(self, frame: StartFrame):
self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
# We will write 20ms audio at a time. If we receive long audio frames we
# will chunk them. This will help with interruption handling.
audio_bytes_10ms = int(self._sample_rate / 100) * self._params.audio_out_channels * 2
self._audio_chunk_size = audio_bytes_10ms * 2
# Start audio mixer.
if self._params.audio_out_mixer:
await self._params.audio_out_mixer.start(self._params.audio_out_sample_rate)
await self._params.audio_out_mixer.start(self._sample_rate)
self._create_camera_task()
self._create_sink_tasks()
@@ -87,9 +97,9 @@ class BaseOutputTransport(FrameProcessor):
# for these tasks before cancelling the camera and audio tasks below
# because they might be still rendering.
if self._sink_task:
await self._sink_task
await self.wait_for_task(self._sink_task)
if self._sink_clock_task:
await self._sink_clock_task
await self.wait_for_task(self._sink_clock_task)
# We can now cancel the camera task.
await self._cancel_camera_task()
@@ -217,22 +227,19 @@ class BaseOutputTransport(FrameProcessor):
#
def _create_sink_tasks(self):
loop = self.get_event_loop()
self._sink_queue = asyncio.Queue()
self._sink_task = loop.create_task(self._sink_task_handler())
self._sink_clock_queue = asyncio.PriorityQueue()
self._sink_clock_task = loop.create_task(self._sink_clock_task_handler())
self._sink_task = self.create_task(self._sink_task_handler())
self._sink_clock_task = self.create_task(self._sink_clock_task_handler())
async def _cancel_sink_tasks(self):
# Stop sink tasks.
if self._sink_task:
self._sink_task.cancel()
await self._sink_task
await self.cancel_task(self._sink_task)
self._sink_task = None
# Stop sink clock tasks.
if self._sink_clock_task:
self._sink_clock_task.cancel()
await self._sink_clock_task
await self.cancel_task(self._sink_clock_task)
self._sink_clock_task = None
async def _sink_frame_handler(self, frame: Frame):
@@ -269,7 +276,7 @@ class BaseOutputTransport(FrameProcessor):
self._sink_clock_queue.task_done()
except asyncio.CancelledError:
break
raise
except Exception as e:
logger.exception(f"{self} error processing sink clock queue: {e}")
@@ -301,7 +308,7 @@ class BaseOutputTransport(FrameProcessor):
# Generate an audio frame with only the mixer's part.
frame = OutputAudioRawFrame(
audio=await self._params.audio_out_mixer.mix(silence),
sample_rate=self._params.audio_out_sample_rate,
sample_rate=self._sample_rate,
num_channels=self._params.audio_out_channels,
)
yield frame
@@ -317,49 +324,42 @@ class BaseOutputTransport(FrameProcessor):
return without_mixer(vad_stop_secs)
async def _sink_task_handler(self):
try:
async for frame in self._next_frame():
# Notify the bot started speaking upstream if necessary and that
# it's actually speaking.
if isinstance(frame, TTSAudioRawFrame):
await self._bot_started_speaking()
await self.push_frame(BotSpeakingFrame())
await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM)
async for frame in self._next_frame():
# Notify the bot started speaking upstream if necessary and that
# it's actually speaking.
if isinstance(frame, TTSAudioRawFrame):
await self._bot_started_speaking()
await self.push_frame(BotSpeakingFrame())
await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM)
# No need to push EndFrame, it's pushed from process_frame().
if isinstance(frame, EndFrame):
break
# No need to push EndFrame, it's pushed from process_frame().
if isinstance(frame, EndFrame):
break
# Handle frame.
await self._sink_frame_handler(frame)
# Handle frame.
await self._sink_frame_handler(frame)
# Also, push frame downstream in case anyone else needs it.
await self.push_frame(frame)
# Also, push frame downstream in case anyone else needs it.
await self.push_frame(frame)
# Send audio.
if isinstance(frame, OutputAudioRawFrame):
await self.write_raw_audio_frames(frame.audio)
except asyncio.CancelledError:
pass
except Exception as e:
logger.exception(f"{self} error writing to microphone: {e}")
# Send audio.
if isinstance(frame, OutputAudioRawFrame):
await self.write_raw_audio_frames(frame.audio)
#
# Camera task
#
def _create_camera_task(self):
loop = self.get_event_loop()
# Create camera output queue and task if needed.
if self._params.camera_out_enabled:
self._camera_out_queue = asyncio.Queue()
self._camera_out_task = loop.create_task(self._camera_out_task_handler())
self._camera_out_task = self.create_task(self._camera_out_task_handler())
async def _cancel_camera_task(self):
# Stop camera output task.
if self._camera_out_task and self._params.camera_out_enabled:
self._camera_out_task.cancel()
await self._camera_out_task
await self.cancel_task(self._camera_out_task)
self._camera_out_task = None
async def _draw_image(self, frame: OutputImageRawFrame):
@@ -387,19 +387,14 @@ class BaseOutputTransport(FrameProcessor):
self._camera_out_frame_duration = 1 / self._params.camera_out_framerate
self._camera_out_frame_reset = self._camera_out_frame_duration * 5
while True:
try:
if self._params.camera_out_is_live:
await self._camera_out_is_live_handler()
elif self._camera_images:
image = next(self._camera_images)
await self._draw_image(image)
await asyncio.sleep(self._camera_out_frame_duration)
else:
await asyncio.sleep(self._camera_out_frame_duration)
except asyncio.CancelledError:
break
except Exception as e:
logger.exception(f"{self} error writing to camera: {e}")
if self._params.camera_out_is_live:
await self._camera_out_is_live_handler()
elif self._camera_images:
image = next(self._camera_images)
await self._draw_image(image)
await asyncio.sleep(self._camera_out_frame_duration)
else:
await asyncio.sleep(self._camera_out_frame_duration)
async def _camera_out_is_live_handler(self):
image = await self._camera_out_queue.get()

26
src/pipecat/transports/base_transport.py
View File

@@ -16,6 +16,7 @@ from pipecat.audio.filters.base_audio_filter import BaseAudioFilter
from pipecat.audio.mixers.base_audio_mixer import BaseAudioMixer
from pipecat.audio.vad.vad_analyzer import VADAnalyzer
from pipecat.processors.frame_processor import FrameProcessor
from pipecat.utils.utils import obj_count, obj_id
class TransportParams(BaseModel):
@@ -30,12 +31,12 @@ class TransportParams(BaseModel):
camera_out_color_format: str = "RGB"
audio_out_enabled: bool = False
audio_out_is_live: bool = False
audio_out_sample_rate: int = 24000
audio_out_sample_rate: Optional[int] = None
audio_out_channels: int = 1
audio_out_bitrate: int = 96000
audio_out_mixer: Optional[BaseAudioMixer] = None
audio_in_enabled: bool = False
audio_in_sample_rate: int = 16000
audio_in_sample_rate: Optional[int] = None
audio_in_channels: int = 1
audio_in_filter: Optional[BaseAudioFilter] = None
vad_enabled: bool = False
@@ -46,15 +47,25 @@ class TransportParams(BaseModel):
class BaseTransport(ABC):
def __init__(
self,
input_name: str | None = None,
output_name: str | None = None,
loop: asyncio.AbstractEventLoop | None = None,
*,
name: Optional[str] = None,
input_name: Optional[str] = None,
output_name: Optional[str] = None,
):
self._id: int = obj_id()
self._name = name or f"{self.__class__.__name__}#{obj_count(self)}"
self._input_name = input_name
self._output_name = output_name
self._loop = loop or asyncio.get_running_loop()
self._event_handlers: dict = {}
@property
def id(self) -> int:
return self._id
@property
def name(self) -> str:
return self._name
@abstractmethod
def input(self) -> FrameProcessor:
raise NotImplementedError
@@ -89,3 +100,6 @@ class BaseTransport(ABC):
handler(self, *args, **kwargs)
except Exception as e:
logger.exception(f"Exception in event handler {event_name}: {e}")
def __str__(self):
return self.name

71
src/pipecat/transports/local/audio.py
View File

@@ -28,35 +28,40 @@ except ModuleNotFoundError as e:
class LocalAudioInputTransport(BaseInputTransport):
def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams):
super().__init__(params)
self._py_audio = py_audio
self._in_stream = None
self._sample_rate = 0
sample_rate = self._params.audio_in_sample_rate
num_frames = int(sample_rate / 100) * 2 # 20ms of audio
async def start(self, frame: StartFrame):
await super().start(frame)
self._in_stream = py_audio.open(
format=py_audio.get_format_from_width(2),
channels=params.audio_in_channels,
rate=params.audio_in_sample_rate,
self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
num_frames = int(self._sample_rate / 100) * 2 # 20ms of audio
self._in_stream = self._py_audio.open(
format=self._py_audio.get_format_from_width(2),
channels=self._params.audio_in_channels,
rate=self._sample_rate,
frames_per_buffer=num_frames,
stream_callback=self._audio_in_callback,
input=True,
)
async def start(self, frame: StartFrame):
await super().start(frame)
self._in_stream.start_stream()
async def cleanup(self):
await super().cleanup()
self._in_stream.stop_stream()
# This is not very pretty (taken from PyAudio docs).
while self._in_stream.is_active():
await asyncio.sleep(0.1)
self._in_stream.close()
if self._in_stream:
self._in_stream.stop_stream()
# This is not very pretty (taken from PyAudio docs).
while self._in_stream.is_active():
await asyncio.sleep(0.1)
self._in_stream.close()
self._in_stream = None
def _audio_in_callback(self, in_data, frame_count, time_info, status):
frame = InputAudioRawFrame(
audio=in_data,
sample_rate=self._params.audio_in_sample_rate,
sample_rate=self._sample_rate,
num_channels=self._params.audio_in_channels,
)
@@ -68,36 +73,46 @@ class LocalAudioInputTransport(BaseInputTransport):
class LocalAudioOutputTransport(BaseOutputTransport):
def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams):
super().__init__(params)
self._py_audio = py_audio
self._out_stream = None
self._sample_rate = 0
# We only write audio frames from a single task, so only one thread
# should be necessary.
self._executor = ThreadPoolExecutor(max_workers=1)
self._out_stream = py_audio.open(
format=py_audio.get_format_from_width(2),
channels=params.audio_out_channels,
rate=params.audio_out_sample_rate,
output=True,
)
async def start(self, frame: StartFrame):
await super().start(frame)
self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
self._out_stream = self._py_audio.open(
format=self._py_audio.get_format_from_width(2),
channels=self._params.audio_out_channels,
rate=self._sample_rate,
output=True,
)
self._out_stream.start_stream()
async def cleanup(self):
await super().cleanup()
self._out_stream.stop_stream()
# This is not very pretty (taken from PyAudio docs).
while self._out_stream.is_active():
await asyncio.sleep(0.1)
self._out_stream.close()
if self._out_stream:
self._out_stream.stop_stream()
# This is not very pretty (taken from PyAudio docs).
while self._out_stream.is_active():
await asyncio.sleep(0.1)
self._out_stream.close()
async def write_raw_audio_frames(self, frames: bytes):
await self.get_event_loop().run_in_executor(self._executor, self._out_stream.write, frames)
if self._out_stream:
await self.get_event_loop().run_in_executor(
self._executor, self._out_stream.write, frames
)
class LocalAudioTransport(BaseTransport):
def __init__(self, params: TransportParams):
super().__init__()
self._params = params
self._pyaudio = pyaudio.PyAudio()

70
src/pipecat/transports/local/tk.py
View File

@@ -36,35 +36,39 @@ except ModuleNotFoundError as e:
class TkInputTransport(BaseInputTransport):
def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams):
super().__init__(params)
self._py_audio = py_audio
self._in_stream = None
self._sample_rate = 0
sample_rate = self._params.audio_in_sample_rate
num_frames = int(sample_rate / 100) * 2 # 20ms of audio
async def start(self, frame: StartFrame):
await super().start(frame)
self._in_stream = py_audio.open(
format=py_audio.get_format_from_width(2),
channels=params.audio_in_channels,
rate=params.audio_in_sample_rate,
self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
num_frames = int(self._sample_rate / 100) * 2 # 20ms of audio
self._in_stream = self._py_audio.open(
format=self._py_audio.get_format_from_width(2),
channels=self._params.audio_in_channels,
rate=self._sample_rate,
frames_per_buffer=num_frames,
stream_callback=self._audio_in_callback,
input=True,
)
async def start(self, frame: StartFrame):
await super().start(frame)
self._in_stream.start_stream()
async def cleanup(self):
await super().cleanup()
self._in_stream.stop_stream()
# This is not very pretty (taken from PyAudio docs).
while self._in_stream.is_active():
await asyncio.sleep(0.1)
self._in_stream.close()
if self._in_stream:
self._in_stream.stop_stream()
# This is not very pretty (taken from PyAudio docs).
while self._in_stream.is_active():
await asyncio.sleep(0.1)
self._in_stream.close()
def _audio_in_callback(self, in_data, frame_count, time_info, status):
frame = InputAudioRawFrame(
audio=in_data,
sample_rate=self._params.audio_in_sample_rate,
sample_rate=self._sample_rate,
num_channels=self._params.audio_in_channels,
)
@@ -76,18 +80,14 @@ class TkInputTransport(BaseInputTransport):
class TkOutputTransport(BaseOutputTransport):
def __init__(self, tk_root: tk.Tk, py_audio: pyaudio.PyAudio, params: TransportParams):
super().__init__(params)
self._py_audio = py_audio
self._out_stream = None
self._sample_rate = 0
# We only write audio frames from a single task, so only one thread
# should be necessary.
self._executor = ThreadPoolExecutor(max_workers=1)
self._out_stream = py_audio.open(
format=py_audio.get_format_from_width(2),
channels=params.audio_out_channels,
rate=params.audio_out_sample_rate,
output=True,
)
# Start with a neutral gray background.
array = np.ones((1024, 1024, 3)) * 128
data = f"P5 {1024} {1024} 255 ".encode() + array.astype(np.uint8).tobytes()
@@ -97,18 +97,31 @@ class TkOutputTransport(BaseOutputTransport):
async def start(self, frame: StartFrame):
await super().start(frame)
self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
self._out_stream = self._py_audio.open(
format=self._py_audio.get_format_from_width(2),
channels=self._params.audio_out_channels,
rate=self._sample_rate,
output=True,
)
self._out_stream.start_stream()
async def cleanup(self):
await super().cleanup()
self._out_stream.stop_stream()
# This is not very pretty (taken from PyAudio docs).
while self._out_stream.is_active():
await asyncio.sleep(0.1)
self._out_stream.close()
if self._out_stream:
self._out_stream.stop_stream()
# This is not very pretty (taken from PyAudio docs).
while self._out_stream.is_active():
await asyncio.sleep(0.1)
self._out_stream.close()
async def write_raw_audio_frames(self, frames: bytes):
await self.get_event_loop().run_in_executor(self._executor, self._out_stream.write, frames)
if self._out_stream:
await self.get_event_loop().run_in_executor(
self._executor, self._out_stream.write, frames
)
async def write_frame_to_camera(self, frame: OutputImageRawFrame):
self.get_event_loop().call_soon(self._write_frame_to_tk, frame)
@@ -127,6 +140,7 @@ class TkOutputTransport(BaseOutputTransport):
class TkLocalTransport(BaseTransport):
def __init__(self, tk_root: tk.Tk, params: TransportParams):
super().__init__()
self._tk_root = tk_root
self._params = params
self._pyaudio = pyaudio.PyAudio()

69
src/pipecat/transports/network/fastapi_websocket.py
View File

@@ -16,6 +16,8 @@ from loguru import logger
from pydantic import BaseModel
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
Frame,
InputAudioRawFrame,
OutputAudioRawFrame,
@@ -67,12 +69,19 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
async def start(self, frame: StartFrame):
await super().start(frame)
await self._params.serializer.setup(frame)
if self._params.session_timeout:
self._monitor_websocket_task = self.get_event_loop().create_task(
self._monitor_websocket()
)
self._monitor_websocket_task = self.create_task(self._monitor_websocket())
await self._callbacks.on_client_connected(self._websocket)
self._receive_task = self.get_event_loop().create_task(self._receive_messages())
self._receive_task = self.create_task(self._receive_messages())
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self.cancel_task(self._receive_task)
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self.cancel_task(self._receive_task)
def _iter_data(self) -> typing.AsyncIterator[bytes | str]:
if self._params.serializer.type == FrameSerializerType.BINARY:
@@ -81,26 +90,26 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
return self._websocket.iter_text()
async def _receive_messages(self):
async for message in self._iter_data():
frame = self._params.serializer.deserialize(message)
try:
async for message in self._iter_data():
frame = await self._params.serializer.deserialize(message)
if not frame:
continue
if not frame:
continue
if isinstance(frame, InputAudioRawFrame):
await self.push_audio_frame(frame)
else:
await self.push_frame(frame)
if isinstance(frame, InputAudioRawFrame):
await self.push_audio_frame(frame)
else:
await self.push_frame(frame)
except Exception as e:
logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
await self._callbacks.on_client_disconnected(self._websocket)
async def _monitor_websocket(self):
"""Wait for self._params.session_timeout seconds, if the websocket is still open, trigger timeout event."""
try:
await asyncio.sleep(self._params.session_timeout)
await self._callbacks.on_session_timeout(self._websocket)
except asyncio.CancelledError:
logger.info(f"Monitoring task cancelled for: {self._websocket}")
await asyncio.sleep(self._params.session_timeout)
await self._callbacks.on_session_timeout(self._websocket)
class FastAPIWebsocketOutputTransport(BaseOutputTransport):
@@ -110,9 +119,19 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
self._websocket = websocket
self._params = params
self._send_interval = (self._audio_chunk_size / self._params.audio_out_sample_rate) / 2
# write_raw_audio_frames() is called quickly, as soon as we get audio
# (e.g. from the TTS), and since this is just a network connection we
# would be sending it to quickly. Instead, we want to block to emulate
# an audio device, this is what the send interval is. It will be
# computed on StartFrame.
self._send_interval = 0
self._next_send_time = 0
async def start(self, frame: StartFrame):
await super().start(frame)
await self._params.serializer.setup(frame)
self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -128,7 +147,7 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
frame = OutputAudioRawFrame(
audio=frames,
sample_rate=self._params.audio_out_sample_rate,
sample_rate=self.sample_rate,
num_channels=self._params.audio_out_channels,
)
@@ -154,9 +173,12 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
await self._write_audio_sleep()
async def _write_frame(self, frame: Frame):
payload = self._params.serializer.serialize(frame)
if payload and self._websocket.client_state == WebSocketState.CONNECTED:
await self._send_data(payload)
try:
payload = await self._params.serializer.serialize(frame)
if payload and self._websocket.client_state == WebSocketState.CONNECTED:
await self._send_data(payload)
except Exception as e:
logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})")
def _send_data(self, data: str | bytes):
if self._params.serializer.type == FrameSerializerType.BINARY:
@@ -182,9 +204,8 @@ class FastAPIWebsocketTransport(BaseTransport):
params: FastAPIWebsocketParams,
input_name: str | None = None,
output_name: str | None = None,
loop: asyncio.AbstractEventLoop | None = None,
):
super().__init__(input_name=input_name, output_name=output_name, loop=loop)
super().__init__(input_name=input_name, output_name=output_name)
self._params = params
self._callbacks = FastAPIWebsocketCallbacks(

269
src/pipecat/transports/network/websocket_client.py Normal file
View File

@@ -0,0 +1,269 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
import io
import time
import wave
from typing import Awaitable, Callable, Optional
import websockets
from loguru import logger
from pydantic.main import BaseModel
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
Frame,
InputAudioRawFrame,
OutputAudioRawFrame,
StartFrame,
TransportMessageFrame,
TransportMessageUrgentFrame,
)
from pipecat.serializers.base_serializer import FrameSerializer
from pipecat.serializers.protobuf import ProtobufFrameSerializer
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.utils.asyncio import TaskManager
class WebsocketClientParams(TransportParams):
add_wav_header: bool = True
serializer: FrameSerializer = ProtobufFrameSerializer()
class WebsocketClientCallbacks(BaseModel):
on_connected: Callable[[websockets.WebSocketClientProtocol], Awaitable[None]]
on_disconnected: Callable[[websockets.WebSocketClientProtocol], Awaitable[None]]
on_message: Callable[[websockets.WebSocketClientProtocol, websockets.Data], Awaitable[None]]
class WebsocketClientSession:
def __init__(
self,
uri: str,
params: WebsocketClientParams,
callbacks: WebsocketClientCallbacks,
transport_name: str,
):
self._uri = uri
self._params = params
self._callbacks = callbacks
self._transport_name = transport_name
self._task_manager: Optional[TaskManager] = None
self._websocket: websockets.WebSocketClientProtocol | None = None
@property
def task_manager(self) -> TaskManager:
if not self._task_manager:
raise Exception(
f"{self._transport_name}::WebsocketClientSession: TaskManager not initialized (pipeline not started?)"
)
return self._task_manager
async def setup(self, frame: StartFrame):
if not self._task_manager:
self._task_manager = frame.task_manager
async def connect(self):
if self._websocket:
return
try:
self._websocket = await websockets.connect(uri=self._uri, open_timeout=10)
self._client_task = self.task_manager.create_task(
self._client_task_handler(),
f"{self._transport_name}::WebsocketClientSession::_client_task_handler",
)
await self._callbacks.on_connected(self._websocket)
except TimeoutError:
logger.error(f"Timeout connecting to {self._uri}")
async def disconnect(self):
if not self._websocket:
return
await self.task_manager.cancel_task(self._client_task)
await self._websocket.close()
self._websocket = None
async def send(self, message: websockets.Data):
try:
if self._websocket:
await self._websocket.send(message)
except Exception as e:
logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})")
async def _client_task_handler(self):
try:
# Handle incoming messages
async for message in self._websocket:
await self._callbacks.on_message(self._websocket, message)
except Exception as e:
logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
await self._callbacks.on_disconnected(self._websocket)
def __str__(self):
return f"{self._transport_name}::WebsocketClientSession"
class WebsocketClientInputTransport(BaseInputTransport):
def __init__(self, session: WebsocketClientSession, params: WebsocketClientParams):
super().__init__(params)
self._session = session
self._params = params
async def start(self, frame: StartFrame):
await super().start(frame)
await self._params.serializer.setup(frame)
await self._session.setup(frame)
await self._session.connect()
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._session.disconnect()
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._session.disconnect()
async def on_message(self, websocket, message):
frame = await self._params.serializer.deserialize(message)
if not frame:
return
if isinstance(frame, InputAudioRawFrame) and self._params.audio_in_enabled:
await self.push_audio_frame(frame)
else:
await self.push_frame(frame)
class WebsocketClientOutputTransport(BaseOutputTransport):
def __init__(self, session: WebsocketClientSession, params: WebsocketClientParams):
super().__init__(params)
self._session = session
self._params = params
# write_raw_audio_frames() is called quickly, as soon as we get audio
# (e.g. from the TTS), and since this is just a network connection we
# would be sending it to quickly. Instead, we want to block to emulate
# an audio device, this is what the send interval is. It will be
# computed on StartFrame.
self._send_interval = 0
self._next_send_time = 0
async def start(self, frame: StartFrame):
await super().start(frame)
self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2
await self._params.serializer.setup(frame)
await self._session.setup(frame)
await self._session.connect()
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._session.disconnect()
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._session.disconnect()
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
await self._write_frame(frame)
async def write_raw_audio_frames(self, frames: bytes):
frame = OutputAudioRawFrame(
audio=frames,
sample_rate=self.sample_rate,
num_channels=self._params.audio_out_channels,
)
if self._params.add_wav_header:
with io.BytesIO() as buffer:
with wave.open(buffer, "wb") as wf:
wf.setsampwidth(2)
wf.setnchannels(frame.num_channels)
wf.setframerate(frame.sample_rate)
wf.writeframes(frame.audio)
wav_frame = OutputAudioRawFrame(
buffer.getvalue(),
sample_rate=frame.sample_rate,
num_channels=frame.num_channels,
)
frame = wav_frame
await self._write_frame(frame)
# Simulate audio playback with a sleep.
await self._write_audio_sleep()
async def _write_frame(self, frame: Frame):
payload = await self._params.serializer.serialize(frame)
if payload:
await self._session.send(payload)
async def _write_audio_sleep(self):
# Simulate a clock.
current_time = time.monotonic()
sleep_duration = max(0, self._next_send_time - current_time)
await asyncio.sleep(sleep_duration)
if sleep_duration == 0:
self._next_send_time = time.monotonic() + self._send_interval
else:
self._next_send_time += self._send_interval
class WebsocketClientTransport(BaseTransport):
def __init__(
self,
uri: str,
params: WebsocketClientParams = WebsocketClientParams(),
):
super().__init__()
self._params = params
callbacks = WebsocketClientCallbacks(
on_connected=self._on_connected,
on_disconnected=self._on_disconnected,
on_message=self._on_message,
)
self._session = WebsocketClientSession(uri, params, callbacks, self.name)
self._input: WebsocketClientInputTransport | None = None
self._output: WebsocketClientOutputTransport | None = None
# Register supported handlers. The user will only be able to register
# these handlers.
self._register_event_handler("on_connected")
self._register_event_handler("on_disconnected")
def input(self) -> WebsocketClientInputTransport:
if not self._input:
self._input = WebsocketClientInputTransport(self._session, self._params)
return self._input
def output(self) -> WebsocketClientOutputTransport:
if not self._output:
self._output = WebsocketClientOutputTransport(self._session, self._params)
return self._output
async def _on_connected(self, websocket):
await self._call_event_handler("on_connected", websocket)
async def _on_disconnected(self, websocket):
await self._call_event_handler("on_disconnected", websocket)
async def _on_message(self, websocket, message):
if self._input:
await self._input.on_message(websocket, message)

87
src/pipecat/transports/network/websocket_server.py
View File

@@ -24,7 +24,6 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.serializers.base_serializer import FrameSerializer
from pipecat.serializers.protobuf import ProtobufFrameSerializer
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import BaseTransport, TransportParams
@@ -39,7 +38,7 @@ except ModuleNotFoundError as e:
class WebsocketServerParams(TransportParams):
add_wav_header: bool = False
serializer: FrameSerializer = ProtobufFrameSerializer()
serializer: FrameSerializer
session_timeout: int | None = None
@@ -67,21 +66,32 @@ class WebsocketServerInputTransport(BaseInputTransport):
self._websocket: websockets.WebSocketServerProtocol | None = None
self._server_task = None
# This task will monitor the websocket connection periodically.
self._monitor_task = None
self._stop_server_event = asyncio.Event()
async def start(self, frame: StartFrame):
await super().start(frame)
self._server_task = self.get_event_loop().create_task(self._server_task_handler())
await self._params.serializer.setup(frame)
self._server_task = self.create_task(self._server_task_handler())
async def stop(self, frame: EndFrame):
await super().stop(frame)
self._stop_server_event.set()
await self._server_task
if self._monitor_task:
await self.cancel_task(self._monitor_task)
if self._server_task:
await self.wait_for_task(self._server_task)
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
self._stop_server_event.set()
await self._server_task
if self._monitor_task:
await self.cancel_task(self._monitor_task)
if self._server_task:
await self.cancel_task(self._server_task)
async def _server_task_handler(self):
logger.info(f"Starting websocket server on {self._host}:{self._port}")
@@ -101,19 +111,24 @@ class WebsocketServerInputTransport(BaseInputTransport):
# Create a task to monitor the websocket connection
if self._params.session_timeout:
self.get_event_loop().create_task(self._monitor_websocket(websocket))
self._monitor_task = self.create_task(
self._monitor_websocket(websocket, self._params.session_timeout)
)
# Handle incoming messages
async for message in websocket:
frame = self._params.serializer.deserialize(message)
try:
async for message in websocket:
frame = await self._params.serializer.deserialize(message)
if not frame:
continue
if not frame:
continue
if isinstance(frame, InputAudioRawFrame):
await self.push_audio_frame(frame)
else:
await self.push_frame(frame)
if isinstance(frame, InputAudioRawFrame):
await self.push_audio_frame(frame)
else:
await self.push_frame(frame)
except Exception as e:
logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
# Notify disconnection
await self._callbacks.on_client_disconnected(websocket)
@@ -123,14 +138,18 @@ class WebsocketServerInputTransport(BaseInputTransport):
logger.info(f"Client {websocket.remote_address} disconnected")
async def _monitor_websocket(self, websocket: websockets.WebSocketServerProtocol):
"""Wait for self._params.session_timeout seconds, if the websocket is still open, trigger timeout event."""
async def _monitor_websocket(
self, websocket: websockets.WebSocketServerProtocol, session_timeout: int
):
"""Wait for session_timeout seconds, if the websocket is still open,
trigger timeout event."""
try:
await asyncio.sleep(self._params.session_timeout)
await asyncio.sleep(session_timeout)
if not websocket.closed:
await self._callbacks.on_session_timeout(websocket)
except asyncio.CancelledError:
logger.info(f"Monitoring task cancelled for: {websocket.remote_address}")
raise
class WebsocketServerOutputTransport(BaseOutputTransport):
@@ -141,9 +160,12 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
self._websocket: websockets.WebSocketServerProtocol | None = None
self._websocket_audio_buffer = bytes()
self._send_interval = (self._audio_chunk_size / self._params.audio_out_sample_rate) / 2
# write_raw_audio_frames() is called quickly, as soon as we get audio
# (e.g. from the TTS), and since this is just a network connection we
# would be sending it to quickly. Instead, we want to block to emulate
# an audio device, this is what the send interval is. It will be
# computed on StartFrame.
self._send_interval = 0
self._next_send_time = 0
async def set_client_connection(self, websocket: websockets.WebSocketServerProtocol | None):
@@ -152,6 +174,11 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
logger.warning("Only one client allowed, using new connection")
self._websocket = websocket
async def start(self, frame: StartFrame):
await super().start(frame)
await self._params.serializer.setup(frame)
self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -167,7 +194,7 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
frame = OutputAudioRawFrame(
audio=frames,
sample_rate=self._params.audio_out_sample_rate,
sample_rate=self.sample_rate,
num_channels=self._params.audio_out_channels,
)
@@ -187,15 +214,16 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
await self._write_frame(frame)
self._websocket_audio_buffer = bytes()
# Simulate audio playback with a sleep.
await self._write_audio_sleep()
async def _write_frame(self, frame: Frame):
payload = self._params.serializer.serialize(frame)
if payload and self._websocket:
await self._websocket.send(payload)
try:
payload = await self._params.serializer.serialize(frame)
if payload and self._websocket:
await self._websocket.send(payload)
except Exception as e:
logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})")
async def _write_audio_sleep(self):
# Simulate a clock.
@@ -211,14 +239,13 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
class WebsocketServerTransport(BaseTransport):
def __init__(
self,
params: WebsocketServerParams,
host: str = "localhost",
port: int = 8765,
params: WebsocketServerParams = WebsocketServerParams(),
input_name: str | None = None,
output_name: str | None = None,
loop: asyncio.AbstractEventLoop | None = None,
):
super().__init__(input_name=input_name, output_name=output_name, loop=loop)
super().__init__(input_name=input_name, output_name=output_name)
self._host = host
self._port = port
self._params = params

221
src/pipecat/transports/services/daily.py
View File

@@ -46,6 +46,7 @@ from pipecat.transcriptions.language import Language
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.utils.asyncio import TaskManager
try:
from daily import CallClient, Daily, EventHandler
@@ -70,11 +71,11 @@ class DailyTransportMessageUrgentFrame(TransportMessageUrgentFrame):
class WebRTCVADAnalyzer(VADAnalyzer):
def __init__(self, *, sample_rate=16000, num_channels=1, params: VADParams = VADParams()):
super().__init__(sample_rate=sample_rate, num_channels=num_channels, params=params)
def __init__(self, *, sample_rate: Optional[int] = None, params: VADParams = VADParams()):
super().__init__(sample_rate=sample_rate, params=params)
self._webrtc_vad = Daily.create_native_vad(
reset_period_ms=VAD_RESET_PERIOD_MS, sample_rate=sample_rate, channels=num_channels
reset_period_ms=VAD_RESET_PERIOD_MS, sample_rate=self.sample_rate, channels=1
)
logger.debug("Loaded native WebRTC VAD")
@@ -139,7 +140,6 @@ class DailyCallbacks(BaseModel):
on_dialout_stopped: Callable[[Any], Awaitable[None]]
on_dialout_error: Callable[[Any], Awaitable[None]]
on_dialout_warning: Callable[[Any], Awaitable[None]]
on_first_participant_joined: Callable[[Mapping[str, Any]], Awaitable[None]]
on_participant_joined: Callable[[Mapping[str, Any]], Awaitable[None]]
on_participant_left: Callable[[Mapping[str, Any], str], Awaitable[None]]
on_participant_updated: Callable[[Mapping[str, Any]], Awaitable[None]]
@@ -179,7 +179,7 @@ class DailyTransportClient(EventHandler):
bot_name: str,
params: DailyParams,
callbacks: DailyCallbacks,
loop: asyncio.AbstractEventLoop,
transport_name: str,
):
super().__init__()
@@ -192,17 +192,19 @@ class DailyTransportClient(EventHandler):
self._bot_name: str = bot_name
self._params: DailyParams = params
self._callbacks = callbacks
self._loop = loop
self._transport_name = transport_name
self._participant_id: str = ""
self._video_renderers = {}
self._transcription_ids = []
self._transcription_status = None
self._other_participant_has_joined = False
self._joined = False
self._joined_event = asyncio.Event()
self._leave_counter = 0
self._task_manager: Optional[TaskManager] = None
# We use the executor to cleanup the client. We just do it from one
# place, so only one thread is really needed.
self._executor = ThreadPoolExecutor(max_workers=1)
@@ -218,35 +220,15 @@ class DailyTransportClient(EventHandler):
# future) we will deadlock because completions use event handlers (which
# are holding the GIL).
self._callback_queue = asyncio.Queue()
self._callback_task = self._loop.create_task(self._callback_task_handler())
self._callback_task = None
# Input and ouput sample rates. They will be initialize on setup().
self._in_sample_rate = 0
self._out_sample_rate = 0
self._camera: VirtualCameraDevice | None = None
if self._params.camera_out_enabled:
self._camera = Daily.create_camera_device(
self._camera_name(),
width=self._params.camera_out_width,
height=self._params.camera_out_height,
color_format=self._params.camera_out_color_format,
)
self._mic: VirtualMicrophoneDevice | None = None
if self._params.audio_out_enabled:
self._mic = Daily.create_microphone_device(
self._mic_name(),
sample_rate=self._params.audio_out_sample_rate,
channels=self._params.audio_out_channels,
non_blocking=True,
)
self._speaker: VirtualSpeakerDevice | None = None
if self._params.audio_in_enabled or self._params.vad_enabled:
self._speaker = Daily.create_speaker_device(
self._speaker_name(),
sample_rate=self._params.audio_in_sample_rate,
channels=self._params.audio_in_channels,
non_blocking=True,
)
Daily.select_speaker_device(self._speaker_name())
def _camera_name(self):
return f"camera-{self}"
@@ -261,9 +243,6 @@ class DailyTransportClient(EventHandler):
def participant_id(self) -> str:
return self._participant_id
def set_callbacks(self, callbacks: DailyCallbacks):
self._callbacks = callbacks
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
if not self._joined:
return
@@ -272,7 +251,7 @@ class DailyTransportClient(EventHandler):
if isinstance(frame, (DailyTransportMessageFrame, DailyTransportMessageUrgentFrame)):
participant_id = frame.participant_id
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._client.send_app_message(
frame.message, participant_id, completion=completion_callback(future)
)
@@ -282,11 +261,11 @@ class DailyTransportClient(EventHandler):
if not self._speaker:
return None
sample_rate = self._params.audio_in_sample_rate
sample_rate = self._in_sample_rate
num_channels = self._params.audio_in_channels
num_frames = int(sample_rate / 100) * 2 # 20ms of audio
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._speaker.read_frames(num_frames, completion=completion_callback(future))
audio = await future
@@ -305,7 +284,7 @@ class DailyTransportClient(EventHandler):
if not self._mic:
return None
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._mic.write_frames(frames, completion=completion_callback(future))
await future
@@ -315,6 +294,42 @@ class DailyTransportClient(EventHandler):
self._camera.write_frame(frame.image)
async def setup(self, frame: StartFrame):
self._in_sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
self._out_sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
if self._params.camera_out_enabled and not self._camera:
self._camera = Daily.create_camera_device(
self._camera_name(),
width=self._params.camera_out_width,
height=self._params.camera_out_height,
color_format=self._params.camera_out_color_format,
)
if self._params.audio_out_enabled and not self._mic:
self._mic = Daily.create_microphone_device(
self._mic_name(),
sample_rate=self._out_sample_rate,
channels=self._params.audio_out_channels,
non_blocking=True,
)
if (self._params.audio_in_enabled or self._params.vad_enabled) and not self._speaker:
self._speaker = Daily.create_speaker_device(
self._speaker_name(),
sample_rate=self._in_sample_rate,
channels=self._params.audio_in_channels,
non_blocking=True,
)
Daily.select_speaker_device(self._speaker_name())
if not self._task_manager:
self._task_manager = frame.task_manager
self._callback_task = self._task_manager.create_task(
self._callback_task_handler(),
f"{self}::callback_task",
)
async def join(self):
# Transport already joined, ignore.
if self._joined:
@@ -346,6 +361,8 @@ class DailyTransportClient(EventHandler):
await self._start_transcription()
await self._callbacks.on_joined(data)
self._joined_event.set()
else:
error_msg = f"Error joining {self._room_url}: {error}"
logger.error(error_msg)
@@ -362,7 +379,7 @@ class DailyTransportClient(EventHandler):
logger.info(f"Enabling transcription with settings {self._params.transcription_settings}")
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._client.start_transcription(
settings=self._params.transcription_settings.model_dump(exclude_none=True),
completion=completion_callback(future),
@@ -373,7 +390,7 @@ class DailyTransportClient(EventHandler):
return
async def _join(self):
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._client.join(
self._room_url,
@@ -434,6 +451,7 @@ class DailyTransportClient(EventHandler):
return
self._joined = False
self._joined_event.clear()
logger.info(f"Leaving {self._room_url}")
@@ -457,23 +475,24 @@ class DailyTransportClient(EventHandler):
async def _stop_transcription(self):
if not self._token:
return
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._client.stop_transcription(completion=completion_callback(future))
error = await future
if error:
logger.error(f"Unable to stop transcription: {error}")
async def _leave(self):
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._client.leave(completion=completion_callback(future))
return await asyncio.wait_for(future, timeout=10)
async def cleanup(self):
self._callback_task.cancel()
await self._callback_task
if self._callback_task and self._task_manager:
await self._task_manager.cancel_task(self._callback_task)
self._callback_task = None
# Make sure we don't block the event loop in case `client.release()`
# takes extra time.
await self._loop.run_in_executor(self._executor, self._cleanup)
await self._get_event_loop().run_in_executor(self._executor, self._cleanup)
def _cleanup(self):
if self._client:
@@ -487,39 +506,39 @@ class DailyTransportClient(EventHandler):
return self._client.participant_counts()
async def start_dialout(self, settings):
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._client.start_dialout(settings, completion=completion_callback(future))
await future
async def stop_dialout(self, participant_id):
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._client.stop_dialout(participant_id, completion=completion_callback(future))
await future
async def send_dtmf(self, settings):
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._client.send_dtmf(settings, completion=completion_callback(future))
await future
async def sip_call_transfer(self, settings):
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._client.sip_call_transfer(settings, completion=completion_callback(future))
await future
async def sip_refer(self, settings):
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._client.sip_refer(settings, completion=completion_callback(future))
await future
async def start_recording(self, streaming_settings, stream_id, force_new):
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._client.start_recording(
streaming_settings, stream_id, force_new, completion=completion_callback(future)
)
await future
async def stop_recording(self, stream_id):
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._client.stop_recording(stream_id, completion=completion_callback(future))
await future
@@ -527,7 +546,7 @@ class DailyTransportClient(EventHandler):
if not self._joined:
return
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._client.send_prebuilt_chat_message(
message, user_name=user_name, completion=completion_callback(future)
)
@@ -564,14 +583,14 @@ class DailyTransportClient(EventHandler):
)
async def update_transcription(self, participants=None, instance_id=None):
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._client.update_transcription(
participants, instance_id, completion=completion_callback(future)
)
await future
async def update_subscriptions(self, participant_settings=None, profile_settings=None):
future = self._loop.create_future()
future = self._get_event_loop().create_future()
self._client.update_subscriptions(
participant_settings=participant_settings,
profile_settings=profile_settings,
@@ -621,19 +640,9 @@ class DailyTransportClient(EventHandler):
self._call_async_callback(self._callbacks.on_dialout_warning, data)
def on_participant_joined(self, participant):
id = participant["id"]
logger.info(f"Participant joined {id}")
if not self._other_participant_has_joined:
self._other_participant_has_joined = True
self._call_async_callback(self._callbacks.on_first_participant_joined, participant)
self._call_async_callback(self._callbacks.on_participant_joined, participant)
def on_participant_left(self, participant, reason):
id = participant["id"]
logger.info(f"Participant left {id}")
self._call_async_callback(self._callbacks.on_participant_left, participant, reason)
def on_participant_updated(self, participant):
@@ -681,17 +690,24 @@ class DailyTransportClient(EventHandler):
def _call_async_callback(self, callback, *args):
future = asyncio.run_coroutine_threadsafe(
self._callback_queue.put((callback, *args)), self._loop
self._callback_queue.put((callback, *args)), self._get_event_loop()
)
future.result()
async def _callback_task_handler(self):
while True:
try:
(callback, *args) = await self._callback_queue.get()
await callback(*args)
except asyncio.CancelledError:
break
# Wait to process any callback until we are joined.
await self._joined_event.wait()
(callback, *args) = await self._callback_queue.get()
await callback(*args)
def _get_event_loop(self) -> asyncio.AbstractEventLoop:
if not self._task_manager:
raise Exception(f"{self}: missing task manager (pipeline not started?)")
return self._task_manager.get_event_loop()
def __str__(self):
return f"{self._transport_name}::DailyTransportClient"
class DailyInputTransport(BaseInputTransport):
@@ -699,6 +715,7 @@ class DailyInputTransport(BaseInputTransport):
super().__init__(params, **kwargs)
self._client = client
self._params = params
self._video_renderers = {}
@@ -707,21 +724,25 @@ class DailyInputTransport(BaseInputTransport):
self._audio_in_task = None
self._vad_analyzer: VADAnalyzer | None = params.vad_analyzer
if params.vad_enabled and not params.vad_analyzer:
self._vad_analyzer = WebRTCVADAnalyzer(
sample_rate=self._params.audio_in_sample_rate,
num_channels=self._params.audio_in_channels,
)
@property
def vad_analyzer(self) -> VADAnalyzer | None:
return self._vad_analyzer
async def start(self, frame: StartFrame):
# Parent start.
await super().start(frame)
# Setup client.
await self._client.setup(frame)
# Join the room.
await self._client.join()
# Inialize WebRTC VAD if needed.
if self._params.vad_enabled and not self._params.vad_analyzer:
self._vad_analyzer = WebRTCVADAnalyzer(sample_rate=self.sample_rate)
# Create audio task. It reads audio frames from Daily and push them
# internally for VAD processing.
if self._params.audio_in_enabled or self._params.vad_enabled:
self._audio_in_task = self.get_event_loop().create_task(self._audio_in_task_handler())
self._audio_in_task = self.create_task(self._audio_in_task_handler())
async def stop(self, frame: EndFrame):
# Parent stop.
@@ -730,8 +751,7 @@ class DailyInputTransport(BaseInputTransport):
await self._client.leave()
# Stop audio thread.
if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
self._audio_in_task.cancel()
await self._audio_in_task
await self.cancel_task(self._audio_in_task)
self._audio_in_task = None
async def cancel(self, frame: CancelFrame):
@@ -741,17 +761,13 @@ class DailyInputTransport(BaseInputTransport):
await self._client.leave()
# Stop audio thread.
if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
self._audio_in_task.cancel()
await self._audio_in_task
await self.cancel_task(self._audio_in_task)
self._audio_in_task = None
async def cleanup(self):
await super().cleanup()
await self._client.cleanup()
def vad_analyzer(self) -> VADAnalyzer | None:
return self._vad_analyzer
#
# FrameProcessor
#
@@ -779,12 +795,9 @@ class DailyInputTransport(BaseInputTransport):
async def _audio_in_task_handler(self):
while True:
try:
frame = await self._client.read_next_audio_frame()
if frame:
await self.push_audio_frame(frame)
except asyncio.CancelledError:
break
frame = await self._client.read_next_audio_frame()
if frame:
await self.push_audio_frame(frame)
#
# Camera in
@@ -843,6 +856,8 @@ class DailyOutputTransport(BaseOutputTransport):
async def start(self, frame: StartFrame):
# Parent start.
await super().start(frame)
# Setup client.
await self._client.setup(frame)
# Join the room.
await self._client.join()
@@ -881,9 +896,8 @@ class DailyTransport(BaseTransport):
params: DailyParams = DailyParams(),
input_name: str | None = None,
output_name: str | None = None,
loop: asyncio.AbstractEventLoop | None = None,
):
super().__init__(input_name=input_name, output_name=output_name, loop=loop)
super().__init__(input_name=input_name, output_name=output_name)
callbacks = DailyCallbacks(
on_joined=self._on_joined,
@@ -901,7 +915,6 @@ class DailyTransport(BaseTransport):
on_dialout_stopped=self._on_dialout_stopped,
on_dialout_error=self._on_dialout_error,
on_dialout_warning=self._on_dialout_warning,
on_first_participant_joined=self._on_first_participant_joined,
on_participant_joined=self._on_participant_joined,
on_participant_left=self._on_participant_left,
on_participant_updated=self._on_participant_updated,
@@ -912,12 +925,12 @@ class DailyTransport(BaseTransport):
)
self._params = params
self._client = DailyTransportClient(
room_url, token, bot_name, params, callbacks, self._loop
)
self._client = DailyTransportClient(room_url, token, bot_name, params, callbacks, self.name)
self._input: DailyInputTransport | None = None
self._output: DailyOutputTransport | None = None
self._other_participant_has_joined = False
# Register supported handlers. The user will only be able to register
# these handlers.
self._register_event_handler("on_joined")
@@ -1047,7 +1060,7 @@ class DailyTransport(BaseTransport):
await self._output.push_error(error_frame)
else:
logger.error("Both input and output are None while trying to push error")
raise RuntimeError("No valid input or output channel to push error")
raise Exception("No valid input or output channel to push error")
async def _on_app_message(self, message: Any, sender: str):
if self._input:
@@ -1124,17 +1137,23 @@ class DailyTransport(BaseTransport):
await self._call_event_handler("on_dialout_warning", data)
async def _on_participant_joined(self, participant):
id = participant["id"]
logger.info(f"Participant joined {id}")
if not self._other_participant_has_joined:
self._other_participant_has_joined = True
await self._call_event_handler("on_first_participant_joined", participant)
await self._call_event_handler("on_participant_joined", participant)
async def _on_participant_left(self, participant, reason):
id = participant["id"]
logger.info(f"Participant left {id}")
await self._call_event_handler("on_participant_left", participant, reason)
async def _on_participant_updated(self, participant):
await self._call_event_handler("on_participant_updated", participant)
async def _on_first_participant_joined(self, participant):
await self._call_event_handler("on_first_participant_joined", participant)
async def _on_transcription_message(self, message):
await self._call_event_handler("on_transcription_message", message)

125
src/pipecat/transports/services/helpers/daily_rest.py
View File

@@ -33,6 +33,19 @@ class DailyRoomSipParams(BaseModel):
num_endpoints: int = 1
class RecordingsBucketConfig(BaseModel):
"""Configuration for storing Daily recordings in a custom S3 bucket.
Refer to the Daily API documentation for more information:
https://docs.daily.co/guides/products/live-streaming-recording/storing-recordings-in-a-custom-s3-bucket
"""
bucket_name: str
bucket_region: str
assume_role_arn: str
allow_api_access: bool = False
class DailyRoomProperties(BaseModel, extra="allow"):
"""Properties for configuring a Daily room.
@@ -43,6 +56,8 @@ class DailyRoomProperties(BaseModel, extra="allow"):
enable_emoji_reactions: Whether emoji reactions are enabled
eject_at_room_exp: Whether to remove participants when room expires
enable_dialout: Whether SIP dial-out is enabled
enable_recording: Recording settings ('cloud', 'local', 'raw-tracks')
geo: Geographic region for room
max_participants: Maximum number of participants allowed in the room
sip: SIP configuration parameters
sip_uri: SIP URI information returned by Daily
@@ -57,7 +72,10 @@ class DailyRoomProperties(BaseModel, extra="allow"):
enable_emoji_reactions: bool = False
eject_at_room_exp: bool = True
enable_dialout: Optional[bool] = None
enable_recording: Optional[Literal["cloud", "local", "raw-tracks"]] = None
geo: Optional[str] = None
max_participants: Optional[int] = None
recordings_bucket: Optional[RecordingsBucketConfig] = None
sip: Optional[DailyRoomSipParams] = None
sip_uri: Optional[dict] = None
start_video_off: bool = False
@@ -111,6 +129,84 @@ class DailyRoomObject(BaseModel):
config: DailyRoomProperties
class DailyMeetingTokenProperties(BaseModel):
"""Properties for configuring a Daily meeting token.
Refer to the Daily API documentation for more information:
https://docs.daily.co/reference/rest-api/meeting-tokens/create-meeting-token#properties
"""
room_name: Optional[str] = Field(
default=None,
description="The room for which this token is valid. If not set, the token is valid for all rooms in your domain. You should always set room_name if using this token to control meeting access.",
)
eject_at_token_exp: Optional[bool] = Field(
default=None,
description="If `true`, the user will be ejected from the room when the token expires. Defaults to `false`.",
)
eject_after_elapsed: Optional[int] = Field(
default=None,
description="The number of seconds after which the user will be ejected from the room. If not provided, the user will not be ejected based on elapsed time.",
)
nbf: Optional[int] = Field(
default=None,
description="Not before. This is a unix timestamp (seconds since the epoch.) Users cannot join a meeting in with this token before this time.",
)
exp: Optional[int] = Field(
default=None,
description="Expiration time (unix timestamp in seconds). We strongly recommend setting this value for security. If not set, the token will not expire. Refer docs for more info.",
)
is_owner: Optional[bool] = Field(
default=None,
description="If `true`, the token will grant owner privileges in the room. Defaults to `false`.",
)
user_name: Optional[str] = Field(
default=None,
description="The name of the user. This will be added to the token payload.",
)
user_id: Optional[str] = Field(
default=None,
description="A unique identifier for the user. This will be added to the token payload.",
)
enable_screenshare: Optional[bool] = Field(
default=None,
description="If `true`, the user will be able to share their screen. Defaults to `true`.",
)
start_video_off: Optional[bool] = Field(
default=None,
description="If `true`, the user's video will be turned off when they join the room. Defaults to `false`.",
)
start_audio_off: Optional[bool] = Field(
default=None,
description="If `true`, the user's audio will be turned off when they join the room. Defaults to `false`.",
)
enable_recording: Optional[Literal["cloud", "local", "raw-tracks"]] = Field(
default=None,
description="Recording settings for the token. Must be one of `cloud`, `local` or `raw-tracks`.",
)
enable_prejoin_ui: Optional[bool] = Field(
default=None,
description="If `true`, the user will see the prejoin UI before joining the room.",
)
start_cloud_recording: Optional[bool] = Field(
default=None,
description="Start cloud recording when the user joins the room. This can be used to always record and archive meetings, for example in a customer support context.",
)
class DailyMeetingTokenParams(BaseModel):
"""Parameters for creating a Daily meeting token.
Refer to the Daily API documentation for more information:
https://docs.daily.co/reference/rest-api/meeting-tokens/create-meeting-token#body-params
"""
properties: DailyMeetingTokenProperties = Field(default_factory=DailyMeetingTokenProperties)
class DailyRESTHelper:
"""Helper class for interacting with Daily's REST API.
@@ -129,6 +225,7 @@ class DailyRESTHelper:
daily_api_url: str = "https://api.daily.co/v1",
aiohttp_session: aiohttp.ClientSession,
):
"""Initialize the Daily REST helper."""
self.daily_api_key = daily_api_key
self.daily_api_url = daily_api_url
self.aiohttp_session = aiohttp_session
@@ -169,7 +266,7 @@ class DailyRESTHelper:
Exception: If room creation fails or response is invalid
"""
headers = {"Authorization": f"Bearer {self.daily_api_key}"}
json = {**params.model_dump(exclude_none=True)}
json = params.model_dump(exclude_none=True)
async with self.aiohttp_session.post(
f"{self.daily_api_url}/rooms", headers=headers, json=json
) as r:
@@ -187,7 +284,11 @@ class DailyRESTHelper:
return room
async def get_token(
self, room_url: str, expiry_time: float = 60 * 60, owner: bool = True
self,
room_url: str,
expiry_time: float = 60 * 60,
owner: bool = True,
params: Optional[DailyMeetingTokenParams] = None,
) -> str:
"""Generate a meeting token for user to join a Daily room.
@@ -195,6 +296,9 @@ class DailyRESTHelper:
room_url: Daily room URL
expiry_time: Token validity duration in seconds (default: 1 hour)
owner: Whether token has owner privileges
params: Optional additional token properties. Note that room_name,
exp, and is_owner will be set based on the other function
parameters regardless of values in params.
Returns:
str: Meeting token
@@ -207,12 +311,25 @@ class DailyRESTHelper:
"No Daily room specified. You must specify a Daily room in order a token to be generated."
)
expiration: float = time.time() + expiry_time
expiration: int = int(time.time() + expiry_time)
room_name = self.get_name_from_url(room_url)
headers = {"Authorization": f"Bearer {self.daily_api_key}"}
json = {"properties": {"room_name": room_name, "is_owner": owner, "exp": expiration}}
if params is None:
params = DailyMeetingTokenParams(
properties=DailyMeetingTokenProperties(
room_name=room_name, is_owner=owner, exp=expiration
)
)
else:
params.properties.room_name = room_name
params.properties.exp = expiration
params.properties.is_owner = owner
json = params.model_dump(exclude_none=True)
async with self.aiohttp_session.post(
f"{self.daily_api_url}/meeting-tokens", headers=headers, json=json
) as r:

174
src/pipecat/transports/services/livekit.py
View File

@@ -6,18 +6,17 @@
import asyncio
from dataclasses import dataclass
from typing import Any, Awaitable, Callable, List
from typing import Any, Awaitable, Callable, List, Optional
from loguru import logger
from pydantic import BaseModel
from pipecat.audio.utils import resample_audio
from pipecat.audio.utils import create_default_resampler
from pipecat.audio.vad.vad_analyzer import VADAnalyzer
from pipecat.frames.frames import (
AudioRawFrame,
CancelFrame,
EndFrame,
Frame,
InputAudioRawFrame,
OutputAudioRawFrame,
StartFrame,
@@ -28,6 +27,7 @@ from pipecat.processors.frame_processor import FrameDirection
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.utils.asyncio import TaskManager
try:
from livekit import rtc
@@ -71,15 +71,15 @@ class LiveKitTransportClient:
room_name: str,
params: LiveKitParams,
callbacks: LiveKitCallbacks,
loop: asyncio.AbstractEventLoop,
transport_name: str,
):
self._url = url
self._token = token
self._room_name = room_name
self._params = params
self._callbacks = callbacks
self._loop = loop
self._room = rtc.Room(loop=loop)
self._transport_name = transport_name
self._room: rtc.Room | None = None
self._participant_id: str = ""
self._connected = False
self._disconnect_counter = 0
@@ -88,20 +88,33 @@ class LiveKitTransportClient:
self._audio_tracks = {}
self._audio_queue = asyncio.Queue()
self._other_participant_has_joined = False
# Set up room event handlers
self._room.on("participant_connected")(self._on_participant_connected_wrapper)
self._room.on("participant_disconnected")(self._on_participant_disconnected_wrapper)
self._room.on("track_subscribed")(self._on_track_subscribed_wrapper)
self._room.on("track_unsubscribed")(self._on_track_unsubscribed_wrapper)
self._room.on("data_received")(self._on_data_received_wrapper)
self._room.on("connected")(self._on_connected_wrapper)
self._room.on("disconnected")(self._on_disconnected_wrapper)
self._task_manager: Optional[TaskManager] = None
@property
def participant_id(self) -> str:
return self._participant_id
@property
def room(self) -> rtc.Room:
if not self._room:
raise Exception(f"{self}: missing room object (pipeline not started?)")
return self._room
async def setup(self, frame: StartFrame):
self._out_sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
if not self._task_manager:
self._task_manager = frame.task_manager
self._room = rtc.Room(loop=self._task_manager.get_event_loop())
# Set up room event handlers
self.room.on("participant_connected")(self._on_participant_connected_wrapper)
self.room.on("participant_disconnected")(self._on_participant_disconnected_wrapper)
self.room.on("track_subscribed")(self._on_track_subscribed_wrapper)
self.room.on("track_unsubscribed")(self._on_track_unsubscribed_wrapper)
self.room.on("data_received")(self._on_data_received_wrapper)
self.room.on("connected")(self._on_connected_wrapper)
self.room.on("disconnected")(self._on_disconnected_wrapper)
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
async def connect(self):
if self._connected:
@@ -112,7 +125,7 @@ class LiveKitTransportClient:
logger.info(f"Connecting to {self._room_name}")
try:
await self._room.connect(
await self.room.connect(
self._url,
self._token,
options=rtc.RoomOptions(auto_subscribe=True),
@@ -121,19 +134,19 @@ class LiveKitTransportClient:
# Increment disconnect counter if we successfully connected.
self._disconnect_counter += 1
self._participant_id = self._room.local_participant.sid
self._participant_id = self.room.local_participant.sid
logger.info(f"Connected to {self._room_name}")
# Set up audio source and track
self._audio_source = rtc.AudioSource(
self._params.audio_out_sample_rate, self._params.audio_out_channels
self._out_sample_rate, self._params.audio_out_channels
)
self._audio_track = rtc.LocalAudioTrack.create_audio_track(
"pipecat-audio", self._audio_source
)
options = rtc.TrackPublishOptions()
options.source = rtc.TrackSource.SOURCE_MICROPHONE
await self._room.local_participant.publish_track(self._audio_track, options)
await self.room.local_participant.publish_track(self._audio_track, options)
await self._callbacks.on_connected()
@@ -154,7 +167,7 @@ class LiveKitTransportClient:
return
logger.info(f"Disconnecting from {self._room_name}")
await self._room.disconnect()
await self.room.disconnect()
self._connected = False
logger.info(f"Disconnected from {self._room_name}")
await self._callbacks.on_disconnected()
@@ -165,11 +178,11 @@ class LiveKitTransportClient:
try:
if participant_id:
await self._room.local_participant.publish_data(
await self.room.local_participant.publish_data(
data, reliable=True, destination_identities=[participant_id]
)
else:
await self._room.local_participant.publish_data(data, reliable=True)
await self.room.local_participant.publish_data(data, reliable=True)
except Exception as e:
logger.error(f"Error sending data: {e}")
@@ -183,10 +196,10 @@ class LiveKitTransportClient:
logger.error(f"Error publishing audio: {e}")
def get_participants(self) -> List[str]:
return [p.sid for p in self._room.remote_participants.values()]
return [p.sid for p in self.room.remote_participants.values()]
async def get_participant_metadata(self, participant_id: str) -> dict:
participant = self._room.remote_participants.get(participant_id)
participant = self.room.remote_participants.get(participant_id)
if participant:
return {
"id": participant.sid,
@@ -197,17 +210,17 @@ class LiveKitTransportClient:
return {}
async def set_participant_metadata(self, metadata: str):
await self._room.local_participant.set_metadata(metadata)
await self.room.local_participant.set_metadata(metadata)
async def mute_participant(self, participant_id: str):
participant = self._room.remote_participants.get(participant_id)
participant = self.room.remote_participants.get(participant_id)
if participant:
for track in participant.tracks.values():
if track.kind == "audio":
await track.set_enabled(False)
async def unmute_participant(self, participant_id: str):
participant = self._room.remote_participants.get(participant_id)
participant = self.room.remote_participants.get(participant_id)
if participant:
for track in participant.tracks.values():
if track.kind == "audio":
@@ -215,10 +228,16 @@ class LiveKitTransportClient:
# Wrapper methods for event handlers
def _on_participant_connected_wrapper(self, participant: rtc.RemoteParticipant):
asyncio.create_task(self._async_on_participant_connected(participant))
self._task_manager.create_task(
self._async_on_participant_connected(participant),
f"{self}::_async_on_participant_connected",
)
def _on_participant_disconnected_wrapper(self, participant: rtc.RemoteParticipant):
asyncio.create_task(self._async_on_participant_disconnected(participant))
self._task_manager.create_task(
self._async_on_participant_disconnected(participant),
f"{self}::_async_on_participant_disconnected",
)
def _on_track_subscribed_wrapper(
self,
@@ -226,7 +245,10 @@ class LiveKitTransportClient:
publication: rtc.RemoteTrackPublication,
participant: rtc.RemoteParticipant,
):
asyncio.create_task(self._async_on_track_subscribed(track, publication, participant))
self._task_manager.create_task(
self._async_on_track_subscribed(track, publication, participant),
f"{self}::_async_on_track_subscribed",
)
def _on_track_unsubscribed_wrapper(
self,
@@ -234,16 +256,24 @@ class LiveKitTransportClient:
publication: rtc.RemoteTrackPublication,
participant: rtc.RemoteParticipant,
):
asyncio.create_task(self._async_on_track_unsubscribed(track, publication, participant))
self._task_manager.create_task(
self._async_on_track_unsubscribed(track, publication, participant),
f"{self}::_async_on_track_unsubscribed",
)
def _on_data_received_wrapper(self, data: rtc.DataPacket):
asyncio.create_task(self._async_on_data_received(data))
self._task_manager.create_task(
self._async_on_data_received(data),
f"{self}::_async_on_data_received",
)
def _on_connected_wrapper(self):
asyncio.create_task(self._async_on_connected())
self._task_manager.create_task(self._async_on_connected(), f"{self}::_async_on_connected")
def _on_disconnected_wrapper(self):
asyncio.create_task(self._async_on_disconnected())
self._task_manager.create_task(
self._async_on_disconnected(), f"{self}::_async_on_disconnected"
)
# Async methods for event handling
async def _async_on_participant_connected(self, participant: rtc.RemoteParticipant):
@@ -269,7 +299,10 @@ class LiveKitTransportClient:
logger.info(f"Audio track subscribed: {track.sid} from participant {participant.sid}")
self._audio_tracks[participant.sid] = track
audio_stream = rtc.AudioStream(track)
asyncio.create_task(self._process_audio_stream(audio_stream, participant.sid))
self._task_manager.create_task(
self._process_audio_stream(audio_stream, participant.sid),
f"{self}::_process_audio_stream",
)
async def _async_on_track_unsubscribed(
self,
@@ -307,6 +340,9 @@ class LiveKitTransportClient:
frame, participant_id = await self._audio_queue.get()
return frame, participant_id
def __str__(self):
return f"{self._transport_name}::LiveKitTransportClient"
class LiveKitInputTransport(BaseInputTransport):
def __init__(self, client: LiveKitTransportClient, params: LiveKitParams, **kwargs):
@@ -314,31 +350,32 @@ class LiveKitInputTransport(BaseInputTransport):
self._client = client
self._audio_in_task = None
self._vad_analyzer: VADAnalyzer | None = params.vad_analyzer
self._resampler = create_default_resampler()
@property
def vad_analyzer(self) -> VADAnalyzer | None:
return self._vad_analyzer
async def start(self, frame: StartFrame):
await super().start(frame)
await self._client.setup(frame)
await self._client.connect()
if self._params.audio_in_enabled or self._params.vad_enabled:
self._audio_in_task = asyncio.create_task(self._audio_in_task_handler())
self._audio_in_task = self.create_task(self._audio_in_task_handler())
logger.info("LiveKitInputTransport started")
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._client.disconnect()
if self._audio_in_task:
self._audio_in_task.cancel()
await self._audio_in_task
await self.cancel_task(self._audio_in_task)
logger.info("LiveKitInputTransport stopped")
async def cancel(self, frame: CancelFrame):
await super().cancel(frame)
await self._client.disconnect()
if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled):
self._audio_in_task.cancel()
await self._audio_in_task
def vad_analyzer(self) -> VADAnalyzer | None:
return self._vad_analyzer
await self.cancel_task(self._audio_in_task)
async def push_app_message(self, message: Any, sender: str):
frame = LiveKitTransportMessageUrgentFrame(message=message, participant_id=sender)
@@ -347,38 +384,31 @@ class LiveKitInputTransport(BaseInputTransport):
async def _audio_in_task_handler(self):
logger.info("Audio input task started")
while True:
try:
audio_data = await self._client.get_next_audio_frame()
if audio_data:
audio_frame_event, participant_id = audio_data
pipecat_audio_frame = self._convert_livekit_audio_to_pipecat(audio_frame_event)
input_audio_frame = InputAudioRawFrame(
audio=pipecat_audio_frame.audio,
sample_rate=pipecat_audio_frame.sample_rate,
num_channels=pipecat_audio_frame.num_channels,
)
await self.push_audio_frame(input_audio_frame)
except asyncio.CancelledError:
logger.info("Audio input task cancelled")
break
except Exception as e:
logger.error(f"Error in audio input task: {e}")
audio_data = await self._client.get_next_audio_frame()
if audio_data:
audio_frame_event, participant_id = audio_data
pipecat_audio_frame = await self._convert_livekit_audio_to_pipecat(
audio_frame_event
)
input_audio_frame = InputAudioRawFrame(
audio=pipecat_audio_frame.audio,
sample_rate=pipecat_audio_frame.sample_rate,
num_channels=pipecat_audio_frame.num_channels,
)
await self.push_audio_frame(input_audio_frame)
def _convert_livekit_audio_to_pipecat(
async def _convert_livekit_audio_to_pipecat(
self, audio_frame_event: rtc.AudioFrameEvent
) -> AudioRawFrame:
audio_frame = audio_frame_event.frame
audio_data = audio_frame.data
original_sample_rate = audio_frame.sample_rate
if original_sample_rate != self._params.audio_in_sample_rate:
audio_data = resample_audio(
audio_data, original_sample_rate, self._params.audio_in_sample_rate
)
audio_data = await self._resampler.resample(
audio_frame.data.tobytes(), audio_frame.sample_rate, self.sample_rate
)
return AudioRawFrame(
audio=audio_data,
sample_rate=self._params.audio_in_sample_rate,
sample_rate=self.sample_rate,
num_channels=audio_frame.num_channels,
)
@@ -390,6 +420,7 @@ class LiveKitOutputTransport(BaseOutputTransport):
async def start(self, frame: StartFrame):
await super().start(frame)
await self._client.setup(frame)
await self._client.connect()
logger.info("LiveKitOutputTransport started")
@@ -419,7 +450,7 @@ class LiveKitOutputTransport(BaseOutputTransport):
return rtc.AudioFrame(
data=pipecat_audio,
sample_rate=self._params.audio_out_sample_rate,
sample_rate=self.sample_rate,
num_channels=self._params.audio_out_channels,
samples_per_channel=samples_per_channel,
)
@@ -434,9 +465,8 @@ class LiveKitTransport(BaseTransport):
params: LiveKitParams = LiveKitParams(),
input_name: str | None = None,
output_name: str | None = None,
loop: asyncio.AbstractEventLoop | None = None,
):
super().__init__(input_name=input_name, output_name=output_name, loop=loop)
super().__init__(input_name=input_name, output_name=output_name)
callbacks = LiveKitCallbacks(
on_connected=self._on_connected,
@@ -451,7 +481,7 @@ class LiveKitTransport(BaseTransport):
self._params = params
self._client = LiveKitTransportClient(
url, token, room_name, self._params, callbacks, self._loop
url, token, room_name, self._params, callbacks, self.name
)
self._input: LiveKitInputTransport | None = None
self._output: LiveKitOutputTransport | None = None
@@ -517,7 +547,7 @@ class LiveKitTransport(BaseTransport):
async def _on_audio_track_subscribed(self, participant_id: str):
await self._call_event_handler("on_audio_track_subscribed", participant_id)
participant = self._client._room.remote_participants.get(participant_id)
participant = self._client.room.remote_participants.get(participant_id)
if participant:
for publication in participant.audio_tracks.values():
self._client._on_track_subscribed_wrapper(

129
src/pipecat/utils/asyncio.py Normal file
View File

@@ -0,0 +1,129 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
from typing import Coroutine, Optional, Set
from loguru import logger
class TaskManager:
def __init__(self) -> None:
self._tasks: Set[asyncio.Task] = set()
self._loop: Optional[asyncio.AbstractEventLoop] = None
def set_event_loop(self, loop: asyncio.AbstractEventLoop):
self._loop = loop
def get_event_loop(self) -> asyncio.AbstractEventLoop:
if not self._loop:
raise Exception("TaskManager missing event loop, use TaskManager.set_event_loop().")
return self._loop
def create_task(self, coroutine: Coroutine, name: str) -> asyncio.Task:
"""
Creates and schedules a new asyncio Task that runs the given coroutine.
The task is added to a global set of created tasks.
Args:
loop (asyncio.AbstractEventLoop): The event loop to use for creating the task.
coroutine (Coroutine): The coroutine to be executed within the task.
name (str): The name to assign to the task for identification.
Returns:
asyncio.Task: The created task object.
"""
async def run_coroutine():
try:
await coroutine
except asyncio.CancelledError:
logger.trace(f"{name}: task cancelled")
# Re-raise the exception to ensure the task is cancelled.
raise
except Exception as e:
logger.exception(f"{name}: unexpected exception: {e}")
if not self._loop:
raise Exception("TaskManager missing event loop, use TaskManager.set_event_loop().")
task = self._loop.create_task(run_coroutine())
task.set_name(name)
self._add_task(task)
logger.trace(f"{name}: task created")
return task
async def wait_for_task(self, task: asyncio.Task, timeout: Optional[float] = None):
"""Wait for an asyncio.Task to complete with optional timeout handling.
This function awaits the specified asyncio.Task and handles scenarios for
timeouts, cancellations, and other exceptions. It also ensures that the task
is removed from the set of registered tasks upon completion or failure.
Args:
task (asyncio.Task): The asyncio Task to wait for.
timeout (Optional[float], optional): The maximum number of seconds
to wait for the task to complete. If None, waits indefinitely.
Defaults to None.
"""
name = task.get_name()
try:
if timeout:
await asyncio.wait_for(task, timeout=timeout)
else:
await task
except asyncio.TimeoutError:
logger.warning(f"{name}: timed out waiting for task to finish")
except asyncio.CancelledError:
logger.trace(f"{name}: unexpected task cancellation (maybe Ctrl-C?)")
except Exception as e:
logger.exception(f"{name}: unexpected exception while stopping task: {e}")
finally:
self._remove_task(task)
async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None):
"""Cancels the given asyncio Task and awaits its completion with an
optional timeout.
This function removes the task from the set of registered tasks upon
completion or failure.
Args:
task (asyncio.Task): The task to be cancelled.
timeout (Optional[float]): The optional timeout in seconds to wait for the task to cancel.
"""
name = task.get_name()
task.cancel()
try:
if timeout:
await asyncio.wait_for(task, timeout=timeout)
else:
await task
except asyncio.TimeoutError:
logger.warning(f"{name}: timed out waiting for task to cancel")
except asyncio.CancelledError:
# Here are sure the task is cancelled properly.
pass
except Exception as e:
logger.exception(f"{name}: unexpected exception while cancelling task: {e}")
finally:
self._remove_task(task)
def current_tasks(self) -> Set[asyncio.Task]:
"""Returns the list of currently created/registered tasks."""
return self._tasks
def _add_task(self, task: asyncio.Task):
self._tasks.add(task)
def _remove_task(self, task: asyncio.Task):
name = task.get_name()
try:
self._tasks.remove(task)
except KeyError as e:
logger.trace(f"{name}: unable to remove task (already removed?): {e}")

2
src/pipecat/utils/string.py
View File

@@ -14,7 +14,7 @@ ENDOFSENTENCE_PATTERN_STR = r"""
(?<!Mrs) # Negative lookbehind: not preceded by "Mrs"
(?<!Prof) # Negative lookbehind: not preceded by "Prof"
[\.\?\!:;]| # Match a period, question mark, exclamation point, colon, or semicolon
[。？！：；] # the full-width version (mainly used in East Asian languages such as Chinese)
[。？！：；।] # the full-width version (mainly used in East Asian languages such as Chinese, Hindi)
$ # End of string
"""
ENDOFSENTENCE_PATTERN = re.compile(ENDOFSENTENCE_PATTERN_STR, re.VERBOSE)

10
src/pipecat/utils/utils.py
View File

@@ -3,11 +3,15 @@
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import collections
import itertools
import threading
_COUNTS = collections.defaultdict(itertools.count)
_COUNTS_LOCK = threading.Lock()
_ID = itertools.count()
_ID_LOCK = threading.Lock()
def obj_id() -> int:
@@ -20,7 +24,8 @@ def obj_id() -> int:
>>> obj_id()
2
"""
return next(_ID)
with _ID_LOCK:
return next(_ID)
def obj_count(obj) -> int:
@@ -34,4 +39,5 @@ def obj_count(obj) -> int:
>>> obj_count(new_type())
0
"""
return next(_COUNTS[obj.__class__.__name__])
with _COUNTS_LOCK:
return next(_COUNTS[obj.__class__.__name__])