wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use new InputAudioRawFrame and OutputAudioRawFrame

Browse Source
This commit is contained in:
joachimchauvet
2024-10-05 14:16:44 +03:00
parent b373bc82b5
commit 86143f79a1
1 changed files with 34 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
src/pipecat/transports/services/livekit.py
View File

@@ -1,25 +1,17 @@
#
# Copyright (c) 2024, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio import asyncio
from dataclasses import dataclass from dataclasses import dataclass
from typing import Any, Awaitable, Callable, List from typing import Any, Awaitable, Callable, List
from pydantic import BaseModel
import numpy as np import numpy as np
from scipy import signal from loguru import logger
from pipecat.frames.frames import ( from pipecat.frames.frames import (
AudioRawFrame, AudioRawFrame,
CancelFrame, CancelFrame,
EndFrame, EndFrame,
Frame, Frame,
InputAudioRawFrame,
MetricsFrame, MetricsFrame,
OutputAudioRawFrame,
StartFrame, StartFrame,
TransportMessageFrame, TransportMessageFrame,
) )
@@ -29,13 +21,13 @@ from pipecat.metrics.metrics import (
TTFBMetricsData, TTFBMetricsData,
TTSUsageMetricsData, TTSUsageMetricsData,
) )
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection
from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.vad.vad_analyzer import VADAnalyzer from pipecat.vad.vad_analyzer import VADAnalyzer
from pydantic import BaseModel
from loguru import logger from scipy import signal
try: try:
from livekit import rtc from livekit import rtc
@@ -364,10 +356,15 @@ class LiveKitInputTransport(BaseInputTransport):
if audio_data: if audio_data:
audio_frame_event, participant_id = audio_data audio_frame_event, participant_id = audio_data
pipecat_audio_frame = self._convert_livekit_audio_to_pipecat(audio_frame_event) pipecat_audio_frame = self._convert_livekit_audio_to_pipecat(audio_frame_event)
await self.push_audio_frame(pipecat_audio_frame) input_audio_frame = InputAudioRawFrame(
audio=pipecat_audio_frame.audio,
sample_rate=pipecat_audio_frame.sample_rate,
num_channels=pipecat_audio_frame.num_channels,
)
await self.push_frame( await self.push_frame(
pipecat_audio_frame pipecat_audio_frame
) # TODO: ensure audio frames are pushed with the default BaseInputTransport.push_audio_frame() ) # TODO: ensure audio frames are pushed with the default BaseInputTransport.push_audio_frame()
await self.push_audio_frame(input_audio_frame)
except asyncio.CancelledError: except asyncio.CancelledError:
logger.info("Audio input task cancelled") logger.info("Audio input task cancelled")
break break
@@ -390,9 +387,11 @@ class LiveKitInputTransport(BaseInputTransport):
if sample_rate != self._current_sample_rate: if sample_rate != self._current_sample_rate:
self._current_sample_rate = sample_rate self._current_sample_rate = sample_rate
self._vad_analyzer = VADAnalyzer( if self._params.vad_enabled:
sample_rate=self._current_sample_rate, num_channels=self._params.audio_in_channels self._vad_analyzer = VADAnalyzer(
) sample_rate=self._current_sample_rate,
num_channels=self._params.audio_in_channels,
)
return AudioRawFrame( return AudioRawFrame(
audio=audio_data.tobytes(), audio=audio_data.tobytes(),
@@ -445,11 +444,11 @@ class LiveKitOutputTransport(BaseOutputTransport):
if isinstance(d, TTFBMetricsData): if isinstance(d, TTFBMetricsData):
if "ttfb" not in metrics: if "ttfb" not in metrics:
metrics["ttfb"] = [] metrics["ttfb"] = []
metrics["ttfb"].append(d.model_dump()) metrics["ttfb"].append(d.model_dump(exclude_none=True))
elif isinstance(d, ProcessingMetricsData): elif isinstance(d, ProcessingMetricsData):
if "processing" not in metrics: if "processing" not in metrics:
metrics["processing"] = [] metrics["processing"] = []
metrics["processing"].append(d.model_dump()) metrics["processing"].append(d.model_dump(exclude_none=True))
elif isinstance(d, LLMUsageMetricsData): elif isinstance(d, LLMUsageMetricsData):
if "tokens" not in metrics: if "tokens" not in metrics:
metrics["tokens"] = [] metrics["tokens"] = []
@@ -457,7 +456,7 @@ class LiveKitOutputTransport(BaseOutputTransport):
elif isinstance(d, TTSUsageMetricsData): elif isinstance(d, TTSUsageMetricsData):
if "characters" not in metrics: if "characters" not in metrics:
metrics["characters"] = [] metrics["characters"] = []
metrics["characters"].append(d.model_dump()) metrics["characters"].append(d.model_dump(exclude_none=True))
message = LiveKitTransportMessageFrame( message = LiveKitTransportMessageFrame(
message={"type": "pipecat-metrics", "metrics": metrics} message={"type": "pipecat-metrics", "metrics": metrics}
@@ -494,13 +493,20 @@ class LiveKitTransport(BaseTransport):
): ):
super().__init__(input_name=input_name, output_name=output_name, loop=loop) super().__init__(input_name=input_name, output_name=output_name, loop=loop)
self._url = url callbacks = LiveKitCallbacks(
self._token = token on_connected=self._on_connected,
self._room_name = room_name on_disconnected=self._on_disconnected,
on_participant_connected=self._on_participant_connected,
on_participant_disconnected=self._on_participant_disconnected,
on_audio_track_subscribed=self._on_audio_track_subscribed,
on_audio_track_unsubscribed=self._on_audio_track_unsubscribed,
on_data_received=self._on_data_received,
on_first_participant_joined=self._on_first_participant_joined,
)
self._params = params self._params = params
self._client = LiveKitTransportClient( self._client = LiveKitTransportClient(
url, token, room_name, self._params, self._create_callbacks(), self._loop url, token, room_name, self._params, callbacks, self._loop
) )
self._input: LiveKitInputTransport | None = None self._input: LiveKitInputTransport | None = None
self._output: LiveKitOutputTransport | None = None self._output: LiveKitOutputTransport | None = None
@@ -516,24 +522,12 @@ class LiveKitTransport(BaseTransport):
self._register_event_handler("on_participant_left") self._register_event_handler("on_participant_left")
self._register_event_handler("on_call_state_updated") self._register_event_handler("on_call_state_updated")
def _create_callbacks(self) -> LiveKitCallbacks: def input(self) -> LiveKitInputTransport:
return LiveKitCallbacks(
on_connected=self._on_connected,
on_disconnected=self._on_disconnected,
on_participant_connected=self._on_participant_connected,
on_participant_disconnected=self._on_participant_disconnected,
on_audio_track_subscribed=self._on_audio_track_subscribed,
on_audio_track_unsubscribed=self._on_audio_track_unsubscribed,
on_data_received=self._on_data_received,
on_first_participant_joined=self._on_first_participant_joined,
)
def input(self) -> FrameProcessor:
if not self._input: if not self._input:
self._input = LiveKitInputTransport(self._client, self._params, name=self._input_name) self._input = LiveKitInputTransport(self._client, self._params, name=self._input_name)
return self._input return self._input
def output(self) -> FrameProcessor: def output(self) -> LiveKitOutputTransport:
if not self._output: if not self._output:
self._output = LiveKitOutputTransport( self._output = LiveKitOutputTransport(
self._client, self._params, name=self._output_name self._client, self._params, name=self._output_name
@@ -544,7 +538,7 @@ class LiveKitTransport(BaseTransport):
def participant_id(self) -> str: def participant_id(self) -> str:
return self._client.participant_id return self._client.participant_id
async def send_audio(self, frame: AudioRawFrame): async def send_audio(self, frame: OutputAudioRawFrame):
if self._output: if self._output:
await self._output.process_frame(frame, FrameDirection.DOWNSTREAM) await self._output.process_frame(frame, FrameDirection.DOWNSTREAM)