290 lines
11 KiB
Python
290 lines
11 KiB
Python
import array
|
|
import asyncio
|
|
import datetime
|
|
import os
|
|
import signal
|
|
import wave
|
|
|
|
from daily import (
|
|
AudioData,
|
|
CallClient,
|
|
CustomAudioSource,
|
|
CustomAudioTrack,
|
|
Daily,
|
|
EventHandler,
|
|
)
|
|
from dotenv import load_dotenv
|
|
from loguru import logger
|
|
|
|
load_dotenv(override=True)
|
|
|
|
# Pipecat sends audio at this true content rate but declares it as
|
|
# DECLARED_SAMPLE_RATE to write_frames(), which makes delivery faster than
|
|
# real-time. We receive at the declared rate (no resampling) and play back at
|
|
# the true rate so the avatar consumes audio at normal speed.
|
|
TRUE_SAMPLE_RATE = 24000
|
|
DECLARED_SAMPLE_RATE = 48000
|
|
SPEEDUP = DECLARED_SAMPLE_RATE // TRUE_SAMPLE_RATE
|
|
CHUNK_BYTES = int(TRUE_SAMPLE_RATE * 20 / 1000) * 2 # 20 ms, 16-bit mono
|
|
MIN_AUDIO_BUFFER = CHUNK_BYTES * 5 # 100 ms pre-buffer
|
|
|
|
|
|
def completion_callback(future):
|
|
def _callback(*args):
|
|
def set_result(future, *args):
|
|
try:
|
|
if len(args) > 1:
|
|
future.set_result(args)
|
|
else:
|
|
future.set_result(*args)
|
|
except asyncio.InvalidStateError:
|
|
pass
|
|
|
|
future.get_loop().call_soon_threadsafe(set_result, future, *args)
|
|
|
|
return _callback
|
|
|
|
|
|
class DailyProxyApp(EventHandler):
|
|
# This is necessary to override EventHandler's __new__ method.
|
|
def __new__(cls, *args, **kwargs):
|
|
return super().__new__(cls)
|
|
|
|
def __init__(self):
|
|
super().__init__()
|
|
self._loop = asyncio.new_event_loop()
|
|
# Raw PCM buffer — filled at DECLARED_SAMPLE_RATE speed, drained at TRUE_SAMPLE_RATE speed.
|
|
self._buffer = bytearray()
|
|
self._audio_task: asyncio.Task | None = None
|
|
self._wav_file: wave.Wave_write | None = None
|
|
|
|
self._client: CallClient = CallClient(event_handler=self)
|
|
self._client.update_subscription_profiles(
|
|
{"base": {"camera": "unsubscribed", "microphone": "subscribed"}}
|
|
)
|
|
|
|
# Playback source declared at TRUE_SAMPLE_RATE — consumes audio at real-time speed.
|
|
self._audio_source = CustomAudioSource(TRUE_SAMPLE_RATE, 1, False)
|
|
self._audio_track = CustomAudioTrack(self._audio_source)
|
|
|
|
def on_joined(self, data, error):
|
|
logger.debug("Local participant Joined!")
|
|
if error:
|
|
print(f"Unable to join meeting: {error}")
|
|
self._loop.call_soon_threadsafe(self._loop.stop)
|
|
|
|
def _open_wav(self):
|
|
os.makedirs("recordings", exist_ok=True)
|
|
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
path = f"recordings/received_pos_speed_{timestamp}.wav"
|
|
self._wav_file = wave.open(path, "wb")
|
|
self._wav_file.setnchannels(1)
|
|
self._wav_file.setsampwidth(2)
|
|
# Declare TRUE_SAMPLE_RATE so timestamps match bot_*.wav for comparison.
|
|
# Bytes arrive at DECLARED_SAMPLE_RATE speed (2x real-time) but each byte
|
|
# is 24kHz content, so the WAV plays back at normal speed.
|
|
self._wav_file.setframerate(TRUE_SAMPLE_RATE)
|
|
logger.info(f"Recording received audio to {path}")
|
|
|
|
def _close_wav(self):
|
|
if self._wav_file:
|
|
self._wav_file.close()
|
|
self._wav_file = None
|
|
|
|
def run(self, meeting_url: str):
|
|
asyncio.set_event_loop(self._loop)
|
|
self._open_wav()
|
|
self._create_audio_task()
|
|
|
|
def handle_exit():
|
|
logger.info("Ctrl+C pressed. Leaving the meeting...")
|
|
self._loop.call_soon_threadsafe(self._loop.stop)
|
|
|
|
for sig in (signal.SIGINT, signal.SIGTERM):
|
|
self._loop.add_signal_handler(sig, handle_exit)
|
|
|
|
self._client.set_user_name("TestTavusTransport")
|
|
self._client.join(
|
|
meeting_url,
|
|
completion=self.on_joined,
|
|
client_settings={
|
|
"inputs": {
|
|
"microphone": {
|
|
"isEnabled": True,
|
|
"settings": {"customTrack": {"id": self._audio_track.id}},
|
|
},
|
|
}
|
|
},
|
|
)
|
|
|
|
try:
|
|
self._loop.run_forever()
|
|
finally:
|
|
self.leave()
|
|
|
|
def leave(self):
|
|
if self._audio_task:
|
|
self._loop.run_until_complete(self._cancel_audio_task())
|
|
|
|
self._close_wav()
|
|
self._client.leave()
|
|
self._client.release()
|
|
|
|
async def update_subscriptions(self, participant_settings=None, profile_settings=None):
|
|
logger.info(f"Updating subscriptions participant_settings: {participant_settings}")
|
|
future = asyncio.get_running_loop().create_future()
|
|
self._client.update_subscriptions(
|
|
participant_settings=participant_settings,
|
|
profile_settings=profile_settings,
|
|
completion=completion_callback(future),
|
|
)
|
|
await future
|
|
|
|
def _create_audio_task(self):
|
|
if not self._audio_task:
|
|
self._audio_task = self._loop.create_task(self._audio_task_handler())
|
|
|
|
async def _cancel_audio_task(self):
|
|
if self._audio_task:
|
|
self._audio_task.cancel()
|
|
try:
|
|
await self._audio_task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
self._audio_task = None
|
|
|
|
async def capture_participant_audio(self, participant_id: str):
|
|
logger.info(f"Capturing participant audio: {participant_id}")
|
|
audio_source: str = "stream"
|
|
media = {"media": {"customAudio": {audio_source: "subscribed"}}}
|
|
await self.update_subscriptions(participant_settings={participant_id: media})
|
|
|
|
# Must match the declared rate Pipecat used so WebRTC skips resampling —
|
|
# every original byte arrives intact.
|
|
self._client.set_audio_renderer(
|
|
participant_id,
|
|
self._audio_data_received,
|
|
audio_source=audio_source,
|
|
sample_rate=DECLARED_SAMPLE_RATE,
|
|
callback_interval_ms=20,
|
|
)
|
|
logger.info(
|
|
f"Receiving at declared_rate={DECLARED_SAMPLE_RATE} Hz "
|
|
f"(true content: {TRUE_SAMPLE_RATE} Hz, ~{SPEEDUP}x faster than real-time)"
|
|
)
|
|
|
|
@staticmethod
|
|
def _is_silence(data: bytes, threshold: int = 5) -> bool:
|
|
# Interpret as 16-bit signed PCM samples and check peak amplitude.
|
|
# WebRTC-injected silence is all zeros; real TTS audio has non-trivial
|
|
# amplitude. This lets us skip buffering frames that Pipecat never wrote,
|
|
# so the buffer only grows when actual speech arrives (via our trick).
|
|
samples = array.array("h", data)
|
|
return max(abs(s) for s in samples) < threshold
|
|
|
|
async def _buffer_audio(self, audio_data: AudioData):
|
|
"""Append received bytes to the buffer, skipping WebRTC-injected silence.
|
|
|
|
Speech frames arrive at DECLARED_SAMPLE_RATE speed (~2x real-time) so the
|
|
buffer grows ahead of the drain. WebRTC-injected silence (all-zero PCM) is
|
|
handled differently based on buffer level: below MIN_AUDIO_BUFFER we keep it
|
|
so the pre-buffer can fill; above that threshold we discard it so the buffer
|
|
drains back down between utterances.
|
|
"""
|
|
new_bytes = audio_data.audio_frames
|
|
if self._is_silence(new_bytes):
|
|
if len(self._buffer) < MIN_AUDIO_BUFFER:
|
|
# Below pre-buffer threshold: add silence so the buffer fills up.
|
|
self._buffer.extend(new_bytes)
|
|
# else: buffer is healthy, discard silence so it can drain.
|
|
return
|
|
|
|
self._buffer.extend(new_bytes)
|
|
|
|
def _audio_data_received(self, participant_id: str, audio_data: AudioData, audio_source: str):
|
|
if self._wav_file:
|
|
self._wav_file.writeframes(audio_data.audio_frames)
|
|
asyncio.run_coroutine_threadsafe(self._buffer_audio(audio_data), self._loop)
|
|
|
|
async def _handle_interrupt(self):
|
|
"""Clear the audio buffer, mimicking the avatar stopping mid-speech."""
|
|
dropped = len(self._buffer)
|
|
self._buffer.clear()
|
|
logger.info(
|
|
f"Interrupt received — dropped {dropped}B ({dropped / (TRUE_SAMPLE_RATE * 2):.3f}s) from buffer"
|
|
)
|
|
|
|
#
|
|
# Daily (EventHandler)
|
|
#
|
|
|
|
def on_app_message(self, message, sender):
|
|
if not isinstance(message, dict):
|
|
return
|
|
if message.get("event_type") == "conversation.interrupt":
|
|
asyncio.run_coroutine_threadsafe(self._handle_interrupt(), self._loop)
|
|
|
|
async def _audio_task_handler(self):
|
|
"""Drain the buffer at TRUE_SAMPLE_RATE speed (real-time playback).
|
|
|
|
Waits until min_audio_buffer bytes are accumulated before starting
|
|
playback, then drains freely in chunk_bytes steps. If the buffer runs
|
|
dry it re-enters the waiting state so the next burst also gets the
|
|
pre-buffer delay.
|
|
"""
|
|
buffering = True
|
|
last_log_time = self._loop.time()
|
|
|
|
while True:
|
|
if buffering:
|
|
if len(self._buffer) >= MIN_AUDIO_BUFFER:
|
|
buffering = False
|
|
logger.debug(f"Pre-buffer reached ({MIN_AUDIO_BUFFER}B) — starting playback")
|
|
else:
|
|
await asyncio.sleep(0.001)
|
|
continue
|
|
|
|
if len(self._buffer) >= CHUNK_BYTES:
|
|
chunk = bytes(self._buffer[:CHUNK_BYTES])
|
|
del self._buffer[:CHUNK_BYTES]
|
|
|
|
future = asyncio.get_running_loop().create_future()
|
|
self._audio_source.write_frames(chunk, completion=completion_callback(future))
|
|
await future
|
|
else:
|
|
buffering = True
|
|
await asyncio.sleep(0.001)
|
|
|
|
now = self._loop.time()
|
|
if now - last_log_time >= 1.0:
|
|
buffer_seconds = len(self._buffer) / (TRUE_SAMPLE_RATE * 2)
|
|
if buffer_seconds > 0:
|
|
logger.info(
|
|
f"Buffer status: {len(self._buffer)}B ({buffer_seconds:.3f}s buffered)"
|
|
)
|
|
last_log_time = now
|
|
|
|
def on_participant_joined(self, participant):
|
|
participant_name = participant["info"]["userName"]
|
|
logger.info(f"Participant {participant_name} joined")
|
|
if participant_name != "Pipecat":
|
|
# We are only subscribing for audio from Pipecat.
|
|
return
|
|
asyncio.run_coroutine_threadsafe(
|
|
self.capture_participant_audio(participant_id=participant["id"]), self._loop
|
|
)
|
|
|
|
def on_participant_left(self, participant, reason):
|
|
logger.info(f"Participant {participant['id']} left {reason}")
|
|
|
|
|
|
def main():
|
|
Daily.init()
|
|
room_url = os.environ["TAVUS_SAMPLE_ROOM_URL"]
|
|
app = DailyProxyApp()
|
|
app.run(room_url)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|