Fixed SmallWebRTCTransport to support dynamic chunk values.
This commit is contained in:
@@ -18,6 +18,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
|
- Fixed `SmallWebRTCTransport` to support dynamic values for
|
||||||
|
`TransportParams.audio_out_10ms_chunks`. Previously, it only worked with 20ms
|
||||||
|
chunks.
|
||||||
|
|
||||||
- Fixed an issue where `LLMAssistantContextAggregator` would prevent a
|
- Fixed an issue where `LLMAssistantContextAggregator` would prevent a
|
||||||
`BotStoppedSpeakingFrame` from moving through the pipeline.
|
`BotStoppedSpeakingFrame` from moving through the pipeline.
|
||||||
|
|
||||||
@@ -26,8 +30,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
### Added
|
### Added
|
||||||
|
|
||||||
- Added `TransportParams.audio_out_10ms_chunks` parameter to allow controlling
|
- Added `TransportParams.audio_out_10ms_chunks` parameter to allow controlling
|
||||||
the amount of audio being sent by the output transport. It defaults to 2, so
|
the amount of audio being sent by the output transport. It defaults to 4, so
|
||||||
20ms audio chunks are sent.
|
40ms audio chunks are sent.
|
||||||
|
|
||||||
- Added `QwenLLMService` for Qwen integration with an OpenAI-compatible
|
- Added `QwenLLMService` for Qwen integration with an OpenAI-compatible
|
||||||
interface. Added foundational example `14q-function-calling-qwen.py`.
|
interface. Added foundational example `14q-function-calling-qwen.py`.
|
||||||
|
|||||||
@@ -51,19 +51,28 @@ class RawAudioTrack(AudioStreamTrack):
|
|||||||
def __init__(self, sample_rate):
|
def __init__(self, sample_rate):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._sample_rate = sample_rate
|
self._sample_rate = sample_rate
|
||||||
self._samples_per_frame = self._sample_rate // 50 # 20ms per frame
|
self._samples_per_10ms = sample_rate * 10 // 1000
|
||||||
|
self._bytes_per_10ms = self._samples_per_10ms * 2 # 16-bit (2 bytes per sample)
|
||||||
self._timestamp = 0
|
self._timestamp = 0
|
||||||
self._audio_buffer = deque()
|
|
||||||
self._start = time.time()
|
self._start = time.time()
|
||||||
|
# Queue of (bytes, future), broken into 10ms sub chunks as needed
|
||||||
|
self._chunk_queue = deque()
|
||||||
|
|
||||||
def add_audio_bytes(self, audio_bytes: bytes):
|
def add_audio_bytes(self, audio_bytes: bytes):
|
||||||
"""
|
"""
|
||||||
Adds bytes to the audio buffer and returns a Future that completes when the data is processed.
|
Adds bytes to the audio buffer and returns a Future that completes when the data is processed.
|
||||||
"""
|
"""
|
||||||
if len(audio_bytes) % 2 != 0:
|
if len(audio_bytes) % self._bytes_per_10ms != 0:
|
||||||
raise ValueError("Audio bytes length must be even (16-bit samples).")
|
raise ValueError("Audio bytes must be a multiple of 10ms size.")
|
||||||
future = asyncio.get_running_loop().create_future()
|
future = asyncio.get_running_loop().create_future()
|
||||||
self._audio_buffer.append((audio_bytes, future))
|
|
||||||
|
# Break input into 10ms chunks
|
||||||
|
for i in range(0, len(audio_bytes), self._bytes_per_10ms):
|
||||||
|
chunk = audio_bytes[i : i + self._bytes_per_10ms]
|
||||||
|
# Only the last chunk carries the future to be resolved once fully consumed
|
||||||
|
fut = future if i + self._bytes_per_10ms >= len(audio_bytes) else None
|
||||||
|
self._chunk_queue.append((chunk, fut))
|
||||||
|
|
||||||
return future
|
return future
|
||||||
|
|
||||||
async def recv(self):
|
async def recv(self):
|
||||||
@@ -76,36 +85,22 @@ class RawAudioTrack(AudioStreamTrack):
|
|||||||
if wait > 0:
|
if wait > 0:
|
||||||
await asyncio.sleep(wait)
|
await asyncio.sleep(wait)
|
||||||
|
|
||||||
# Check if we have enough data
|
if self._chunk_queue:
|
||||||
needed_bytes = self._samples_per_frame * 2 # 16-bit (2 bytes per sample)
|
chunk, future = self._chunk_queue.popleft()
|
||||||
available_bytes = sum(len(audio_bytes) for audio_bytes, _ in self._audio_buffer)
|
if future and not future.done():
|
||||||
consumed_futures = [] # Track futures for processed data
|
future.set_result(True)
|
||||||
if available_bytes >= needed_bytes:
|
|
||||||
# Extract data from deque
|
|
||||||
chunk = bytearray()
|
|
||||||
while len(chunk) < needed_bytes:
|
|
||||||
audio_bytes, future = self._audio_buffer.popleft()
|
|
||||||
chunk.extend(audio_bytes)
|
|
||||||
consumed_futures.append(future) # Track the future
|
|
||||||
chunk = bytes(chunk[:needed_bytes]) # Trim excess bytes
|
|
||||||
else:
|
else:
|
||||||
chunk = bytes(needed_bytes) # Generate silent frame
|
chunk = bytes(self._bytes_per_10ms) # silence
|
||||||
|
|
||||||
# Convert the byte data to an ndarray of int16 samples
|
# Convert the byte data to an ndarray of int16 samples
|
||||||
samples = np.frombuffer(chunk, dtype=np.int16)
|
samples = np.frombuffer(chunk, dtype=np.int16)
|
||||||
|
|
||||||
# Create AudioFrame
|
# Create AudioFrame
|
||||||
frame = AudioFrame.from_ndarray(samples[None, :], layout="mono")
|
frame = AudioFrame.from_ndarray(samples[None, :], layout="mono")
|
||||||
self._timestamp += self._samples_per_frame
|
|
||||||
frame.pts = self._timestamp
|
|
||||||
frame.sample_rate = self._sample_rate
|
frame.sample_rate = self._sample_rate
|
||||||
|
frame.pts = self._timestamp
|
||||||
frame.time_base = fractions.Fraction(1, self._sample_rate)
|
frame.time_base = fractions.Fraction(1, self._sample_rate)
|
||||||
|
self._timestamp += self._samples_per_10ms
|
||||||
# Resolve all futures corresponding to consumed data
|
|
||||||
for future in consumed_futures:
|
|
||||||
if not future.done():
|
|
||||||
future.set_result(True)
|
|
||||||
|
|
||||||
return frame
|
return frame
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user