wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize overlap working-memory audio preprocessing

Browse Source
This commit is contained in:
Xin Wang
2026-08-24 10:18:55 +08:00
parent 20446c0bd8
commit 1731f2b6f7
3 changed files with 82 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
streaming_turn_model.py
View File

@@ -1672,6 +1672,8 @@ class OverlappingWorkingMemoryExtractor:
working_memory_seconds: float,
fixed_grid_av: FixedGridAV,
working_grid_av: FixedGridAV,
audio_feature_device: str | torch.device | None = None,
audio_right_padding_samples: int | None = None,
) -> None:
config.validate()
if working_memory_seconds <= 0:
@@ -1700,6 +1702,13 @@ class OverlappingWorkingMemoryExtractor:
self.processor = processor
self.config = config
self.working_memory_seconds = float(working_memory_seconds)
if (
audio_right_padding_samples is not None
and audio_right_padding_samples < 0
):
raise ValueError("audio_right_padding_samples must be non-negative")
self.audio_feature_device = audio_feature_device
self.audio_right_padding_samples = audio_right_padding_samples
# The working recap consumes one block from the matched token budget.
self.fixed_block_budget = total_blocks - 1
self.fixed_extractor = OpenStreamReplayExtractor(
@@ -1822,6 +1831,10 @@ class OverlappingWorkingMemoryExtractor:
audio_path=audio_path,
interval=(block_start, block_end),
use_audio_in_video=use_audio_in_video,
audio_feature_device=self.audio_feature_device,
audio_right_padding_samples=(
self.audio_right_padding_samples
),
)
completed_blocks.put(
CachedOpenMediaBlock(
@@ -1874,6 +1887,8 @@ class OverlappingWorkingMemoryExtractor:
audio_path=audio_path,
interval=(working_start, current_time),
use_audio_in_video=use_audio_in_video,
audio_feature_device=self.audio_feature_device,
audio_right_padding_samples=self.audio_right_padding_samples,
)
state = self._state_from_prepared(
video_path=video_path,