Update README and streaming_turn_model.py to implement half-open interval ownership for fixed-grid AV frames. Adjust verification parameters to 1 for testing, and enhance documentation for clarity on audio slicing and frame selection. Add runtime checks to prevent overlapping temporal coordinates in open-stream intervals.

This commit is contained in:
Xin Wang
2026-08-17 22:45:17 +08:00
parent bde72487bb
commit 5ee371f0ae
2 changed files with 51 additions and 14 deletions

View File

@@ -167,7 +167,13 @@ class FixedGridAV:
)
def frames(self, start: float, end: float) -> list[Any]:
"""Return fixed-grid frames whose timestamps lie in [start, end]."""
"""Return fixed-grid frames whose timestamps lie in ``[start, end)``.
Half-open ownership is essential at native block boundaries. For
example, the frame at exactly 2.0 s belongs to ``[2.0, 4.0)`` and must
not also be committed in ``[0.0, 2.0)``. Audio slicing below follows
the same convention.
"""
import numpy as np
from PIL import Image
@@ -175,7 +181,7 @@ class FixedGridAV:
first = max(0, int(math.ceil(start * self.fps - 1e-7)))
last = min(
self.num_frames - 1,
int(math.floor(end * self.fps + 1e-7)),
int(math.ceil(end * self.fps - 1e-7)) - 1,
)
if first > last:
# This can occur only when a caller starts between global visual
@@ -1255,11 +1261,32 @@ class OpenStreamReplayExtractor:
manual_audios=manual_audios,
video_fps=self.config.video_fps,
)
return _open_media_pieces(
pieces = _open_media_pieces(
self.model,
inputs,
use_audio_in_video=use_audio_in_video,
)
# A standalone tail is at most one native 2 s block. Its temporal
# TMRoPE coordinates must therefore stay inside one 50-position chunk.
# This catches endpoint-inclusive frame selection (or another
# preprocessing regression) before overlapping coordinates are
# committed to the next block.
duration = interval[1] - interval[0]
if duration <= self.config.native_block_seconds + 1e-6:
temporal_positions = pieces.payload_position_ids[0]
temporal_span = int(
temporal_positions.max().item()
- temporal_positions.min().item()
)
positions_per_chunk = _tmrope_positions_per_chunk(self.model)
if temporal_span >= positions_per_chunk:
raise RuntimeError(
"One open-stream AV interval spans multiple native TMRoPE "
f"chunks: interval={interval}, temporal_span={temporal_span}, "
f"chunk_size={positions_per_chunk}. Check frame-boundary "
"ownership and audio preprocessing."
)
return pieces
def state_from_scratch(
self,
@@ -1812,6 +1839,11 @@ def main() -> None:
if fixed_grid_av is not None
else "qwen_utility_adaptive_sampling"
),
"media_interval_policy": (
"half_open_[start,end)"
if fixed_grid_av is not None
else "qwen_utility_defined"
),
"open_stream_policy": (
{
"commit": "completed_2s_av_payload_only",