diff --git a/README.md b/README.md index 5872967..db0316a 100644 --- a/README.md +++ b/README.md @@ -450,8 +450,8 @@ python streaming_turn_model.py \ --start 0 --end 4 \ --video-fps 2 \ --strict-fixed-video-grid \ - --verify-every 5 \ - --verify-continuous-every 5 \ + --verify-every 1 \ + --verify-continuous-every 1 \ --flash-attention-2 \ --output features/Session_1_open_first_4s.pt ``` @@ -510,10 +510,12 @@ The sidecar selects the matching fixed-FPS MP4 and sets `N` to its exact `--strict-fixed-video-grid` is enabled. In strict A+V mode the extractor does not delegate temporal frame selection to -the generic video utility. It reads the already-fixed MP4 grid explicitly, -uses only frames with timestamps `<= t`, and duplicates the most recent past -frame when the visual tower needs an even frame count. Audio is loaded once at -16 kHz and sliced exactly at each causal boundary. The processor still receives +the generic video utility. It reads the already-fixed MP4 grid explicitly and +assigns both video frames and audio samples with half-open intervals +`[start, end)`. Thus a frame exactly on a two-second boundary belongs only to +the following block. It duplicates the most recent available frame when the +visual tower needs an even frame count. Audio is loaded once at 16 kHz and +sliced at the same causal boundary. The processor still receives `use_audio_in_video=True`, so Qwen's audio/video token interleaving is retained. Before a full session, run only the first four seconds and inspect both check @@ -528,15 +530,17 @@ python streaming_turn_model.py \ --start 0 --end 4 \ --video-fps 2 \ --strict-fixed-video-grid \ - --verify-every 5 \ - --verify-continuous-every 5 \ + --verify-every 1 \ + --verify-continuous-every 1 \ --flash-attention-2 \ --output features/Session_1_first_4s_check.pt ``` -Do not launch all sessions unless `cache_replay` cosine is nearly 1. The -`continuous_window_semantics` cosine answers a different question and may be -lower because media-turn boundaries change the representation. +Do not launch all sessions unless `cache_replay` cosine is nearly 1. For +`open-stream-replay`, a materially lower `continuous_window_semantics` cosine +means that the custom payload stitching is not equivalent to one continuous +Qwen input; it must not be dismissed as a media-turn-boundary effect because +this mode intentionally keeps a single media turn open. Video can be sampled at 2 FPS while predictions remain at 5 Hz. Audio and the causal boundary update every 200 ms; consecutive prediction times may use the @@ -557,6 +561,7 @@ Each `.pt` file contains: - `config`: step, native block, memory, FPS, and pixel settings; - `verification`: cached-versus-exact numerical checks; - `input_video_fps` and `video_sampling_grid`: sampling provenance; +- `media_interval_policy`: fixed-grid AV uses half-open `[start, end)` ownership; - `logits`: optional `[N, 10, 3]` output if a trained head is supplied; - `probabilities`: optional softmax probabilities. diff --git a/streaming_turn_model.py b/streaming_turn_model.py index a822db1..365f13c 100644 --- a/streaming_turn_model.py +++ b/streaming_turn_model.py @@ -167,7 +167,13 @@ class FixedGridAV: ) def frames(self, start: float, end: float) -> list[Any]: - """Return fixed-grid frames whose timestamps lie in [start, end].""" + """Return fixed-grid frames whose timestamps lie in ``[start, end)``. + + Half-open ownership is essential at native block boundaries. For + example, the frame at exactly 2.0 s belongs to ``[2.0, 4.0)`` and must + not also be committed in ``[0.0, 2.0)``. Audio slicing below follows + the same convention. + """ import numpy as np from PIL import Image @@ -175,7 +181,7 @@ class FixedGridAV: first = max(0, int(math.ceil(start * self.fps - 1e-7))) last = min( self.num_frames - 1, - int(math.floor(end * self.fps + 1e-7)), + int(math.ceil(end * self.fps - 1e-7)) - 1, ) if first > last: # This can occur only when a caller starts between global visual @@ -1255,11 +1261,32 @@ class OpenStreamReplayExtractor: manual_audios=manual_audios, video_fps=self.config.video_fps, ) - return _open_media_pieces( + pieces = _open_media_pieces( self.model, inputs, use_audio_in_video=use_audio_in_video, ) + # A standalone tail is at most one native 2 s block. Its temporal + # TMRoPE coordinates must therefore stay inside one 50-position chunk. + # This catches endpoint-inclusive frame selection (or another + # preprocessing regression) before overlapping coordinates are + # committed to the next block. + duration = interval[1] - interval[0] + if duration <= self.config.native_block_seconds + 1e-6: + temporal_positions = pieces.payload_position_ids[0] + temporal_span = int( + temporal_positions.max().item() + - temporal_positions.min().item() + ) + positions_per_chunk = _tmrope_positions_per_chunk(self.model) + if temporal_span >= positions_per_chunk: + raise RuntimeError( + "One open-stream AV interval spans multiple native TMRoPE " + f"chunks: interval={interval}, temporal_span={temporal_span}, " + f"chunk_size={positions_per_chunk}. Check frame-boundary " + "ownership and audio preprocessing." + ) + return pieces def state_from_scratch( self, @@ -1812,6 +1839,11 @@ def main() -> None: if fixed_grid_av is not None else "qwen_utility_adaptive_sampling" ), + "media_interval_policy": ( + "half_open_[start,end)" + if fixed_grid_av is not None + else "qwen_utility_defined" + ), "open_stream_policy": ( { "commit": "completed_2s_av_payload_only",