diff --git a/README.md b/README.md index 3e71cc4..1bcfecd 100644 --- a/README.md +++ b/README.md @@ -30,17 +30,39 @@ classification head. | File | Status | Purpose | |---|---|---| | `smoke_test.py` | Verified on one NVIDIA L20 | Text and 12-second EasyCom audio-video inference | -| `easycom_manifest.py` | Implemented, awaiting dataset-side run | Discover one-minute clips, preserve session timecodes, and check 5 Hz targets | +| `easycom_manifest.py` | Verified on EasyCom | Discover one-minute clips, preserve session timecodes, and check 5 Hz targets | | `qwen_omni_features.py` | Implemented, not executed here | Load only the 3B Thinker and extract fused states from one AV window | -| `streaming_turn_model.py` | Implemented | Produce one causal Omni state every 200 ms, with causal-prefix and open-stream modes | -| `TurnPredictionHead` | Defined, untrained | Map `[2048]` to `[10, 3]` logits | -| `train_turn_head.py` | Implemented | Train the frozen-Omni anticipation head with the official EasyCom split | -| `infer_turn_head.py` | Implemented | Save EgoSpeak-format predictions and reproduce per-horizon AP/mAP | +| `streaming_turn_model.py` | Verified on EasyCom | Produce one causal Omni state every 200 ms, with causal-prefix and open-stream modes | +| `TurnPredictionHead` | Trained and evaluated | Map `[2048]` to `[10, 3]` logits | +| `train_turn_head.py` | Verified on EasyCom | Train the frozen-Omni anticipation head with the official EasyCom split | +| `infer_turn_head.py` | Verified on EasyCom | Save EgoSpeak-format predictions and reproduce per-horizon AP/mAP | -The official full model, Thinker input path, embedded audio, video decoding, -and FlashAttention-2 have passed a 12-second EasyCom smoke test on one NVIDIA -L20. Streaming extraction, cache equivalence, target alignment, training, and -paper metrics remain unverified. +The full Thinker input path, embedded audio, direct video decoding, +FlashAttention-2, 5 Hz streaming extraction, cache replay, target alignment, +head training, and EgoSpeak-compatible evaluation have now been exercised on +EasyCom. Ego4D feature extraction and multi-seed EasyCom evaluation remain in +progress. + +## Current EasyCom results + +The current single-run comparison was recorded on 2026-08-19. Both modes use +the frozen Qwen2.5-Omni-3B Thinker, 2 FPS video, 5 Hz predictions, a 20-second +causal context budget, the same `10 x 3` head architecture, and EgoSpeak's +official EasyCom train/test split. + +| Representation | EgoSpeak full-timeline mean mAP | Strict causal mean mAP | +|---|---:|---:| +| Open-stream replay | 58.316 | 58.326 | +| Causal-prefix | 58.830 | 58.841 | + +Causal-prefix is 0.514 mAP points higher in the exact Table 2 protocol, but +the difference is concentrated at 0.2--0.6 seconds. Open-stream is slightly +higher at 1.6--2.0 seconds. These are not yet final paper values because the +head-training variance has not been measured across multiple seeds. + +The full per-horizon and per-class snapshot, machine-readable CSV, provenance, +and follow-up checks are in +[`results/easycom_open_stream_vs_causal_prefix_2026-08-19/`](results/easycom_open_stream_vs_causal_prefix_2026-08-19/README.md). ## Why Omni-only instead of Omni + RNN diff --git a/results/easycom_open_stream_vs_causal_prefix_2026-08-19/README.md b/results/easycom_open_stream_vs_causal_prefix_2026-08-19/README.md new file mode 100644 index 0000000..6e3857c --- /dev/null +++ b/results/easycom_open_stream_vs_causal_prefix_2026-08-19/README.md @@ -0,0 +1,86 @@ +# EasyCom: open-stream replay versus causal-prefix + +Recorded on **2026-08-19**. These are the current single-run results reported +from the remote EasyCom experiments. They are an experiment snapshot, not yet +the final multi-seed paper numbers. + +## Experiment contract + +- Dataset: EasyCom using EgoSpeak's official split. +- Train sessions: 7, 11, 10, 6, 9, 4, 8, 5, and 12. +- Test sessions: 3, 2, and 1. +- Model: frozen Qwen2.5-Omni-3B Thinker plus the trainable `10 x 3` head. +- Prediction rate: 5 Hz, one state every 0.2 seconds. +- Video sampling: 2 FPS. +- Causal context budget: 20 seconds. +- Targets: ten future horizons from 0.2 through 2.0 seconds. +- Metric: frame AP for all three classes, followed by per-horizon mAP and the + mean over all ten horizons. + +The two representations are trained and evaluated separately. A head trained +on one representation must not be reused on the other. + +## Summary + +All values below are percentages. + +| Representation | EgoSpeak full-timeline mean mAP | Strict causal mean mAP | Difference from open stream | +|---|---:|---:|---:| +| Open-stream replay | 58.316 | 58.326 | — | +| Causal-prefix | 58.830 | 58.841 | +0.514 / +0.515 | + +For context, the EgoSpeak Table 2 EasyCom reference values currently recorded +in the project README are 58.7 for Transformer A+V and 60.6 for GRU A+V. +Accordingly, causal-prefix is 0.130 points above the Transformer reference, +whereas open-stream replay is 0.384 points below it. Neither reaches the GRU +A+V result. + +## Full-timeline result by horizon + +| Horizon | Time | Open-stream mAP | Causal-prefix mAP | Causal minus open | +|---:|---:|---:|---:|---:| +| 1 | 0.2 s | 69.756 | 72.149 | +2.393 | +| 2 | 0.4 s | 65.163 | 66.904 | +1.741 | +| 3 | 0.6 s | 61.578 | 62.468 | +0.890 | +| 4 | 0.8 s | 58.783 | 59.214 | +0.431 | +| 5 | 1.0 s | 56.913 | 57.037 | +0.124 | +| 6 | 1.2 s | 55.727 | 55.789 | +0.062 | +| 7 | 1.4 s | 54.819 | 54.845 | +0.026 | +| 8 | 1.6 s | 53.957 | 53.905 | -0.052 | +| 9 | 1.8 s | 53.459 | 53.265 | -0.194 | +| 10 | 2.0 s | 53.002 | 52.723 | -0.279 | + +The complete per-class AP values for both evaluation protocols are stored in +[`metrics.csv`](metrics.csv). [`summary.json`](summary.json) contains the +machine-readable aggregate comparison. + +## Current interpretation + +1. Causal-prefix is 0.514 mAP points better overall in the exact EgoSpeak + full-timeline protocol. +2. Almost all of that advantage occurs at 0.2--0.6 seconds. At 1.2--1.4 + seconds the difference is negligible, and open-stream is slightly better + at 1.6--2.0 seconds. +3. The full-timeline and strict-causal means differ by only 0.010--0.011 + points, so unavailable pre-session rows are not driving the comparison. +4. The short-horizon pattern is consistent with block-local audio processing, + partial-block AV/TMRoPE organization, or other 2-second boundary effects + weakening precise speech-onset/offset cues. It does not look like a general + loss of longer-horizon conversational information. +5. Previous smoke tests showed exact open-stream cache replay but a lower + cosine against the continuous-window representation. This supports a real + representation-semantics difference rather than a cache-copy error. + +## Required follow-up before publication + +- Repeat head training with at least three identical seeds for both modes and + report mean and standard deviation. +- Group feature cosine and AP/mAP by the ten 0.2-second phases inside each + 2-second block to test for a block-boundary effect. +- Compare samples before and after the 20-second memory budget fills to isolate + input segmentation from rolling-memory behavior. +- Archive the original remote `metrics.json`, checkpoint configuration, git + commit, Transformers version, and Qwen model revision alongside the final + paper results. The numbers in this snapshot were transcribed from the + reported console tables. + diff --git a/results/easycom_open_stream_vs_causal_prefix_2026-08-19/metrics.csv b/results/easycom_open_stream_vs_causal_prefix_2026-08-19/metrics.csv new file mode 100644 index 0000000..a486b39 --- /dev/null +++ b/results/easycom_open_stream_vs_causal_prefix_2026-08-19/metrics.csv @@ -0,0 +1,42 @@ +representation,protocol,horizon,seconds,background_ap_percent,camera_wearer_ap_percent,other_people_ap_percent,map_percent +open_stream_replay,egospeak_full_timeline,1,0.2,53.605,74.331,81.331,69.756 +open_stream_replay,egospeak_full_timeline,2,0.4,47.066,69.402,79.020,65.163 +open_stream_replay,egospeak_full_timeline,3,0.6,42.615,64.600,77.520,61.578 +open_stream_replay,egospeak_full_timeline,4,0.8,39.463,60.641,76.244,58.783 +open_stream_replay,egospeak_full_timeline,5,1.0,38.002,57.371,75.367,56.913 +open_stream_replay,egospeak_full_timeline,6,1.2,37.398,55.258,74.523,55.727 +open_stream_replay,egospeak_full_timeline,7,1.4,36.933,53.400,74.126,54.819 +open_stream_replay,egospeak_full_timeline,8,1.6,36.653,51.502,73.716,53.957 +open_stream_replay,egospeak_full_timeline,9,1.8,36.527,50.346,73.504,53.459 +open_stream_replay,egospeak_full_timeline,10,2.0,36.673,49.089,73.246,53.002 +open_stream_replay,causal_valid_only,1,0.2,53.605,74.331,81.336,69.757 +open_stream_replay,causal_valid_only,2,0.4,47.066,69.402,79.029,65.166 +open_stream_replay,causal_valid_only,3,0.6,42.615,64.607,77.532,61.585 +open_stream_replay,causal_valid_only,4,0.8,39.463,60.652,76.259,58.792 +open_stream_replay,causal_valid_only,5,1.0,38.002,57.386,75.385,56.924 +open_stream_replay,causal_valid_only,6,1.2,37.398,55.277,74.543,55.739 +open_stream_replay,causal_valid_only,7,1.4,36.933,53.421,74.148,54.834 +open_stream_replay,causal_valid_only,8,1.6,36.656,51.522,73.740,53.973 +open_stream_replay,causal_valid_only,9,1.8,36.529,50.365,73.530,53.475 +open_stream_replay,causal_valid_only,10,2.0,36.676,49.107,73.275,53.019 +causal_prefix,egospeak_full_timeline,1,0.2,56.402,75.843,84.203,72.149 +causal_prefix,egospeak_full_timeline,2,0.4,48.713,70.695,81.305,66.904 +causal_prefix,egospeak_full_timeline,3,0.6,43.007,65.451,78.946,62.468 +causal_prefix,egospeak_full_timeline,4,0.8,39.482,61.051,77.107,59.214 +causal_prefix,egospeak_full_timeline,5,1.0,37.773,57.423,75.915,57.037 +causal_prefix,egospeak_full_timeline,6,1.2,37.197,54.929,75.241,55.789 +causal_prefix,egospeak_full_timeline,7,1.4,36.884,53.060,74.590,54.845 +causal_prefix,egospeak_full_timeline,8,1.6,36.514,50.916,74.283,53.905 +causal_prefix,egospeak_full_timeline,9,1.8,36.424,49.471,73.901,53.265 +causal_prefix,egospeak_full_timeline,10,2.0,36.095,48.407,73.667,52.723 +causal_prefix,causal_valid_only,1,0.2,56.402,75.843,84.209,72.151 +causal_prefix,causal_valid_only,2,0.4,48.713,70.695,81.315,66.908 +causal_prefix,causal_valid_only,3,0.6,43.007,65.458,78.960,62.475 +causal_prefix,causal_valid_only,4,0.8,39.482,61.062,77.123,59.223 +causal_prefix,causal_valid_only,5,1.0,37.773,57.438,75.934,57.048 +causal_prefix,causal_valid_only,6,1.2,37.197,54.947,75.262,55.802 +causal_prefix,causal_valid_only,7,1.4,36.884,53.081,74.612,54.859 +causal_prefix,causal_valid_only,8,1.6,36.517,50.936,74.308,53.920 +causal_prefix,causal_valid_only,9,1.8,36.426,49.489,73.928,53.281 +causal_prefix,causal_valid_only,10,2.0,36.098,48.425,73.697,52.740 + diff --git a/results/easycom_open_stream_vs_causal_prefix_2026-08-19/summary.json b/results/easycom_open_stream_vs_causal_prefix_2026-08-19/summary.json new file mode 100644 index 0000000..ad112f4 --- /dev/null +++ b/results/easycom_open_stream_vs_causal_prefix_2026-08-19/summary.json @@ -0,0 +1,74 @@ +{ + "schema_version": 1, + "recorded_date": "2026-08-19", + "status": "single_run_snapshot", + "dataset": "EasyCom", + "split": { + "train_sessions": [7, 11, 10, 6, 9, 4, 8, 5, 12], + "test_sessions": [3, 2, 1] + }, + "model": "Qwen2.5-Omni-3B Thinker", + "head": "MLP 2048 -> 1024 -> 30, reshaped to 10 x 3", + "prediction_step_seconds": 0.2, + "video_fps": 2, + "memory_seconds": 20, + "metrics_percent": { + "open_stream_replay": { + "egospeak_full_timeline_mean_map": 58.316, + "causal_valid_only_mean_map": 58.326, + "egospeak_full_timeline_horizon_map": [ + 69.756, + 65.163, + 61.578, + 58.783, + 56.913, + 55.727, + 54.819, + 53.957, + 53.459, + 53.002 + ] + }, + "causal_prefix": { + "egospeak_full_timeline_mean_map": 58.830, + "causal_valid_only_mean_map": 58.841, + "egospeak_full_timeline_horizon_map": [ + 72.149, + 66.904, + 62.468, + 59.214, + 57.037, + 55.789, + 54.845, + 53.905, + 53.265, + 52.723 + ] + }, + "causal_prefix_minus_open_stream": { + "egospeak_full_timeline_mean_map": 0.514, + "causal_valid_only_mean_map": 0.515, + "egospeak_full_timeline_horizon_map": [ + 2.393, + 1.741, + 0.890, + 0.431, + 0.124, + 0.062, + 0.026, + -0.052, + -0.194, + -0.279 + ] + } + }, + "egospeak_table2_reference_percent": { + "easycom_transformer_audio_video_mean_map": 58.7, + "easycom_gru_audio_video_mean_map": 60.6 + }, + "provenance": { + "source": "User-reported remote infer_turn_head.py console tables", + "raw_metrics_json_archived_here": false, + "multi_seed": false + } +}