From 4b7d2c0c204a3cd703cd3498e53fb42c8f8f2cf9 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Thu, 10 Sep 2026 11:12:02 +0800 Subject: [PATCH] Add competing-risk evaluation and align ICASSP story --- PROJECT_CONTEXT.md | 26 +- README.md | 83 +- docs/README.md | 1 + ...easycom_competing_risk_floor_evaluation.md | 109 +++ docs/easycom_muvap_stage3_baseline.md | 9 +- docs/easycom_oracle_information.md | 6 +- ...ssp2027_acceptance_storyline_2026-09-10.md | 30 + ...peaker_floor_experiment_plan_2026-08-30.md | 5 +- evaluate_floor_state_changes.py | 886 ++++++++++++++++-- manuscript/main/README.md | 48 +- manuscript/main/REVIEWER_CHECKLIST.md | 110 +-- manuscript/main/main.tex | 787 ++++++---------- ...omni_turn_research_direction_2026-08-26.md | 353 +++++++ ..._and_future_floor_projection_2026-08-27.md | 280 ++++++ results/README.md | 1 + .../README.md | 159 ++++ .../summary.json | 93 ++ .../README.md | 84 ++ .../summary.json | 76 ++ .../README.md | 121 +++ .../console_transcription.txt | 41 + .../metrics.csv | 31 + .../summary.json | 73 ++ .../README.md | 71 ++ .../metrics.csv | 19 + .../README.md | 123 +++ .../metrics.xlsx | Bin 0 -> 9181 bytes .../summary.json | 121 +++ run_easycom_oracle_information.sh | 12 +- test_evaluate_floor_state_changes.py | 386 ++++++++ 30 files changed, 3420 insertions(+), 724 deletions(-) create mode 100644 docs/easycom_competing_risk_floor_evaluation.md create mode 100644 reports/omni_turn_research_direction_2026-08-26.md create mode 100644 reports/roma_negative_transfer_and_future_floor_projection_2026-08-27.md create mode 100644 results/easycom_dense_last8_qvo_frozen_head_epoch3_2026-08-25/README.md create mode 100644 results/easycom_dense_last8_qvo_frozen_head_epoch3_2026-08-25/summary.json create mode 100644 results/easycom_dense_last8_qvo_full_train_epoch1_2026-08-25/README.md create mode 100644 results/easycom_dense_last8_qvo_full_train_epoch1_2026-08-25/summary.json create mode 100644 results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/README.md create mode 100644 results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/console_transcription.txt create mode 100644 results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/metrics.csv create mode 100644 results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/summary.json create mode 100644 results/easycom_oracle_pooled_wearer_audit_2026-09-10/README.md create mode 100644 results/easycom_oracle_pooled_wearer_audit_2026-09-10/metrics.csv create mode 100644 results/easycom_roma_frozen_last4_2026-08-27/README.md create mode 100644 results/easycom_roma_frozen_last4_2026-08-27/metrics.xlsx create mode 100644 results/easycom_roma_frozen_last4_2026-08-27/summary.json diff --git a/PROJECT_CONTEXT.md b/PROJECT_CONTEXT.md index 4ce8dc5..8d8fb33 100644 --- a/PROJECT_CONTEXT.md +++ b/PROJECT_CONTEXT.md @@ -124,23 +124,29 @@ result. ### Current blocker -`evaluate_floor_state_changes.py` counts a wearer transition anywhere in a -fixed response window. Before this becomes the paper event protocol, add -competing-risk censoring: +`evaluate_floor_state_changes.py` now implements and unit-tests the required +competing-risk attribution: an other-offset response window ends before the +next other onset, and an other-onset yield window ends before the next other +offset. It also supports disjoint training-side threshold calibration, rejects +official-test calibration on Sessions 1--3, reports false-entry/false-stop +counts, and prints auditable timelines. -- an other-offset response window ends at the next other onset; -- an other-onset yield window ends at the next other offset. - -Until then, EOT/HOLD, response and yield results are proxy diagnostics, not -semantic event ground truth. +P0 is still not frozen. The remaining blocker is a real-data audit: run the v2 +evaluator on saved calibration/test predictions, inspect event timelines and +censoring/invalid counts, independently review sampled events, and freeze the +scenario definitions. These values remain timing-derived behavior proxies, +not semantic EOT/INT or addressee ground truth. See the +[`runbook`](docs/easycom_competing_risk_floor_evaluation.md). ### Next experiment gates The user has temporarily chosen to run the dense-AP model screen while P0 is unfinished. This does not relax the paper claim: event numbers remain proxy -diagnostics until competing-risk censoring is fixed. +diagnostics until the v2 attribution, calibration and scenario audit is frozen +on real data. -1. **P0:** fix/freeze event attribution, splits and scenario definitions. +1. **P0:** run and audit competing-risk v2, then freeze event attribution, + calibration, splits and scenario definitions. 2. **P1:** confirm the aggregate/per-person oracle screen over seeds 1--2, then train a causal visible-person active-speaker/floor-evidence estimator; compare correct, wrong-person and time-shifted visual inputs. diff --git a/README.md b/README.md index 29043e2..48d0c3c 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,8 @@ It separates four questions that must not be conflated: 4. Does any suppression preserve addressed-to-wearer response recall/latency and wearer yield behavior, rather than making the model globally passive? -The immediate order is: freeze the event protocol and competing-risk -attribution; run speaker-grounding probes and wrong-person/time-shift controls; +The immediate order is: run and audit the implemented competing-risk v2 event +protocol, then freeze it; run speaker-grounding probes and wrong-person/time-shift controls; re-evaluate `G`, `G+A`, and `G+A+pooled-V`; then train an anonymous person-wise causal side branch. A person-memory adapter or Thinker LoRA is a second-stage experiment only after the simpler branch passes those diagnostic gates. @@ -392,6 +392,40 @@ bits on CPU (pass `--dataset easycom` for EasyCom; default remains Ego4D). four-epoch heads with no dev set. This native-concatenation baseline is not capacity- or context-matched to the Thinker baseline. +The first epoch-budget sweep shows severe capacity overfitting. Four epochs +reach **73.42020** at 0.2 seconds and a new frozen-Omni best of **60.87144** +mean mAP, whereas 12 and 50 epochs fall to `57.98809` and `53.03515` despite +lower training losses. The 4-epoch concat-MLP is effectively tied with the +scalar mixture at 0.2 seconds (`-0.03065`) but improves mean mAP by `+0.33721`, +mainly at 1.2--1.6 seconds. Because the runs used different total-length cosine +schedules and the official test sessions were inspected, this is a diagnostic +capacity result rather than a valid final epoch selection. Full tables and +protocol caveats are archived in +[`results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/`](results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/README.md). + +### Frozen ROMA last-four-layer transfer + +Three frozen ROMA/Qwen2.5-Omni-7B readout runs were recorded on 2026-08-27 +using the same 8-second open-stream, 2 FPS, 5 Hz EasyCom evaluation path. A +four-epoch fixed mean over the last four `[3584]` query states is the best ROMA +control at **69.21863** mAP for 0.2 seconds and **58.90402** mean mAP. A +51.4M-parameter concat-MLP reaches only `68.65486 / 56.52425` after four +epochs; relative to the design- and epoch-matched frozen Qwen-3B concat model +it is `-4.76534` at 0.2 seconds and `-4.34719` in mean mAP. This is not +parameter-matched because the 3584-dimensional ROMA fusion is much wider. + +This is a negative-transfer diagnostic, not evidence that ROMA is universally +worse: the project used Qwen-style open-stream replay rather than ROMA's native +one-second streaming training recipe. The practical decision is nevertheless +clear: retain frozen Qwen2.5-Omni-3B as the main backbone and stop tuning the +wide ROMA concat head. Full horizon tables, class AP, validity caveats, JSON, +and a formatted workbook are archived in +[`results/easycom_roma_frozen_last4_2026-08-27/`](results/easycom_roma_frozen_last4_2026-08-27/README.md). + +The related-work synthesis and proposed next method—recent-200ms payload +readout plus structured future-floor projection—are documented in +[`reports/roma_negative_transfer_and_future_floor_projection_2026-08-27.md`](reports/roma_negative_transfer_and_future_floor_projection_2026-08-27.md). + ## Current Ego4D context-length results The following single-seed snapshot was recorded on 2026-08-22. Both variants @@ -1678,6 +1712,44 @@ At 0.2 seconds, sparse supervision alone changes mAP from `71.69695` to `71.08589`. This isolates the short-horizon regression primarily to Thinker updating rather than sparse target selection. +### Dense last-8 LoRA result + +Restricting LoRA to `q_proj`, `v_proj`, and `o_proj` in Thinker layers 28--35, +reducing the learning rate to `5e-6`, using dense 5-Hz supervision, and freezing +the trained 8-second head substantially reduces but does not reverse negative +transfer. Five continuous epochs were selected on Session 12 by 0.2-second +mAP; epoch 3 / step 11910 was then evaluated once on Sessions 3, 2, and 1. + +| Checkpoint | 0.2s full mAP | Full mean mAP | Delta from frozen dense mean | +|---|---:|---:|---:| +| Frozen Thinker dense-cache control | **71.69695** | **58.16091** | — | +| Dense last-8 q/v/o LoRA, frozen head, epoch 1, train 4--12 | 71.62100 | 58.03491 | -0.12600 | +| Dense last-8 q/v/o LoRA, frozen head, epoch 1, train 4--11 | 71.59585 | 58.01500 | -0.14591 | +| Dense last-8 q/v/o LoRA, frozen head, epoch 3, train 4--11 | 71.65000 | 57.88833 | -0.27258 | +| Prior frozen-head initialized sparse LoRA, 3 epochs | 71.08589 | 56.19608 | -1.96483 | + +Epoch 1 is only `0.05415` below epoch 3 at 0.2 seconds but is `0.12667` higher +in mean mAP. It reduces the mean regression relative to frozen dense from +`-0.27258` to `-0.14591`, with better results than epoch 3 at every horizon +from 0.6 through 2.0 seconds. This supports fixing one epoch and retraining on +all official training Sessions 4--12 rather than selecting a later checkpoint. +That full-train run reaches `71.62100` at 0.2 seconds and `58.03491` mean, +improving the Sessions 4--11 epoch-1 result by only `0.02515` and `0.01991`. +It remains `0.12600` mean points below frozen dense, confirming that withholding +Session 12 was not the cause of the residual negative transfer. Its complete +record is in +[`results/easycom_dense_last8_qvo_full_train_epoch1_2026-08-25/`](results/easycom_dense_last8_qvo_full_train_epoch1_2026-08-25/README.md). +For epoch 3, the gap grows almost monotonically to `-0.45040` at 2.0 seconds +and camera-wearer AP is the main regression (`-0.62718` average). +The complete development table, per-horizon/class test table, configuration, +and provenance caveats are archived in +[`results/easycom_dense_last8_qvo_frozen_head_epoch3_2026-08-25/`](results/easycom_dense_last8_qvo_frozen_head_epoch3_2026-08-25/README.md). + +This result should be reported as a negative-transfer ablation, not as an Omni +improvement. Any subsequent adapter interpolation or objective change must be +chosen without consulting Sessions 1--3; for a clean publication protocol, +the frozen head must also be retrained without the chosen development session. + For a speed-only alternative, `train_omni_turn_lora.py` supports a two-GPU producer-consumer benchmark. The producer GPU runs the frozen AV towers and places current-tail embeddings in a bounded CPU queue while the consumer GPU @@ -1753,9 +1825,10 @@ The headline comparison cannot establish speaker grounding or social understanding. Experiments therefore proceed through explicit gates: 1. **P0 — protocol correctness:** preserve physical 5 Hz timing and causal - masks, then prevent an other-offset response from being attributed after a - new other onset (and symmetrically censor yield after the next other - offset). Freeze scenario rules without tuning on EasyCom test Sessions 1--3. + masks. The evaluator now prevents an other-offset response from being + attributed at/after a new other onset and symmetrically censors yield at the + next other offset; run the real-data audit and freeze scenario rules without + tuning on EasyCom test Sessions 1--3. 2. **P1 — representation diagnosis:** compare EgoSpeak features, Qwen encoder-only A/V/AV, the global Thinker state, and anonymous person slots on aggregate activity and per-person active-speaker probes. Include wrong-person, diff --git a/docs/README.md b/docs/README.md index 8185583..7d2cbcd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -31,6 +31,7 @@ tests. | [`easycom_highrate_person_av.md`](easycom_highrate_person_av.md) | Frozen 25 Hz causal ASD person evidence fused with 5 Hz global Omni context | | [`easycom_muvap_stage3_baseline.md`](easycom_muvap_stage3_baseline.md) | MuVAP-only frozen VAP+ASD Stage-3 paper reconstruction on EgoSpeak/EasyCom targets | | [`easycom_oracle_information.md`](easycom_oracle_information.md) | Causal oracle per-person speaking-history upper bound and Session-12 screening commands | +| [`easycom_competing_risk_floor_evaluation.md`](easycom_competing_risk_floor_evaluation.md) | Running and auditing the competing-risk entry/yield evaluator with training-side threshold calibration | ## Design history and implementation options diff --git a/docs/easycom_competing_risk_floor_evaluation.md b/docs/easycom_competing_risk_floor_evaluation.md new file mode 100644 index 0000000..d5e6074 --- /dev/null +++ b/docs/easycom_competing_risk_floor_evaluation.md @@ -0,0 +1,109 @@ +# EasyCom competing-risk floor-event evaluation + +This runbook evaluates whether a candidate model reduces **false wearer-entry +triggers** and **false wearer-yield triggers** relative to a baseline. It reads +the saved `pred/` and `valid/` arrays; no GPU is required. + +The protocol is implemented in `evaluate_floor_state_changes.py`. It is a +timing-derived behavior evaluation, not semantic EOT/INT, addressee, or +individual-speaker ground truth: EasyCom's `others` target is aggregate +activity. + +## Event attribution + +At each horizon from 0.2 to 2.0 seconds, every eligible aggregate-other +boundary is counted once. + +- **Response candidate:** `others` changes from active to inactive while the + wearer is inactive. A wearer onset is a response only if it occurs before + the next `others` onset. +- **Yield candidate:** `others` changes from inactive to active while the + wearer is active. A wearer offset is a yield only if it occurs before the + next `others` offset. +- The competing boundary itself is excluded. A simultaneous wearer transition + is therefore not assigned to the earlier candidate. +- An observed wearer outcome before the recording ends is retained. An + unresolved candidate without its full horizon is right-censored and omitted. +- Invalid future-score paths are omitted and counted separately. + +For each event, predictions made from the same source time are max-pooled only +over the observable event window. This is a ranking score, not a calibrated +event probability. + +## Confirmatory test command + +Select each model's operating threshold on training-side predictions, then +apply that frozen threshold to official test Sessions 1--3. The run names in +`--calibration-run` must exactly match the names in `--run`. + +```bash +python3 evaluate_floor_state_changes.py \ + --target-root /path/to/easycom_target_frame \ + --run baseline=/path/to/baseline/test_result \ + --run candidate=/path/to/candidate/test_result \ + --calibration-run baseline=/path/to/baseline/calibration_result \ + --calibration-run candidate=/path/to/candidate/calibration_result \ + --calibration-sessions Session_12 \ + --sessions Session_3 Session_2 Session_1 \ + --matched-recalls 0.7 0.8 0.9 \ + --midturn-pause-max-seconds 2.0 \ + --print-event-timelines 3 \ + --output reports/easycom_floor_competing_risk/test_summary.json +``` + +If the calibration and test predictions already coexist under each `--run` +directory, omit `--calibration-run`. Official-test evaluation rejects missing +calibration and rejects calibration on Sessions 1--3. The +`--allow-in-sample-test-thresholds` option exists only for explicitly marked +exploratory audits and must not be used for a paper result. + +Session 12 is shown as the current development convention. If the final model +has been trained on Session 12, prefer out-of-fold training-side predictions +or a pre-registered cross-fitting scheme; record exactly how calibration +predictions were produced. + +## How to decide whether errors decreased + +The command writes: + +- `test_summary.json`: full protocol metadata, thresholds, counts, and metrics; +- `test_summary_events.csv`: one row per run and horizon for the primary event + evaluation; +- `test_summary_framewise.csv`: the older endpoint diagnostic, retained only + for secondary analysis. + +At the headline recall (normally 0.8), compare the following event-CSV fields: + +| Question | Primary field | Supporting checks | +|---|---|---| +| Did false entry decrease? | `no_response_trigger_rate_at_recall_0.8` | count, `response_recall_0.8`, response/no-response denominators | +| Did false entry during a short other-speaker restart decrease? | `short_resume_trigger_rate_at_recall_0.8` | trigger count and subgroup count | +| Did false yield/stop decrease? | `continue_false_stop_rate_at_recall_0.8` | count, `yield_recall_0.8`, yield/continue denominators | + +Lower is better for all three error rates. A lower error rate is not evidence +of improvement if the candidate also obtains materially lower test response or +yield recall. The requested recall selects the threshold on calibration data; +`response_recall_0.8` and `yield_recall_0.8` report the recall actually achieved +on test and may differ from 0.8. + +Before comparing models, confirm that their event denominators and invalid +window counts match. Then report both absolute percentage-point change and +relative error reduction, for example +`(baseline_rate - candidate_rate) / baseline_rate`. Event AP is a useful +threshold-free secondary result, but it does not replace the error-rate +comparison at a pre-registered operating point. + +## Required manual audit before protocol freeze + +Use `--print-event-timelines 3` first, then inspect a larger random sample with +an independent script or annotation view. For every printed timeline, verify: + +1. the anchor row is the intended aggregate-other boundary; +2. no outcome at or after `C/excluded` is attributed to the earlier event; +3. the fixed-source horizon score and `valid` mask line up with physical time; +4. session/recording boundaries do not create transitions; +5. right-censored and invalid counts are plausible. + +The current evaluator does not yet provide `third_party`, `addressed`, or +`group_or_unknown` strata, nor false entries per minute. Those require a +separately frozen addressee/scenario annotation contract and remain P0 TODOs. diff --git a/docs/easycom_muvap_stage3_baseline.md b/docs/easycom_muvap_stage3_baseline.md index 57ad5f2..deb21ec 100644 --- a/docs/easycom_muvap_stage3_baseline.md +++ b/docs/easycom_muvap_stage3_baseline.md @@ -206,6 +206,9 @@ CUDA_VISIBLE_DEVICES=4 PERSON_EVIDENCE=none bash run_easycom_muvap_stage3.sh tes CUDA_VISIBLE_DEVICES=5 PERSON_EVIDENCE=none bash run_easycom_muvap_stage3.sh test 2 ``` -Dense AP/mAP is currently the primary screening output. Floor-event results -remain timing-derived proxies until competing-risk censoring is frozen in -`evaluate_floor_state_changes.py`. +Dense AP/mAP is currently the primary screening output. For floor-event +comparison, run the competing-risk v2 evaluator with training-side threshold +calibration as documented in +[`easycom_competing_risk_floor_evaluation.md`](easycom_competing_risk_floor_evaluation.md). +The resulting events remain timing-derived behavior proxies; a real-data audit +and scenario freeze are still required before using them in the paper. diff --git a/docs/easycom_oracle_information.md b/docs/easycom_oracle_information.md index 5f34ab3..cd41f6e 100644 --- a/docs/easycom_oracle_information.md +++ b/docs/easycom_oracle_information.md @@ -81,8 +81,10 @@ checkpoint selection, and evaluates Sessions 1--3 once: CUDA_VISIBLE_DEVICES=0 bash run_easycom_oracle_information.sh final 0 1 2 ``` -The `final` stage enforces `EPOCHS=4` and exactly Sessions 4--12, clears the -development-session list internally, and saves the last epoch. Its artifacts +The `final` stage overrides any inherited `EPOCHS` value with `4`, requires +exactly Sessions 4--12, clears the development-session list internally, and +saves the last epoch. It validates this configuration before preparing oracle +sidecars. Its artifacts use separate `_train4_12_fixed4` checkpoint/result roots by default, so they cannot overwrite the historical Session-12 development screen. Override those roots only with `FINAL_CHECKPOINT_ROOT` and `FINAL_RESULT_ROOT`. diff --git a/docs/icassp2027_acceptance_storyline_2026-09-10.md b/docs/icassp2027_acceptance_storyline_2026-09-10.md index 01cc101..e52759d 100644 --- a/docs/icassp2027_acceptance_storyline_2026-09-10.md +++ b/docs/icassp2027_acceptance_storyline_2026-09-10.md @@ -193,3 +193,33 @@ Oracle audit、LoRA 和事件协议分别是动机、baseline 和验证工具， 这个结论只有在 learned causal method、错时/容量控制和冻结后的事件指标共同通过后才能写入 摘要。 + +## 10. 2026-09-10 manuscript 对齐决策 + +对 `manuscript/main/main.tex` 的逐段审查发现并记录以下冲突；稿件骨架已据此收缩： + +1. 标题只写 `floor-entry`，正文却同时主张 entry/yield，统一改为更宽的 floor + forecasting；在 same-rate 对照通过前不在标题使用 `dual-rate`。 +2. 单种子 oracle 不再充当摘要 headline；最终摘要必须以 learned method 和行为级结果为 + 主体。 +3. gap 从“Omni 尚未用于这一任务组合”改为 global semantic representation 与 sub-second + wearer-relative floor evidence 之间的 temporal--role mismatch。 +4. 四项并列贡献压缩为 adaptation mismatch、一个 local adapter、causal behavioral + evidence 三项；LoRA/oracle/event protocol 分别降为 baseline、动机和验证。 +5. 主方法不再预设 track-GRU/identity continuity，改为匿名、no-ID 的短时 visible-person + set；track 只有超过 no-ID/shuffled/capacity controls 后才恢复。 +6. wearer current state 默认只用于 entry/yield event interpretation。若作为模型输入，必须 + 先定义严格因果且部署可得的来源，并加入 no-state 对照。 +7. Qwen2.5-Omni-3B 为主 backbone，7B 仅作参数规模控制；MiniCPM-o 和跨家族 sweep 从四页 + 主线删除。 +8. Introduction 不得把尚未实现的 competing-risk censoring 写成已完成；P0 evaluator、测试 + 和人工时间轴是正式事件结论的硬门槛。 +9. 已查看的 Sessions 1--3 单种子 exploratory pooled-vision 表从投稿骨架删除，不用于模型 + 选择或 confirmatory claim。 +10. 最终只保留一个主结果表和一个机制消融表；oracle table、conditional track table 和 + 候选方法树不得并列占据正文。 +11. Conclusion 必须报告最终 learned method、matched-recall false-entry、onset/yield 和 + causal controls；若这些证据未通过，改写为 information audit 或停止方法型投稿。 + +当前 `manuscript/main/main.tex` 是带显式 TODO 的内容骨架。TODO 表示未实现、未冻结或未测量， +不得被引用为项目当前能力或实验结论。 diff --git a/docs/omni_speaker_floor_experiment_plan_2026-08-30.md b/docs/omni_speaker_floor_experiment_plan_2026-08-30.md index 232f9f4..b3ffd3e 100644 --- a/docs/omni_speaker_floor_experiment_plan_2026-08-30.md +++ b/docs/omni_speaker_floor_experiment_plan_2026-08-30.md @@ -86,8 +86,9 @@ speaker grounding 或 identity-aware reasoning。 2. 修正事件归因的 competing risk： - other offset 后的 wearer response 只在下一次 other onset 之前归因； - other onset 后的 wearer yield 只在下一次 other offset 之前归因。 - 现有 `evaluate_floor_state_changes.py` 使用固定 horizon 内任意 transition，暂时只能作为 - proxy，可能把后来无关的动作归给前一个边界。 + `evaluate_floor_state_changes.py` 已实现并通过合成单元测试；投稿前仍需在真实数据上打印 + 时间轴、复核 censoring/invalid 计数并冻结协议。运行方式见 + [`easycom_competing_risk_floor_evaluation.md`](easycom_competing_risk_floor_evaluation.md)。 3. 固定 EasyCom train/test（4--12 / 1--3）；阈值、窗口和规则不能在 test 上调。 4. 从训练 session 内留出 calibration/dev 子集，用来选择阈值和 early stopping；最终表格 仍在官方 test 上一次性报告。 diff --git a/evaluate_floor_state_changes.py b/evaluate_floor_state_changes.py index afde477..9987ec3 100644 --- a/evaluate_floor_state_changes.py +++ b/evaluate_floor_state_changes.py @@ -1,12 +1,15 @@ -"""Evaluate wearer state changes and conservative floor-context proxies. +"""Evaluate wearer state changes and conservative floor-context events. This evaluator consumes the ``pred/`` and ``valid/`` arrays already written by ``infer_turn_head.py``. It does not run a model and does not require a GPU. The activity head predicts future wearer activity, not an explicit event hazard. We therefore evaluate endpoint state changes conditional on the -observed wearer state at the prediction source. EOT/HOLD and yield metrics are -timing-derived proxies; they are deliberately not called semantic ground truth. +observed wearer state at the prediction source. Event windows use +competing-risk censoring: a response window ends before the next aggregate- +others onset, and a yield window ends before the next aggregate-others offset. +The resulting events remain timing-derived behavior proxies; aggregate +activity cannot establish semantic EOT/INT, speaker identity, or addressee. """ from __future__ import annotations @@ -23,6 +26,7 @@ import numpy as np # Keep this read-only evaluator independent of torch/model imports. OFFICIAL_TEST_SESSIONS = ("Session_3", "Session_2", "Session_1") +OFFICIAL_TRAIN_SESSIONS = tuple(f"Session_{index}" for index in range(4, 13)) CLASS_NAMES = ( "background", "camera_wearer_speaking", @@ -71,6 +75,36 @@ def parse_args() -> argparse.Namespace: default=(0.7, 0.8, 0.9), help="EOT-response/yield proxy recall operating points.", ) + parser.add_argument( + "--calibration-sessions", + nargs="+", + default=None, + help=( + "Training-side sessions used to select event thresholds. Required " + "when evaluating official Sessions 1--3 unless the exploratory " + "in-sample override is set." + ), + ) + parser.add_argument( + "--calibration-run", + action="append", + default=None, + metavar="NAME=RESULT_DIR", + help=( + "Optional calibration result directory for each --run name. If " + "omitted, calibration predictions are read from the matching " + "--run directory." + ), + ) + parser.add_argument( + "--allow-in-sample-test-thresholds", + action="store_true", + help=( + "Exploratory override allowing thresholds to be selected on the " + "same official test sessions being evaluated. Never use for a " + "confirmatory paper result." + ), + ) parser.add_argument( "--output", type=Path, @@ -96,6 +130,16 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Also print the older endpoint/framewise diagnostic table.", ) + parser.add_argument( + "--print-event-timelines", + type=int, + default=0, + metavar="N", + help=( + "Print up to N auditable 2-s timelines for each response/yield " + "event type from the first run (default: 0)." + ), + ) return parser.parse_args() @@ -252,6 +296,12 @@ def _rate_at_threshold(scores: np.ndarray, threshold: float | None) -> float | N return float((scores >= threshold).mean()) +def _count_at_threshold(scores: np.ndarray, threshold: float | None) -> int | None: + if threshold is None: + return None + return int((scores >= threshold).sum()) + + def _operating_points( score: np.ndarray, positive: np.ndarray, @@ -259,23 +309,42 @@ def _operating_points( recalls: list[float], *, extra_groups: dict[str, np.ndarray] | None = None, + fixed_thresholds: dict[float, float | None] | None = None, + threshold_source: str = "evaluated_sessions", ) -> list[dict[str, Any]]: positive_scores = score[positive] negative_scores = score[negative] result: list[dict[str, Any]] = [] for requested_recall in recalls: - threshold = _threshold_for_recall(positive_scores, requested_recall) + if fixed_thresholds is None: + threshold = _threshold_for_recall(positive_scores, requested_recall) + else: + if requested_recall not in fixed_thresholds: + raise ValueError( + f"Missing fixed threshold for recall {requested_recall}" + ) + threshold = fixed_thresholds[requested_recall] item: dict[str, Any] = { "requested_positive_recall": requested_recall, "threshold": threshold, + "threshold_source": threshold_source, "achieved_positive_recall": _rate_at_threshold(positive_scores, threshold), + "achieved_positive_count": _count_at_threshold( + positive_scores, threshold + ), "negative_false_positive_rate": _rate_at_threshold( negative_scores, threshold ), + "negative_false_positive_count": _count_at_threshold( + negative_scores, threshold + ), } for name, mask in (extra_groups or {}).items(): item[name] = _rate_at_threshold(score[mask], threshold) item[f"{name}_count"] = int(mask.sum()) + item[f"{name}_trigger_count"] = _count_at_threshold( + score[mask], threshold + ) result.append(item) return result @@ -287,6 +356,9 @@ def evaluate_horizon( session_ids: np.ndarray, horizon_index: int, recalls: list[float], + *, + operating_thresholds: dict[str, dict[float, float | None]] | None = None, + threshold_source: str = "evaluated_sessions", ) -> dict[str, Any]: shift = horizon_index + 1 rows = np.arange(len(target), dtype=np.int64) @@ -350,6 +422,12 @@ def evaluate_horizon( "all_wait_false_entry_rate": endpoint_wait, "early_entry_behavior_recall": early_entry, }, + fixed_thresholds=( + None + if operating_thresholds is None + else operating_thresholds["response"] + ), + threshold_source=threshold_source, ) yield_proxy = _ranking_from_masks( @@ -360,6 +438,10 @@ def evaluate_horizon( other_entry_yield, other_entry_continue, recalls, + fixed_thresholds=( + None if operating_thresholds is None else operating_thresholds["yield"] + ), + threshold_source=threshold_source, ) return { @@ -419,23 +501,76 @@ def _has_transition_within( return bool(transitions.any()) -def _has_others_resume_without_wearer_entry( - others: np.ndarray, - wearer: np.ndarray, +def _first_transition_row( + values: np.ndarray, + source_row: int, + horizon_steps: int, + *, + rising: bool, +) -> int | None: + window = values[source_row : source_row + horizon_steps + 1] + before = window[:-1] + after = window[1:] + transitions = (~before & after) if rising else (before & ~after) + indices = np.flatnonzero(transitions) + return int(source_row + indices[0] + 1) if len(indices) else None + + +def _competing_risk_window( + competing_state: np.ndarray, session_ids: np.ndarray, source_row: int, - max_pause_steps: int, -) -> bool: - # The offset is at source+1, so allow max_pause_steps after that boundary. - last = min(len(others) - 1, source_row + 1 + max_pause_steps) - for row in range(source_row + 2, last + 1): - if session_ids[row] != session_ids[source_row]: + horizon_steps: int, + *, + competing_rising: bool, +) -> tuple[int, int | None, bool] | None: + """Return the observable half-open event window after ``source_row``. + + The returned step count covers target rows ``source+1`` through + ``source+steps``. If an incompatible aggregate-others transition occurs at + row ``c``, that row is excluded and returned as the competing-risk row. A + wearer transition simultaneous with ``c`` is therefore not attributed to + the earlier anchor. The third value records whether the full nominal + horizon is observable. This lets the caller retain an observed wearer + outcome before a recording boundary while dropping an unresolved + right-censored candidate. + """ + + if source_row < 0 or horizon_steps < 1 or source_row + 1 >= len(competing_state): + return None + session_id = session_ids[source_row] + if session_ids[source_row + 1] != session_id: + return None + + nominal_last_row = source_row + horizon_steps + available_last_row = min(nominal_last_row, len(competing_state) - 1) + while ( + available_last_row > source_row + and session_ids[available_last_row] != session_id + ): + available_last_row -= 1 + + # Row source+1 contains the anchor's post-transition state. A competing + # transition can occur no earlier than source+2. + competing_row: int | None = None + for row in range(source_row + 2, available_last_row + 1): + before = bool(competing_state[row - 1]) + after = bool(competing_state[row]) + is_competing = ((not before) and after) if competing_rising else ( + before and (not after) + ) + if is_competing: + competing_row = row break - if not wearer[row - 1] and wearer[row]: - return False - if not others[row - 1] and others[row]: - return True - return False + + if competing_row is not None: + effective_steps = competing_row - source_row - 1 + if effective_steps < 1: + raise AssertionError("Competing-risk window must contain an observable step") + return effective_steps, competing_row, True + + nominal_complete = available_last_row >= nominal_last_row + return available_last_row - source_row, None, nominal_complete def _event_metric( @@ -444,6 +579,8 @@ def _event_metric( recalls: list[float], *, extra_negative_groups: dict[str, list[bool]] | None = None, + fixed_thresholds: dict[float, float | None] | None = None, + threshold_source: str = "evaluated_sessions", ) -> dict[str, Any]: score = np.asarray(scores, dtype=np.float64) positive = np.asarray(labels, dtype=bool) @@ -459,12 +596,45 @@ def _event_metric( negative, recalls, extra_groups=masks, + fixed_thresholds=fixed_thresholds, + threshold_source=threshold_source, ) for name, mask in masks.items(): metric[f"num_{name}"] = int(mask.sum()) return metric +def _event_window_diagnostics( + *, + source_boundaries: int, + effective_steps: list[int], + competing_censored: int, + right_censored: int, + invalid_score: int, +) -> dict[str, Any]: + steps = np.asarray(effective_steps, dtype=np.float64) + evaluated = int(len(steps)) + return { + "num_source_boundaries": int(source_boundaries), + "num_evaluated": evaluated, + "num_competing_risk_censored": int(competing_censored), + "competing_risk_censored_rate": ( + float(competing_censored / evaluated) if evaluated else None + ), + "num_right_censored": int(right_censored), + "num_invalid_score_window": int(invalid_score), + "mean_effective_window_seconds": ( + float(steps.mean() * STEP_SECONDS) if evaluated else None + ), + "min_effective_window_seconds": ( + float(steps.min() * STEP_SECONDS) if evaluated else None + ), + "max_effective_window_seconds": ( + float(steps.max() * STEP_SECONDS) if evaluated else None + ), + } + + def evaluate_event_horizon( target: np.ndarray, prediction: np.ndarray, @@ -474,13 +644,18 @@ def evaluate_event_horizon( recalls: list[float], *, midturn_pause_max_steps: int, + operating_thresholds: dict[str, dict[float, float | None]] | None = None, + threshold_source: str = "evaluated_sessions", ) -> dict[str, Any]: - """Evaluate each aggregate-other boundary once. + """Evaluate each aggregate-other boundary once with competing risks. Other offset events ask whether an inactive wearer starts within the - horizon. Other onset events ask whether an active wearer stops within the - horizon. The labels remain observed-behavior proxies: aggregate activity - cannot establish addressee, speaker identity, or semantic EOT/INT. + horizon but before the next other onset. Other onset events ask whether an + active wearer stops within the horizon but before the next other offset. + Competing boundary rows are excluded, so simultaneous wearer changes are + not attributed to the earlier anchor. The labels remain observed-behavior + proxies: aggregate activity cannot establish addressee, speaker identity, + or semantic EOT/INT. """ horizon_steps = horizon_index + 1 @@ -490,80 +665,166 @@ def evaluate_event_horizon( response_scores: list[float] = [] response_labels: list[bool] = [] short_resume_groups: list[bool] = [] + response_effective_steps: list[int] = [] + response_competing_censored = 0 + response_right_censored = 0 + response_invalid_score = 0 + response_source_boundaries = 0 offset_sources = np.flatnonzero(others[:-1] & ~others[1:]) for source_row in offset_sources: + if session_ids[source_row] != session_ids[source_row + 1]: + continue if wearer[source_row]: continue + response_source_boundaries += 1 + window = _competing_risk_window( + others, + session_ids, + int(source_row), + horizon_steps, + competing_rising=True, + ) + if window is None: + response_right_censored += 1 + continue + effective_steps, competing_row, nominal_complete = window + response_row = _first_transition_row( + wearer, + int(source_row), + effective_steps, + rising=True, + ) + if response_row is None and competing_row is None and not nominal_complete: + response_right_censored += 1 + continue score = _same_source_path_score( prediction, valid, session_ids, int(source_row), - horizon_steps, + effective_steps, offset=False, ) if score is None: + response_invalid_score += 1 continue - response = _has_transition_within( - wearer, - int(source_row), - horizon_steps, - rising=True, - ) + response = response_row is not None response_scores.append(score) response_labels.append(response) + response_effective_steps.append(effective_steps) + response_competing_censored += int( + competing_row is not None and response_row is None + ) + simultaneous_wearer_entry = bool( + competing_row is not None + and not wearer[competing_row - 1] + and wearer[competing_row] + ) short_resume_groups.append( (not response) - and _has_others_resume_without_wearer_entry( - others, - wearer, - session_ids, - int(source_row), - midturn_pause_max_steps, - ) + and competing_row is not None + and competing_row <= source_row + 1 + midturn_pause_max_steps + and not simultaneous_wearer_entry ) yield_scores: list[float] = [] yield_labels: list[bool] = [] + yield_effective_steps: list[int] = [] + yield_competing_censored = 0 + yield_right_censored = 0 + yield_invalid_score = 0 + yield_source_boundaries = 0 onset_sources = np.flatnonzero(~others[:-1] & others[1:]) for source_row in onset_sources: + if session_ids[source_row] != session_ids[source_row + 1]: + continue if not wearer[source_row]: continue + yield_source_boundaries += 1 + window = _competing_risk_window( + others, + session_ids, + int(source_row), + horizon_steps, + competing_rising=False, + ) + if window is None: + yield_right_censored += 1 + continue + effective_steps, competing_row, nominal_complete = window + yield_row = _first_transition_row( + wearer, + int(source_row), + effective_steps, + rising=False, + ) + if yield_row is None and competing_row is None and not nominal_complete: + yield_right_censored += 1 + continue score = _same_source_path_score( prediction, valid, session_ids, int(source_row), - horizon_steps, + effective_steps, offset=True, ) if score is None: + yield_invalid_score += 1 continue - yielded = _has_transition_within( - wearer, - int(source_row), - horizon_steps, - rising=False, - ) + yielded = yield_row is not None yield_scores.append(score) yield_labels.append(yielded) + yield_effective_steps.append(effective_steps) + yield_competing_censored += int( + competing_row is not None and yield_row is None + ) + + response_metric = _event_metric( + response_scores, + response_labels, + recalls, + extra_negative_groups={ + "short_resume_hard_negative": short_resume_groups, + }, + fixed_thresholds=( + None if operating_thresholds is None else operating_thresholds["response"] + ), + threshold_source=threshold_source, + ) + response_metric.update( + _event_window_diagnostics( + source_boundaries=response_source_boundaries, + effective_steps=response_effective_steps, + competing_censored=response_competing_censored, + right_censored=response_right_censored, + invalid_score=response_invalid_score, + ) + ) + yield_metric = _event_metric( + yield_scores, + yield_labels, + recalls, + fixed_thresholds=( + None if operating_thresholds is None else operating_thresholds["yield"] + ), + threshold_source=threshold_source, + ) + yield_metric.update( + _event_window_diagnostics( + source_boundaries=yield_source_boundaries, + effective_steps=yield_effective_steps, + competing_censored=yield_competing_censored, + right_censored=yield_right_censored, + invalid_score=yield_invalid_score, + ) + ) return { "horizon_index": horizon_steps, "nominal_seconds": horizon_steps * STEP_SECONDS, - "other_offset_to_wearer_response_event": _event_metric( - response_scores, - response_labels, - recalls, - extra_negative_groups={ - "short_resume_hard_negative": short_resume_groups, - }, - ), - "other_onset_to_wearer_yield_event": _event_metric( - yield_scores, - yield_labels, - recalls, - ), + "other_offset_to_wearer_response_event": response_metric, + "other_onset_to_wearer_yield_event": yield_metric, } @@ -621,21 +882,206 @@ def _operating_point( return None +def _event_operating_thresholds( + event_horizon: dict[str, Any], recalls: list[float] +) -> dict[str, dict[float, float | None]]: + result: dict[str, dict[float, float | None]] = { + "response": {}, + "yield": {}, + } + for recall in recalls: + response_point = _operating_point( + event_horizon["other_offset_to_wearer_response_event"], recall + ) + yield_point = _operating_point( + event_horizon["other_onset_to_wearer_yield_event"], recall + ) + assert response_point is not None and yield_point is not None + result["response"][recall] = response_point["threshold"] + result["yield"][recall] = yield_point["threshold"] + return result + + +def _framewise_operating_thresholds( + horizon: dict[str, Any], recalls: list[float] +) -> dict[str, dict[float, float | None]]: + result: dict[str, dict[float, float | None]] = { + "response": {}, + "yield": {}, + } + for recall in recalls: + response_point = _operating_point( + horizon["eot_response_vs_other_hold_proxy"], recall + ) + yield_point = _operating_point( + horizon["other_entry_yield_vs_continue_proxy"], recall + ) + assert response_point is not None and yield_point is not None + result["response"][recall] = response_point["threshold"] + result["yield"][recall] = yield_point["threshold"] + return result + + +def print_event_timeline_examples( + target: np.ndarray, + prediction: np.ndarray, + valid: np.ndarray, + session_ids: np.ndarray, + session_names: list[str], + *, + examples_per_type: int, + horizon_steps: int = NUM_HORIZONS, +) -> None: + """Print causal event windows for manual attribution inspection.""" + + if examples_per_type <= 0: + return + wearer = target[:, WEARER_INDEX].astype(bool) + others = target[:, OTHERS_INDEX].astype(bool) + session_starts = { + session_id: int(np.flatnonzero(session_ids == session_id)[0]) + for session_id in np.unique(session_ids) + } + specifications = ( + ( + "response", + np.flatnonzero(others[:-1] & ~others[1:]), + False, + True, + True, + False, + ), + ( + "yield", + np.flatnonzero(~others[:-1] & others[1:]), + True, + False, + False, + True, + ), + ) + print("\nCompeting-risk event timeline audit (first run, 2-s horizon)") + print("C=competing boundary excluded from the earlier event; E=wearer outcome") + for ( + event_name, + sources, + required_wearer_state, + competing_rising, + wearer_rising, + offset_score, + ) in specifications: + printed = 0 + for source_value in sources: + source_row = int(source_value) + if session_ids[source_row] != session_ids[source_row + 1]: + continue + if bool(wearer[source_row]) != required_wearer_state: + continue + window = _competing_risk_window( + others, + session_ids, + source_row, + horizon_steps, + competing_rising=competing_rising, + ) + if window is None: + continue + effective_steps, competing_row, nominal_complete = window + outcome_row = _first_transition_row( + wearer, + source_row, + effective_steps, + rising=wearer_rising, + ) + if outcome_row is not None: + status = "positive" + elif competing_row is not None: + status = "competing-censored negative" + elif nominal_complete: + status = "negative" + else: + status = "right-censored (excluded)" + event_score = ( + None + if status.startswith("right-censored") + else _same_source_path_score( + prediction, + valid, + session_ids, + source_row, + effective_steps, + offset=offset_score, + ) + ) + session_id = int(session_ids[source_row]) + local_source = source_row - session_starts[session_id] + display_last = source_row + effective_steps + if competing_row is not None: + display_last = max(display_last, competing_row) + print( + f"\n{event_name} | {session_names[session_id]} | " + f"source={local_source} ({local_source * STEP_SECONDS:.1f}s) | " + f"status={status} | score={event_score}" + ) + print("local_row | time_s | wearer | others | fixed-source score | mark") + for row in range(source_row, display_last + 1): + local_row = row - session_starts[session_id] + step = row - source_row + if step == 0: + score_text = "source" + elif row == competing_row: + score_text = "excluded" + elif step <= NUM_HORIZONS and valid[row, step - 1]: + raw_score = float(prediction[row, WEARER_INDEX, step - 1]) + score_text = f"{(1.0 - raw_score) if offset_score else raw_score:.3f}" + else: + score_text = "invalid" + marks: list[str] = [] + if row == source_row: + marks.append("anchor") + if row == outcome_row: + marks.append("E") + if row == competing_row: + marks.append("C/excluded") + print( + f"{local_row:9d} | {local_row * STEP_SECONDS:6.1f} | " + f"{int(wearer[row]):6d} | {int(others[row]):6d} | " + f"{score_text:18s} | {','.join(marks)}" + ) + printed += 1 + if printed >= examples_per_type: + break + if printed == 0: + print(f"\n{event_name} | no eligible example") + + def print_results(payload: dict[str, Any], *, print_framewise: bool) -> None: report_recall = min( payload["matched_recalls"], key=lambda value: abs(value - 0.8) ) print("\nWearer floor-event evaluation (primary event-level protocol)") print( - "Each aggregate-other boundary is counted once. AP/rates are percentages; " - "other offset/onset remain timing proxies, not semantic EOT/INT." + "Each aggregate-other boundary is counted once. Response windows stop " + "before the next other onset; yield windows stop before the next other " + "offset. The competing boundary itself is excluded. AP/rates are " + "percentages; events remain timing proxies, not semantic EOT/INT." ) + if payload["calibration_sessions"]: + print( + "Operating thresholds were selected on: " + + ", ".join(payload["calibration_sessions"]) + ) + else: + print( + "Operating thresholds were selected on the evaluated sessions " + "(exploratory/in-sample)." + ) print( "run | h | other-offset response AP | response/no-response/short-resume | " - f"no-response trigger @ response recall {report_recall:.0%} | " - f"short-resume trigger @ response recall {report_recall:.0%} | " - "other-onset yield AP | yield/continue | " - f"continue false-stop @ yield recall {report_recall:.0%}" + f"achieved response recall @ requested {report_recall:.0%} | " + "no-response trigger rate(count) | short-resume trigger rate(count) | " + "other-onset yield AP | yield/continue | achieved yield recall | " + "continue false-stop rate(count)" ) for name, run in payload["runs"].items(): for item in run["event_horizons"]: @@ -648,11 +1094,16 @@ def print_results(payload: dict[str, Any], *, print_framewise: bool) -> None: f"{_percentage(response['average_precision'])} | " f"{response['num_positive']}/{response['num_negative']}/" f"{response['num_short_resume_hard_negative']} | " - f"{_percentage(None if response_point is None else response_point['negative_false_positive_rate'])} | " - f"{_percentage(None if response_point is None else response_point.get('short_resume_hard_negative'))} | " + f"{_percentage(None if response_point is None else response_point['achieved_positive_recall'])} | " + f"{_percentage(None if response_point is None else response_point['negative_false_positive_rate'])} " + f"({None if response_point is None else response_point['negative_false_positive_count']}) | " + f"{_percentage(None if response_point is None else response_point.get('short_resume_hard_negative'))} " + f"({None if response_point is None else response_point.get('short_resume_hard_negative_trigger_count')}) | " f"{_percentage(yield_metric['average_precision'])} | " f"{yield_metric['num_positive']}/{yield_metric['num_negative']} | " - f"{_percentage(None if yield_point is None else yield_point['negative_false_positive_rate'])}" + f"{_percentage(None if yield_point is None else yield_point['achieved_positive_recall'])} | " + f"{_percentage(None if yield_point is None else yield_point['negative_false_positive_rate'])} " + f"({None if yield_point is None else yield_point['negative_false_positive_count']})" ) summary = run["event_summary"] print( @@ -717,8 +1168,14 @@ def write_framewise_csv(path: Path, payload: dict[str, Any]) -> None: ] for recall in payload["matched_recalls"]: fieldnames += [ + f"eot_threshold_{recall:g}", + f"eot_threshold_source_{recall:g}", f"eot_recall_{recall:g}", f"hold_false_entry_rate_at_eot_recall_{recall:g}", + f"other_entry_yield_threshold_{recall:g}", + f"other_entry_yield_threshold_source_{recall:g}", + f"other_entry_yield_recall_{recall:g}", + f"continue_false_stop_rate_at_yield_recall_{recall:g}", ] writer = csv.DictWriter(file, fieldnames=fieldnames) writer.writeheader() @@ -741,12 +1198,31 @@ def write_framewise_csv(path: Path, payload: dict[str, Any]) -> None: "continue_count": yield_metric["num_negative"], } for recall in payload["matched_recalls"]: - point = _operating_point(floor_metric, recall) - assert point is not None - row[f"eot_recall_{recall:g}"] = point["achieved_positive_recall"] - row[f"hold_false_entry_rate_at_eot_recall_{recall:g}"] = point[ - "negative_false_positive_rate" + response_point = _operating_point(floor_metric, recall) + yield_point = _operating_point(yield_metric, recall) + assert response_point is not None and yield_point is not None + row[f"eot_threshold_{recall:g}"] = response_point["threshold"] + row[f"eot_threshold_source_{recall:g}"] = response_point[ + "threshold_source" ] + row[f"eot_recall_{recall:g}"] = response_point[ + "achieved_positive_recall" + ] + row[f"hold_false_entry_rate_at_eot_recall_{recall:g}"] = ( + response_point["negative_false_positive_rate"] + ) + row[f"other_entry_yield_threshold_{recall:g}"] = yield_point[ + "threshold" + ] + row[ + f"other_entry_yield_threshold_source_{recall:g}" + ] = yield_point["threshold_source"] + row[f"other_entry_yield_recall_{recall:g}"] = yield_point[ + "achieved_positive_recall" + ] + row[ + f"continue_false_stop_rate_at_yield_recall_{recall:g}" + ] = yield_point["negative_false_positive_rate"] writer.writerow(row) @@ -761,17 +1237,34 @@ def write_event_csv(path: Path, payload: dict[str, Any]) -> None: "response_count", "no_response_count", "short_resume_hard_negative_count", + "response_source_boundary_count", + "response_competing_censored_count", + "response_right_censored_count", + "response_invalid_score_window_count", + "response_mean_effective_window_seconds", "yield_event_ap", "yield_count", "continue_count", + "yield_source_boundary_count", + "yield_competing_censored_count", + "yield_right_censored_count", + "yield_invalid_score_window_count", + "yield_mean_effective_window_seconds", ] for recall in payload["matched_recalls"]: fieldnames += [ + f"response_threshold_{recall:g}", + f"response_threshold_source_{recall:g}", f"response_recall_{recall:g}", f"no_response_trigger_rate_at_recall_{recall:g}", + f"no_response_trigger_count_at_recall_{recall:g}", f"short_resume_trigger_rate_at_recall_{recall:g}", + f"short_resume_trigger_count_at_recall_{recall:g}", + f"yield_threshold_{recall:g}", + f"yield_threshold_source_{recall:g}", f"yield_recall_{recall:g}", f"continue_false_stop_rate_at_recall_{recall:g}", + f"continue_false_stop_count_at_recall_{recall:g}", ] writer = csv.DictWriter(file, fieldnames=fieldnames) writer.writeheader() @@ -789,48 +1282,150 @@ def write_event_csv(path: Path, payload: dict[str, Any]) -> None: "short_resume_hard_negative_count": response[ "num_short_resume_hard_negative" ], + "response_source_boundary_count": response[ + "num_source_boundaries" + ], + "response_competing_censored_count": response[ + "num_competing_risk_censored" + ], + "response_right_censored_count": response[ + "num_right_censored" + ], + "response_invalid_score_window_count": response[ + "num_invalid_score_window" + ], + "response_mean_effective_window_seconds": response[ + "mean_effective_window_seconds" + ], "yield_event_ap": yield_metric["average_precision"], "yield_count": yield_metric["num_positive"], "continue_count": yield_metric["num_negative"], + "yield_source_boundary_count": yield_metric[ + "num_source_boundaries" + ], + "yield_competing_censored_count": yield_metric[ + "num_competing_risk_censored" + ], + "yield_right_censored_count": yield_metric[ + "num_right_censored" + ], + "yield_invalid_score_window_count": yield_metric[ + "num_invalid_score_window" + ], + "yield_mean_effective_window_seconds": yield_metric[ + "mean_effective_window_seconds" + ], } for recall in payload["matched_recalls"]: response_point = _operating_point(response, recall) yield_point = _operating_point(yield_metric, recall) assert response_point is not None and yield_point is not None + row[f"response_threshold_{recall:g}"] = response_point[ + "threshold" + ] + row[f"response_threshold_source_{recall:g}"] = response_point[ + "threshold_source" + ] row[f"response_recall_{recall:g}"] = response_point[ "achieved_positive_recall" ] row[f"no_response_trigger_rate_at_recall_{recall:g}"] = ( response_point["negative_false_positive_rate"] ) + row[f"no_response_trigger_count_at_recall_{recall:g}"] = ( + response_point["negative_false_positive_count"] + ) row[f"short_resume_trigger_rate_at_recall_{recall:g}"] = ( response_point["short_resume_hard_negative"] ) + row[f"short_resume_trigger_count_at_recall_{recall:g}"] = ( + response_point["short_resume_hard_negative_trigger_count"] + ) + row[f"yield_threshold_{recall:g}"] = yield_point["threshold"] + row[f"yield_threshold_source_{recall:g}"] = yield_point[ + "threshold_source" + ] row[f"yield_recall_{recall:g}"] = yield_point[ "achieved_positive_recall" ] row[f"continue_false_stop_rate_at_recall_{recall:g}"] = ( yield_point["negative_false_positive_rate"] ) + row[f"continue_false_stop_count_at_recall_{recall:g}"] = ( + yield_point["negative_false_positive_count"] + ) writer.writerow(row) def main() -> None: args = parse_args() runs = parse_runs(args.run) + calibration_run_values = parse_runs(args.calibration_run or []) recalls = validate_recalls(args.matched_recalls) if args.midturn_pause_max_seconds <= 0: raise ValueError("--midturn-pause-max-seconds must be positive") + if args.print_event_timelines < 0: + raise ValueError("--print-event-timelines must be non-negative") midturn_pause_max_steps = max( 1, int(math.ceil(args.midturn_pause_max_seconds / STEP_SECONDS)) ) sessions = [normalize_session_name(value) for value in args.sessions] if len(sessions) != len(set(sessions)): raise ValueError("Duplicate sessions") + calibration_sessions = ( + [normalize_session_name(value) for value in args.calibration_sessions] + if args.calibration_sessions + else None + ) + if calibration_sessions and len(calibration_sessions) != len( + set(calibration_sessions) + ): + raise ValueError("Duplicate calibration sessions") + if calibration_run_values and calibration_sessions is None: + raise ValueError("--calibration-run requires --calibration-sessions") + run_names = {name for name, _ in runs} + if calibration_run_values: + calibration_names = {name for name, _ in calibration_run_values} + if calibration_names != run_names: + raise ValueError( + "--calibration-run names must exactly match --run names; " + f"runs={sorted(run_names)}, calibration={sorted(calibration_names)}" + ) + if calibration_sessions and set(calibration_sessions) & set(sessions): + if not args.allow_in_sample_test_thresholds: + raise ValueError( + "Calibration and evaluation sessions overlap. Use disjoint " + "training-side calibration sessions, or set the exploratory " + "--allow-in-sample-test-thresholds override." + ) + if ( + set(sessions) & set(OFFICIAL_TEST_SESSIONS) + and calibration_sessions is None + and not args.allow_in_sample_test_thresholds + ): + raise ValueError( + "Official test evaluation requires --calibration-sessions so event " + "thresholds are selected on training-side data. Use " + "--allow-in-sample-test-thresholds only for an explicitly " + "exploratory audit." + ) + if ( + set(sessions) & set(OFFICIAL_TEST_SESSIONS) + and calibration_sessions is not None + and not set(calibration_sessions).issubset(OFFICIAL_TRAIN_SESSIONS) + ): + raise ValueError( + "Official test thresholds may only be calibrated on training-side " + "Sessions 4--12; got " + ", ".join(calibration_sessions) + ) + calibration_run_dirs = ( + dict(calibration_run_values) if calibration_run_values else dict(runs) + ) payload: dict[str, Any] = { - "protocol": "wearer_endpoint_floor_proxy_v1", + "protocol": "wearer_floor_event_competing_risk_v2", "sessions": sessions, + "calibration_sessions": calibration_sessions, "classes": list(CLASS_NAMES), "matched_recalls": recalls, "prediction_layout": "[target_time, class, future_horizon]", @@ -840,47 +1435,154 @@ def main() -> None: ), "interpretation": ( "OFF->ON and ON->OFF are endpoint state changes. EOT/HOLD, early-entry, " - "and other-entry yield are timing-derived proxies, not semantic judgments." + "and other-entry yield are timing-derived behavior proxies, not semantic " + "judgments. Event operating points report false-entry/false-stop counts " + "and rates at matched observed-behavior recall." ), "event_protocol": ( "Each aggregate-others 1->0 boundary with wearer initially off is one " "response candidate; each aggregate-others 0->1 boundary with wearer " - "initially on is one yield candidate. Labels are observed wearer " - "transitions within the horizon. Fixed-source future activity scores " - "are max-pooled only as ranking scores." + "initially on is one yield candidate. A response window is half-open and " + "ends before the next aggregate-others onset; a yield window ends before " + "the next aggregate-others offset. A simultaneous wearer change at that " + "competing boundary is excluded from the earlier event. Labels are " + "observed wearer transitions within the remaining window. Fixed-source " + "future activity scores are max-pooled over the same remaining window " + "only as ranking scores. Candidates without enough follow-up and without " + "an observed competing boundary are right-censored." ), "midturn_pause_max_seconds": args.midturn_pause_max_seconds, "runs": {}, } + printed_timelines = False for name, run_dir in runs: target, prediction, valid, session_ids = load_arrays( args.target_root, run_dir, sessions ) - horizons = [ - evaluate_horizon( + if args.print_event_timelines and not printed_timelines: + print_event_timeline_examples( target, prediction, valid, session_ids, - horizon_index, - recalls, + sessions, + examples_per_type=args.print_event_timelines, ) - for horizon_index in range(NUM_HORIZONS) - ] - event_horizons = [ - evaluate_event_horizon( - target, - prediction, - valid, - session_ids, - horizon_index, - recalls, - midturn_pause_max_steps=midturn_pause_max_steps, + printed_timelines = True + calibration_thresholds: list[dict[str, Any]] | None = None + calibration_framewise_thresholds: list[dict[str, Any]] | None = None + threshold_source = "evaluated_sessions" + if calibration_sessions is not None: + calibration_dir = calibration_run_dirs[name] + ( + calibration_target, + calibration_prediction, + calibration_valid, + calibration_session_ids, + ) = load_arrays( + args.target_root, + calibration_dir, + calibration_sessions, + ) + threshold_source = "calibration_sessions:" + ",".join( + calibration_sessions + ) + calibration_thresholds = [] + calibration_framewise_thresholds = [] + for horizon_index in range(NUM_HORIZONS): + calibration_horizon = evaluate_horizon( + calibration_target, + calibration_prediction, + calibration_valid, + calibration_session_ids, + horizon_index, + recalls, + ) + framewise_thresholds = _framewise_operating_thresholds( + calibration_horizon, recalls + ) + calibration_framewise_thresholds.append( + { + "horizon_index": horizon_index + 1, + "nominal_seconds": (horizon_index + 1) * STEP_SECONDS, + "response": framewise_thresholds["response"], + "yield": framewise_thresholds["yield"], + } + ) + calibration_event = evaluate_event_horizon( + calibration_target, + calibration_prediction, + calibration_valid, + calibration_session_ids, + horizon_index, + recalls, + midturn_pause_max_steps=midturn_pause_max_steps, + ) + thresholds = _event_operating_thresholds( + calibration_event, recalls + ) + calibration_thresholds.append( + { + "horizon_index": horizon_index + 1, + "nominal_seconds": (horizon_index + 1) * STEP_SECONDS, + "response": thresholds["response"], + "yield": thresholds["yield"], + } + ) + + horizons: list[dict[str, Any]] = [] + event_horizons: list[dict[str, Any]] = [] + for horizon_index in range(NUM_HORIZONS): + fixed_framewise_thresholds = None + fixed_event_thresholds = None + if calibration_thresholds is not None: + assert calibration_framewise_thresholds is not None + event_threshold_row = calibration_thresholds[horizon_index] + framewise_threshold_row = calibration_framewise_thresholds[ + horizon_index + ] + fixed_event_thresholds = { + "response": event_threshold_row["response"], + "yield": event_threshold_row["yield"], + } + fixed_framewise_thresholds = { + "response": framewise_threshold_row["response"], + "yield": framewise_threshold_row["yield"], + } + horizons.append( + evaluate_horizon( + target, + prediction, + valid, + session_ids, + horizon_index, + recalls, + operating_thresholds=fixed_framewise_thresholds, + threshold_source=threshold_source, + ) + ) + event_horizons.append( + evaluate_event_horizon( + target, + prediction, + valid, + session_ids, + horizon_index, + recalls, + midturn_pause_max_steps=midturn_pause_max_steps, + operating_thresholds=fixed_event_thresholds, + threshold_source=threshold_source, + ) ) - for horizon_index in range(NUM_HORIZONS) - ] payload["runs"][name] = { "result_dir": str(run_dir.expanduser().resolve()), + "calibration_result_dir": ( + str(calibration_run_dirs[name].expanduser().resolve()) + if calibration_sessions is not None + else None + ), + "event_thresholds": calibration_thresholds, + "framewise_thresholds": calibration_framewise_thresholds, "horizons": horizons, "summary": summarize_horizons(horizons), "event_horizons": event_horizons, diff --git a/manuscript/main/README.md b/manuscript/main/README.md index 30e7f2b..54f3cae 100644 --- a/manuscript/main/README.md +++ b/manuscript/main/README.md @@ -25,32 +25,28 @@ main/ build/ # generated, ignored ``` -The author/affiliation, final results, Method, Experimental Setup, Results and -Conclusion are still explicit TODOs. Do not turn planned experiments into -reported findings. +The manuscript is intentionally a content skeleton with visible TODOs. Do not +turn planned implementations, controls, or results into completed claims. -The current paper thesis is model-agnostic and deliberately narrower than -addressee prediction. It asks whether general-purpose omni representations -expose the fine-grained evidence needed for 200 ms wearer floor-action -projection, whether PEFT can recover it, and whether causal local audio/person -evidence provides a reliable complement. The initial planned backbone set is: +The frozen paper story is documented in +[`../../docs/icassp2027_acceptance_storyline_2026-09-10.md`](../../docs/icassp2027_acceptance_storyline_2026-09-10.md). +The paper asks how long-context Omni understanding can be adapted into +sub-second, wearer-relative floor control for an invisible egocentric wearer. +Its single candidate contribution is a causal local floor-evidence adapter: -- Qwen2.5-Omni-3B; -- Qwen2.5-Omni-7B; -- MiniCPM-o 4.5. +- Qwen2.5-Omni-3B is the primary global backbone; +- matched LoRA/PEFT is a baseline, not a separate contribution; +- Qwen2.5-Omni-7B is an optional capacity-scaling control; +- the primary local method uses an anonymous, no-ID visible-person set; +- track continuity is conditional and must defeat no-ID/shuffled/capacity + controls before entering the paper; +- MiniCPM-o and a cross-family sweep are removed from the core plan; +- the oracle is analysis-only motivation, not a deployable method or headline; +- the publication gate is lower competing-risk-censored multi-other false + entry at matched wearer-onset recall without worse valid onset or yield. -The backbone and final fusion method are **not frozen yet**. Current Qwen-3B -numbers are exploratory motivation, not evidence that every omni model has the -same limitation. Compare frozen, PEFT, and local-evidence protocols within each -backbone; report common local encoders separately from model-native local -encoders. Raw cross-model scores are secondary because the models expose -different streaming/readout interfaces. - -Track-consistent person fusion remains one candidate after pooled and -capacity-matched controls. If selected, dataset participant IDs route -short-term causal state only; they are not semantic inputs. The publication -gate is an event-level reduction in multi-other false entries at matched -wearer-onset recall without degrading onset latency or wearer-yield behavior. -A general omni claim requires a consistent result across at least two backbone -families; otherwise the paper must state a model-specific finding. Selected -examples are explanatory evidence only. +Competing-risk censoring and training-side threshold calibration are now +implemented and unit-tested. The remaining P0 blocker is the real-data event +audit and protocol/scenario freeze; until then, event values are timing-derived +proxies. Already viewed Sessions 1--3 numbers are exploratory internal evidence +and must not appear in the confirmatory paper table. diff --git a/manuscript/main/REVIEWER_CHECKLIST.md b/manuscript/main/REVIEWER_CHECKLIST.md index 1377914..fdbd93d 100644 --- a/manuscript/main/REVIEWER_CHECKLIST.md +++ b/manuscript/main/REVIEWER_CHECKLIST.md @@ -2,13 +2,11 @@ This is an internal execution checklist. It is not part of the submitted PDF. -## Reviewer verdict on the current draft +## Reviewer verdict and frozen paper shape -The potentially publishable question is now model-agnostic: **do -general-purpose omni representations expose the fine-grained evidence needed -for 200 ms egocentric floor action, and can causal local evidence reduce false -wearer entries at matched onset recall when frozen readout or PEFT is -insufficient?** +The potentially publishable question is: **can a general-purpose Omni readout +be adapted into 200 ms, wearer-relative floor control by explicitly preserving +short-window causal audio and anonymous visible-person evidence?** The contribution is not “the first audio-visual multiparty turn model.” VAP, EgoSpeak, SICNet, and MuVAP already occupy adjacent territory. The defensible @@ -17,17 +15,25 @@ wearer, combined with a controlled diagnosis of global omni states and a behavior-aligned multi-other event protocol. Current Qwen2.5-Omni-3B results are exploratory motivation only. They neither -prove a limitation shared by omni models nor establish track-consistent fusion -as the final method. The planned initial backbone set is Qwen2.5-Omni-3B, -Qwen2.5-Omni-7B, and MiniCPM-o 4.5. +prove a limitation shared by Omni models nor establish track-consistent fusion +as the final method. Qwen2.5-Omni-3B is primary; Qwen2.5-Omni-7B is an optional +capacity control. MiniCPM-o and a cross-family sweep are outside the core paper. -## P0 — Freeze the task before the backbone sweep +The four-page paper has only three contributions: temporal--role adaptation +mismatch, one causal local floor adapter, and causal behavioral evidence. +LoRA, the oracle, and the event protocol are respectively a baseline, +motivation, and validation mechanism. + +## P0 — Freeze the task before confirmatory adaptation runs - [ ] Define the exact 5 Hz causal cutoff and physical target interval for each of the ten 0.2 s horizons. - [ ] Freeze wearer-onset/yield and multi-other exchange definitions. -- [ ] Add competing-risk rules: a candidate event ends at the first matched - wearer action, incompatible floor change, timeout, or invalid source boundary. +- [x] Implement and unit-test competing-risk rules: a response candidate ends + before the next other onset and a yield candidate before the next other + offset; right-censored and invalid windows are counted explicitly. +- [ ] Audit the competing-risk implementation on real timelines and freeze the + timeout/source-boundary convention before confirmatory reporting. - [ ] Specify overlap, minimum silence, maximum response delay, duplicate detections, source discontinuities, track visibility, offscreen speakers, and whether target-of-speech strata are diagnostic only. @@ -41,9 +47,9 @@ Qwen2.5-Omni-7B, and MiniCPM-o 4.5. least 100 randomly selected events. Publish the number of unique other IDs and multi-other opportunity minutes before the main sweep. -## P1 — Audit each backbone interface +## P1 — Audit the primary backbone interface -For Qwen2.5-Omni-3B, Qwen2.5-Omni-7B, and MiniCPM-o 4.5, record: +For Qwen2.5-Omni-3B, and for 7B only if used as the scaling control, record: - [ ] exact model/code revision, processor, dtype, and hardware; - [ ] native audio/video sampling and streaming/chunk semantics; @@ -57,9 +63,9 @@ a model cannot expose a comparable causal state, report that protocol as unsupported. Raw scores are not a pure architecture ranking; emphasize within-backbone deltas. -## P2 — Establish the compact representation matrix +## P2 — Establish the compact adaptation matrix -Run three seeds per supported backbone with the same data, physical cutoffs, +Run three seeds for the primary backbone with the same data, physical cutoffs, target, prediction-head capacity, optimizer exposure, and evaluator: 1. Frozen global causal state. @@ -67,41 +73,39 @@ target, prediction-head capacity, optimizer exposure, and evaluator: 3. Global state plus recent local audio. 4. Global state plus pooled local audio and vision. -Use a common local encoder as the primary cross-backbone control. If native -local encoders are tested, report them in a separate within-model panel. Do not -mix common-encoder and native-encoder deltas in one claim. +Add a parameter- and history-matched temporal local branch before attributing +gain to audiovisual organization. Use the same local adapter for the optional +7B scaling row. **Gate:** baseline standard deviations are known; the evaluator reproduces identical scores from saved predictions; parameter counts and training budgets are reported. A single failed LoRA recipe is evidence about that recipe, not proof that the model cannot adapt. -## P3 — Select local fusion only after P2 +## P3 — Train the no-ID local floor adapter -If local evidence gives a reliable event-level gain, compare candidates in -increasing complexity: +The primary method is a short-window causal encoder over recent mixed audio and +per-frame anonymous visible-person sets. It uses no participant-ID embedding or +cross-time track state. Compare it with: -1. Pooled local audio/vision. -2. Parameter-matched temporal branch. -3. Per-frame person set with no cross-time identity. -4. Shared track-wise temporal encoder. -5. Person–audio cross-attention. -6. Optional per-person active-speaker auxiliary loss, only after selecting a - person-aware activity-only model. +1. Recent local audio only. +2. Global/full-frame visual pooling. +3. A parameter- and history-matched temporal branch. +4. Time-shifted local vision. +5. Removed local vision. +6. A same-rate control if the local path runs faster than 5 Hz. Keep the dense EgoSpeak wearer/others target as the primary task. Target of -speech is analysis-only and never model input. Describe only the train-side -winner as the final method; move rejected candidates to ablations. +speech is analysis-only and never model input. Use ``dual-rate'' only if the +local input/update rate differs explicitly and defeats the same-rate control. -For a track-aware candidate, temporary IDs route hidden state only: no ID -embedding, cross-recording identity, or fixed seating slot. Unit-test variable -person count, empty sets, padding, slot permutation, global ID renaming, +Unit-test variable person count, empty sets, padding, person-set permutation, source/gap resets, and strict causal cutoffs. -## P4 — Run visual counterfactuals conditionally +## P4 — Consider track continuity only after the no-ID method wins -Only if a track-aware candidate is selected, evaluate one frozen checkpoint -with: +Only if a track-aware candidate then beats the no-ID method on training-side +data, evaluate one frozen checkpoint with: - correct person tracks; - clip-consistent global renaming, which must be invariant; @@ -121,11 +125,14 @@ retrain test-time counterfactuals. - [ ] Operating points at wearer-onset recall 0.7, 0.8, and 0.9; headline 0.8. - [ ] False wearer entries per multi-other opportunity minute, onset latency, wearer-yield recall/latency, and original ten-horizon AP/mAP. -- [ ] Error strata: backbone, visibility, number of visible people, overlap, - offscreen activity, track age/fragmentation, and transition proxy. +- [ ] Error strata: visibility, number of visible people, overlap, offscreen + activity, and transition proxy. Add track age/fragmentation only if a + track-aware method is selected. - [ ] Ego4D AVD coverage audit on the exact EgoSpeak split before claiming the same multi-other analysis. -- [ ] Provided-track and online-tracker conditions named separately. +- [ ] Name the causal box source and its limitations. Add separate + provided-track/online-tracker rows only if tracking enters the selected + method. Qualitative cases follow a frozen selection rule and include the largest paired improvement, a median case, a failure, and one preserved true wearer onset. @@ -134,23 +141,20 @@ Examples explain rates; they never replace them. ## Paper assembly order 1. Freeze and audit the event protocol. -2. Implement and causally audit all backbone readouts. -3. Complete the compact frozen/PEFT/local matrix. -4. Decide whether the paper can support a cross-family or only model-specific - representation claim. -5. Select a local fusion candidate using training-side event metrics. -6. Run conditional visual/track counterfactuals. -7. Produce the cross-backbone event table and paired intervals. -8. Draw one compact final system figure; remove rejected methods from the main +2. Audit the Qwen2.5-Omni-3B readout and matched PEFT baseline. +3. Complete the frozen/PEFT/audio/matched-temporal/local-AV matrix. +4. Run correct/time-shift/remove-vision controls and the optional 7B scaling row. +5. Consider track continuity only if the no-ID method already passes. +6. Produce one confirmatory event table and one mechanism-ablation table. +7. Draw one compact final system figure; remove rejected methods from the main narrative. -9. Replace every visible TODO and verify the ICASSP 2027 kit/page limit. +8. Replace every visible TODO and verify the ICASSP 2027 kit/page limit. ## Stop/go decisions -- **General claim:** at least two backbone families show the same frozen/PEFT - weakness and a reliable local-evidence improvement. -- **Model-specific claim:** the effect appears only within Qwen; say so and do - not generalize to all omni models. +- **Claim boundary:** with Qwen-only evidence, describe a Qwen2.5-Omni case + study and the tested adaptation mechanism; do not generalize to all Omni + models. - **Audio-local reframe:** local audio improves but all visual variants fail; claim a fine-grained acoustic readout bottleneck, not social visual reasoning. - **Track-aware method:** correct continuity beats pooled vision, temporal diff --git a/manuscript/main/main.tex b/manuscript/main/main.tex index d2cbbf3..c14a86a 100644 --- a/manuscript/main/main.tex +++ b/manuscript/main/main.tex @@ -1,7 +1,6 @@ -% Internal ICASSP 2027 draft using the official ICASSP 2026 template bundled -% in this repository. Replace the class/style files when the 2027 author kit -% is released, and remove every visible \todo{} marker before submission. -% Compile from this directory with: make +% ICASSP 2027 content skeleton. The final paper must use the official 2027 +% author kit and contain no visible \todo{} markers. Planned experiments are +% intentionally marked rather than presented as completed work. \documentclass{article} \usepackage{spconf} \usepackage{amsmath,amssymb,graphicx,hyperref,xcolor} @@ -10,7 +9,8 @@ \newcommand{\vect}[1]{\boldsymbol{#1}} \setlength{\emergencystretch}{2em} -\title{Adapting Omni Models for Causal Egocentric Multiparty Floor-Entry Forecasting} +\title{Is It My Turn? Causal Local Evidence Adaptation of Omni Models for +Egocentric Floor Forecasting} % TODO: Insert the final author list and affiliations after the ICASSP 2027 % review policy and author order are finalized. @@ -22,603 +22,340 @@ \maketitle \begin{abstract} -In multiparty conversation, a pause need not invite an egocentric participant -to speak: it may instead mark a handover between two other people. We study -causal egocentric multiparty floor-entry forecasting, where the target is the -future speech participation of the first-person camera wearer---the one -participant who is necessarily absent from the visible speaker set. This -wearer-relative target distinguishes our setting from recent multimodal VAP -models that forecast visible floor holders or observe human--human turn -dynamics. Following EgoSpeak, we use causal first-person audio--video and the -human wearer's future speaking behavior as an imitation target, and ask how a -jointly pretrained audio--visual Omni model should be adapted for predictions -every 200~ms. Our global--local formulation combines a causal Omni readout with -a lightweight pathway for recent mixed audio and visible-person observations. -On EasyCom development data, a single-seed oracle analysis finds that perfectly -aligned aggregate activity among visible non-wearers improves 0.2-s mAP by -6.97 points and ten-horizon mean mAP by 2.91 points over global Omni plus recent -audio, whereas person-resolved histories do not outperform an -identity-shuffled control. This motivates timely local floor evidence without -assuming that person continuity is useful. The full study compares frozen, -parameter-efficient, and learned local adaptation and evaluates false wearer -entry at matched wearer-onset recall together with yield behavior. -\todo{After the train-side study is locked, replace the final two sentences -with the selected learned local variant and insert: its oracle-gap recovery, -multi-other false-entry reduction at 80\% onset recall, onset/yield -non-degradation, and 95\% intervals. Mention track continuity only if its -controls separate.} +Real-time audiovisual agents must decide not only what to say, but whether an +emerging floor opportunity belongs to them. This is difficult in multiparty +egocentric interaction: the target wearer is behind the camera, while the +visible people provide only indirect evidence about whether the surrounding +floor remains occupied. We study strictly causal forecasting of the wearer's +future speech activity every 200~ms. Although a general-purpose Omni model +provides rich long-context audio--visual representations, its global readout is +not explicitly trained to expose sub-second, wearer-relative floor evidence. +We therefore study a candidate causal local floor adapter that combines the global +Omni state with short-window mixed audio and an anonymous set of visible-person +observations, without assuming persistent speaker identity. On EasyCom, +\todo{report the final three-seed dense and competing-risk-censored event +results: false-entry reduction at matched wearer-onset recall, onset latency, +yield behavior, confidence intervals, and oracle-gap recovery}. Controlled +\todo{time-shift/remove-vision/same-rate or capacity-matched} interventions +show that \todo{state the supported mechanism, without claiming addressee or +speaker identity}. These results demonstrate \todo{the narrow conclusion +actually supported by the final experiments}. \end{abstract} \begin{keywords} -conversational agents, turn-taking, speech initiation, egocentric video, -omni models +conversational agents, turn-taking, egocentric video, multimodal adaptation, +voice activity projection \end{keywords} \section{Introduction} \label{sec:intro} Natural turn-taking requires a conversational system to anticipate not only -when speech will stop, but whether an emerging floor opportunity actually -belongs to it. This distinction is particularly important in multiparty -interaction. In a dyadic conversation, a pause after one interlocutor finishes -often provides an opportunity for the other to respond; in a group -conversation, the same pause may simply mark a handover between two other -participants. An egocentric conversational agent that reacts only to silence -or aggregate speech activity may therefore enter the floor while the -surrounding conversation should continue without it. We study this problem as -\emph{egocentric multiparty floor-entry forecasting}: continuously predicting -the near-future speech participation of the first-person participant from -causal audio--visual observations. +when speech will stop, but whether a floor opportunity actually belongs to it. +In a dyadic exchange, a pause after one interlocutor finishes often invites the +other to respond. In a group conversation, the same pause may instead mark a +handover between two other participants. An egocentric agent that reacts only +to silence or aggregate speech may therefore interrupt a conversation that +should continue without it. We study \emph{egocentric floor forecasting}: +continuously predicting the near-future speech participation of the +first-person wearer from strictly causal audio--visual observations. -Voice Activity Projection (VAP) provides a predictive formulation of -turn-taking by continuously projecting the joint future voice activity of -conversational participants, from which turn shifts, holds, and backchannels -can be inferred \cite{ekstedt2022vap}. EgoSpeak extends predictive speech -initiation to the first-person setting, using causal, untrimmed egocentric -audio--visual observations and the future speaking behavior of the camera -wearer as supervision \cite{kim2025egospeak}. This behavior is a useful -imitation target for the perceptual decision faced by an egocentric agent: -given everything observed so far, is the first-person participant likely to -enter the floor next? EgoSpeak establishes that this prediction can be learned -from real-world egocentric streams, but leaves open how recent jointly -pretrained audio--visual Omni models should be used for such fine-grained -causal forecasting. +Voice Activity Projection (VAP) introduced continuous prediction of future +voice activity for turn-taking \cite{ekstedt2022vap}. EgoSpeak brought this +formulation to causal, untrimmed first-person streams, using future camera- +wearer activity as an imitation target \cite{kim2025egospeak}. Complementary +work adds visual gaze, head pose, and facial behavior to predictive turn-taking +\cite{russell2025mmvap}, while MuVAP combines global acoustic dynamics with +visible face tracks to forecast visible floor holders in multiparty scenes +\cite{qi2026muvap}. Our target has a different role structure: the participant +whose future action is predicted is the one participant absent from the +visible set. The visible people are contextual evidence for the unseen +wearer's decision, not candidate output identities. -In parallel, recent work has expanded predictive turn-taking along multimodal -and multiparty dimensions. MM-VAP incorporates facial expression, head pose, -and gaze into predictive turn-taking for two-party visual interaction -\cite{russell2025mmvap}. A recent social-robot MM-VAP framework adapts -voice-activity-related audio--visual encoders for anticipating human--human -turn dynamics in observer or mediation settings \cite{cano2026mmvap}. MuVAP -further extends VAP to multiparty interaction using a monaural mixed-audio -stream and a single camera, grounding acoustic dynamics in visible face tracks -and predicting current and future visible floor holders \cite{qi2026muvap}. -These formulations are complementary to our egocentric setting. In MuVAP, the -candidate future speakers belong to the set visible to the camera; in -mediation-oriented MM-VAP, the system observes turn dynamics between human -participants. Here, by contrast, the participant whose future floor entry must -be predicted is located \emph{behind the camera} and is therefore absent from -the visible participant set. +This role asymmetry exposes an adaptation problem for general-purpose Omni +models. Omni pretraining favors globally integrated representations useful for +multimodal understanding and generation. Egocentric floor control instead +depends on timely prosodic and visible-person activity at sub-second scale, +including whether other participants continue to occupy the floor. Increasing +model capacity or applying a generic parameter-efficient update does not +explicitly preserve or route this evidence to a dense wearer-relative +prediction head. The open question is therefore not merely whether an Omni +backbone can be applied to EgoSpeak, but what task structure is still required +to turn general audiovisual understanding into reliable floor control. -This leaves an underexplored intersection: continuous causal prediction of an -egocentric participant's own future floor participation in a changing -multiparty scene using a general-purpose audio--visual Omni model. The defining -difference is its viewpoint and prediction target: the model must project the -activity of the unseen first-person participant from the causal stream used to -observe everyone else. We investigate how pretrained Omni representations can -be adapted under the EgoSpeak wearer-relative forecasting objective. Current -Qwen2.5-Omni results motivate this study, but do not establish a limitation -shared by Omni models in general. +Our candidate decomposition separates long-context understanding from +short-window floor evidence. A +causal global Omni state represents the broader scene and conversation. In +parallel, a lightweight local path encodes recent mixed audio and a changing, +anonymous set of visible-person observations. A residual adapter makes this +local evidence accessible to the common ten-horizon activity head. The primary +variant deliberately avoids cross-time identity: the current development +oracle supports timely aggregate visible-other activity but not person +continuity. Track-aware structure is considered only if it defeats no-ID, +identity-corrupted, and capacity-matched controls. -Our working hypothesis separates broad context from timely floor evidence. A -causal global Omni state can represent the scene and conversational history, -while a lightweight local pathway preserves recent mixed audio and a changing -set of visible-person observations. The latter may be aggregated per frame or -organized into anonymous tracks; track continuity becomes a contribution only -if it outperforms aggregate, no-ID, shuffled, and capacity-matched controls. An -analysis-only oracle screen on held-out EasyCom Session~12 supports the weaker -hypothesis: perfectly timed aggregate visible-other activity improves a -matched global-plus-audio model by 6.97 points at 0.2~s and 2.91 points in -ten-horizon mean mAP, whereas person-resolved histories do not outperform an -identity-shuffled control. Thus the current evidence motivates timely local -floor information, not speaker identity or addressee understanding. - -We retain EgoSpeak-compatible dense future-activity prediction for -comparability and add a behavior-aligned event analysis. In particular, -per-frame average precision can hide the failure that motivates this work: -entering during an exchange among other participants. We therefore measure -false wearer entries during multi-other opportunities at matched wearer-onset -recall, together with response latency and wearer-yield behavior. These events -are observable timing-derived interaction proxies, not semantic EOT, INT, or -addressee labels; the final protocol uses competing-risk censoring so an event -cannot claim a later action across an incompatible floor change. This follows -the broader event-oriented emphasis on recall, false alarms, and latency -\cite{jiang2026turnbench}. - -Our contributions are: (1) a controlled study of frozen and -parameter-efficient Omni adaptation for EgoSpeak-style causal forecasting; -(2) an oracle information audit that distinguishes timely aggregate local -floor evidence from person continuity; (3) a global--local adaptation selected -through causal, alignment, and capacity-matched controls; and (4) an event -protocol that tests whether dense gains reduce false first-person floor entry -without suppressing valid entry or yield. EasyCom supplies synchronized -egocentric audio--video, per-person tracks, and voice activity -\cite{donley2021easycom}; target-of-speech annotations, when used, serve only -as analysis strata and never as model input or required output. +The final paper will claim the following three contributions only if the +corresponding TODO gates pass: +\begin{itemize} + \item We identify and test a temporal--role mismatch between a global Omni + readout and causal egocentric floor forecasting for an invisible wearer. + \item We evaluate a wearer-relative causal local floor adapter that combines + long-context Omni information with short-window audio and anonymous + visible-person evidence. + \item We evaluate whether the adapter reduces false wearer entries during + multi-other exchanges at matched valid-onset recall, using causal alignment + interventions and competing-risk-censored entry/yield analysis. +\end{itemize} \begin{figure}[t] \centering -\fbox{\parbox[c][1.05in][c]{0.94\columnwidth}{\centering -\todo{After model selection, draw the final system figure. Show the shared -interface: raw causal AV to a global Omni readout; a parallel recent-audio and -visible-person path; aggregate/no-ID/track-organized variants; residual fusion; -and the common $10\times3$ head. Draw the transcript-derived oracle only as a -dashed, analysis-only upper-bound path. Mark causal cutoffs, provided versus -online tracks, trainable modules, NULL/OFFSCREEN handling, and resets.}}} -\caption{Global context and causal local evidence for egocentric floor action. -Track organization is a controlled candidate, not a presupposed source of -speaker identity.} +\fbox{\parbox[c][1.00in][c]{0.94\columnwidth}{\centering +\todo{Draw only the selected method: causal AV to the long-context Omni path; +the short-window mixed-audio and anonymous person-set path; residual fusion; +the common $10\times3$ head; and the wearer-silent/active event interpretation. +Mark physical cutoffs, trainable modules, person masks, and NULL/OFFSCREEN. +Do not draw oracle labels, rejected track variants, or multiple backbones.}}} +\caption{Long-context Omni understanding and short-window local floor evidence +for wearer-relative future-activity prediction.} \label{fig:method} \end{figure} -\section{Methods} +\section{Method} \label{sec:method} -\subsection{Wearer-relative activity projection} +\subsection{Causal wearer-relative projection} -At decision time $t$, the model observes only the causal stream -$\mathcal{X}_{\leq t}$ and predicts $K=10$ future bins at $\Delta=0.2$~s. -For bin $k$, the target -$\vect{y}_{t,k}=[y^{\mathrm{bg}},y^{\mathrm{w}},y^{\mathrm{o}}]$ denotes +At decision time $t$, the model observes only the stream +$\mathcal X_{\leq t}$ and predicts $K=10$ future bins at $\Delta=0.2$~s. For +bin $k$, the multilabel target +$\vect y_{t,k}=[y^{\mathrm{bg}},y^{\mathrm{w}},y^{\mathrm{o}}]$ denotes background, camera-wearer activity, and activity by one or more other people. -Wearer--other overlap is allowed; hence this is not a three-way softmax. The -output is +Wearer--other overlap is allowed. The model outputs \begin{equation} - \hat{\vect{Y}}_t=f_\theta(\mathcal{X}_{\leq t})\in[0,1]^{K\times3}, - \qquad q_t=\hat y^{\mathrm{w}}_{t,1}, - \label{eq:task} + \hat{\vect Y}_t=f_\theta(\mathcal X_{\leq t}) + \in[0,1]^{K\times3}. + \label{eq:task} \end{equation} -where $q_t$ is the immediate wearer-entry score used by the event protocol. -It is interpreted as an \emph{entry} score only in wearer-silent opportunity -windows; elsewhere it remains a future-activity probability. This retains the -EgoSpeak target and evaluation while exposing a deployable 5-Hz decision -signal. +The same wearer-activity forecast has two controller interpretations. In a +wearer-silent opportunity, $\hat y^{\mathrm w}_{t,1}$ is an entry score. While +the wearer is active, $1-\hat y^{\mathrm w}_{t,1}$ is a near-term yield score. +The current self-state is used to define these observable event strata, not as +an unreported oracle model input. \todo{If the final adapter explicitly uses +self-state, define its strictly causal and deployment-available source and add +a matched no-state ablation; otherwise keep self-state evaluation-only.} -\subsection{Omni backbones and representation protocols} +\subsection{Global Omni baseline} -Let $m$ index an omni backbone and let $\vect{g}^{(m)}_t$ be a -backbone-specific causal global readout at the same physical cutoff $t$. -Qwen2.5-Omni \cite{xu2025qwenomni} is the currently implemented family; -Qwen2.5-Omni-3B, Qwen2.5-Omni-7B, and MiniCPM-o~4.5 -\cite{cui2026minicpmo45} form the planned initial comparison. These models have -different native streaming APIs, temporal tokenization, hidden dimensions, and -adaptation interfaces. We therefore emphasize within-backbone deltas rather -than treating raw cross-model scores as a pure architecture ranking. +Let $\vect g_t\in\mathbb R^{d_g}$ be the causal global readout of +Qwen2.5-Omni at the physical cutoff $t$ \cite{xu2025qwenomni}. The frozen +baseline trains only a dimension-matched prediction head. A matched PEFT/LoRA +baseline updates a pre-registered subset of the Omni model under the same +examples, context, optimizer budget, head, and evaluator. Its purpose is to +test whether generic parameter adaptation is sufficient; one negative recipe +is not interpreted as a general limitation of LoRA or all Omni models. -Every supported backbone is evaluated under three representation protocols: -(i) a frozen global readout with a dimension-matched prediction head; -(ii) a pre-registered parameter-efficient adaptation budget; and (iii) the -same global readout augmented by causal local evidence. Input cutoffs, EgoSpeak -targets, context duration, training examples, head capacity, and evaluation -are held fixed whenever the backbone permits it. A failure of one LoRA recipe -is reported as that recipe's result, not as proof that the backbone cannot -adapt. +\todo{Freeze the exact Qwen2.5-Omni-3B revision, processor, input rates, +streaming/cache semantics, readout layer, context, dtype, LoRA targets/rank, +trainable parameter budget, and cutoff audit. Add Qwen2.5-Omni-7B only as a +capacity-scaling control. Do not add another backbone family unless it is +needed to support and experimentally earns a cross-family claim.} -\todo{Implement and audit one adapter for each planned backbone. For every -model, document model revision, processor, native audio/video rates, exact -hidden-state readout, causal/streaming semantics, context limit, dtype, -trainable LoRA targets/rank, parameter count, peak memory, and P50/P95 latency. -Verify no future frames or samples enter $\vect g^{(m)}_t$. If a model cannot -expose a comparable causal state, report it as unsupported rather than inventing -a non-equivalent readout. Add future backbones only before the experiment -protocol is locked.} +\subsection{Causal local floor adapter} -\subsection{Global context and causal local evidence} - -The global state $\vect g^{(m)}_t$ is responsible for broad scene and -conversational context, but is not assumed to retain every localized cue needed -for floor control. From the most -recent $L_a$ seconds, a causal local audio encoder produces $\vect a_t$. At -time $t$, provided boxes during method isolation---and an online detector and -tracker in the deployable condition---produce a variable-size anonymous set -$\mathcal P_t=\{(\vect v_{t,i},\vect b_{t,i},\vect r_{t,i})\}_{i=1}^{N_t}$, -where $\vect v_{t,i}$ is a crop feature, $\vect b_{t,i}$ is box geometry, and -$\vect r_{t,i}$ contains only causal reliability variables such as visibility, -track age, and time since observation. A shared causal track encoder updates +The candidate local path operates on the most recent $L_\ell$ seconds. At local instant +$\tau\leq t$, a causal audio encoder produces $\vect a_\tau$ from mixed audio. +For the anonymous visible-person set +$\mathcal P_\tau=\{(\vect x_{\tau,i},\vect b_{\tau,i})\}_{i=1}^{N_\tau}$, +a shared visual encoder and masked set operator produce \begin{equation} - \vect h_{t,i}=F_{\mathrm{trk}}\!\left(\vect h^-_{t,i}, - W_v\vect v_{t,i}+W_b\phi(\vect b_{t,i})+W_r\vect r_{t,i}\right). - \label{eq:track} + \vect v_\tau=\mathrm{SetPool} + \left(\{E_v(\vect x_{\tau,i},\vect b_{\tau,i})}_{i=1}^{N_\tau} + \cup\{\vect v_\varnothing\}\right), + \label{eq:set} \end{equation} -All tracks share $F_{\mathrm{trk}}$; the temporary ID only retrieves -$\vect h^-_{t,i}$ and is never embedded. The global-audio query then reads the -current person set by masked cross-attention, -\begin{align} - \vect u_t &= \mathrm{MHA}\!\left( - W_q[\vect g^{(m)}_t;\vect a_t], - [\vect h_{\varnothing};\vect h_{t,1};\ldots;\vect h_{t,N_t}] - \right), \\ - \vect z^{(m)}_t &= \vect g^{(m)}_t+W_o\vect u_t, +where $\vect v_\varnothing$ handles an empty or offscreen visible set. The +operator is permutation invariant and uses no participant-ID embedding or +cross-time identity. A causal short-window encoder summarizes local dynamics, +\begin{equation} + \vect \ell_t=F_\ell + \left(\{[\vect a_\tau;\vect v_\tau]\}_{\tau\in(t-L_\ell,t]}\right). + \label{eq:local} +\end{equation} +The adapter preserves the global initialization through a zero-initialized +residual output, +\begin{equation} + \vect z_t=\vect g_t+W_o\, + \sigma\!\left(W_g\vect g_t+W_\ell\vect\ell_t\right), \qquad W_o\vert_{\mathrm{init}}=0, - \label{eq:adapter} -\end{align} -where $\vect h_{\varnothing}$ is a learned NULL/OFFSCREEN token. Padding is -masked, so the same network handles $N_t=0$ and changing group size. For a -bounded implementation, $N_t>N_{\max}$ is resolved by a deterministic ranking -using detector confidence and track age available at $t$, and truncation is -reported. The operator is permutation equivariant -\cite{lee2019settransformer}; temporary tracker IDs provide continuity, not -identity semantics. + \label{eq:fusion} +\end{equation} +and the common head maps $\vect z_t$ to Eq.~\eqref{eq:task}. This implements an +evidence-access hypothesis rather than adding an unconstrained second model: +the global path retains broad context, while the local path preserves timely +floor evidence. -Equation~\eqref{eq:track} is a candidate organization of local evidence, not a -claim that recurrent identity is necessary. Simpler alternatives replace each -$\vect h_{t,i}$ with a current-frame observation or pool the set into one -aggregate vector. This distinction is important because the current oracle -screen favors aggregate activity and does not separate correct from shuffled -identity histories. +\todo{Freeze $L_\ell$, local audio/video rates, crop source and resolution, +set pooling, causal temporal encoder, masks, reset rules, parameter count, and +latency. If the local path runs faster than the 5-Hz global/output grid, compare +it with a same-parameter 5-Hz variant before using the term ``dual-rate.'' Use +provided causal boxes only as a clearly labeled method-isolation condition; +online detection/tracking is not a paper contribution unless actually run.} -This decomposition is informed by MuVAP: its global audio representation -queries same-frame causal ASD track representations, followed by temporal -modeling and gated global-to-speaker fusion \cite{qi2026muvap}. We retain the -useful separation between global floor dynamics and person-organized evidence, -but not its role-relative output: our fixed target is the invisible wearer's -entry/yield action, not the next visible speaker. The zero-initialized residual -output follows the conservative adapter principle used to inject visual -evidence without perturbing the base prediction at initialization -\cite{rouditchenko2024whisperflamingo}. Unlike a zero scalar gate, it permits -the residual output layer to receive a gradient immediately. Local evidence -is derived from the same causal raw media, not close microphones, transcripts, -speaker labels, global identity anchors, or future frames. +\subsection{Training} -We evaluate candidates in increasing structural complexity: pooled recent -audio/vision; a capacity-matched temporal branch; a per-frame person set with -no cross-time identity; a shared track-wise temporal encoder; and person--audio -cross-attention as in Eqs.~\eqref{eq:track}--\eqref{eq:adapter}. Only a -candidate that improves the pre-registered event metric over simpler, -aggregate, identity-shuffled, and capacity-matched controls becomes the final -method. If person continuity does not separate from those controls, the final -model remains a global-plus-local aggregate floor-evidence model. A limited latent -bottleneck is tested only if large visible sets make full person attention a -runtime bottleneck; it is not assumed to improve accuracy. - -\todo{Run the candidate ladder before fixing $H_\phi$. Match trainable -parameters and effective local history. Compare a shared local encoder across -backbones (cross-model control) with each backbone's native local encoder -(within-model control); do not mix these protocols in one claimed delta. -For track candidates, specify visibility/age masks, gap/source resets, -OFFSCREEN handling, and global-rename invariance. Describe only the selected -architecture as the final method; move rejected candidates to ablations.} - -\subsection{Training objectives} - -The common main loss is class-balanced binary cross entropy over all horizons -and activity channels. This dense EgoSpeak target is the only supervision -required by the task. If a person-aware candidate survives the activity-only -comparison, an optional auxiliary head predicts current active-speaker state: +The primary objective is class-balanced binary cross entropy over valid +horizons and activity channels, \begin{equation} - \mathcal{L}=\mathcal{L}_{\mathrm{act}}+ - \lambda_{\mathrm{asd}}\mathcal{L}_{\mathrm{asd}}. + \mathcal L_{\mathrm{act}}= + \sum_{t,k,c}m_{t,k}w_c\, + \mathrm{BCE}(\hat y^c_{t,k},y^c_{t,k}). \label{eq:loss} \end{equation} -Participant ID, target-of-speech, transcript, close-microphone audio, and -future frames are never model inputs. Target-of-speech may stratify qualitative -analysis but is not a predicted variable or a requirement for the primary -multi-other event definition. - -\todo{First compare all representation/adaptation protocols with activity loss -only. Then ablate +ASD for the selected person-aware candidate. Add an -event-specific wearer onset/yield loss only as a separately reported extension. -Select all coefficients and early stopping on a training-side calibration -fold; do not inspect Sessions 1--3 during selection.} - -\subsection{Track source and deployment contract} - -The intended inference pipeline runs a face/head detector on each incoming -frame and associates detections with an online multi-object tracker using only -observations at or before $t$. Its temporary IDs route the shared recurrent -state in Eq.~\eqref{eq:track}; they do not denote names, fixed seats, or -cross-session identities. New tracks initialize from a learned state, short -misses retain a causally aged state, and expired tracks are removed. With no -visible face, the global/audio path and NULL/OFFSCREEN token remain valid. No -future-frame smoothing, full-video identity anchor, or cross-session -recognition is permitted. EasyCom state is continued only across genuinely -adjacent source segments and reset across redacted gaps; Ego4D state is local -to a clip. - -During method isolation, dataset-provided boxes and anonymous IDs form a -separate \emph{provided-track} condition. This answers whether person -structure can help under accurate localization, but is not reported as online -detector performance. The deployable \emph{online-track} condition runs the -detector/tracker above and includes its errors and latency. This separation is -important because MuVAP used random full-video identity anchors to build its -dataset tracks; that preprocessing is useful for annotation but is not a -strictly online association algorithm \cite{qi2026muvap}. - -\todo{Select and freeze the face/head detector, causal tracker, thresholds, -maximum age, and $N_{\max}$ on training-side data. Report detector recall, -provided/online-track coverage, mean/maximum people, truncations, ID switches, -fragments, occlusion gaps, source resets, and P50/P95 detector--tracker latency -for both datasets. Compare provided tracks, online tracks, and controlled box -jitter/dropout/fragmentation. Until the online row is run, limit deployment -claims to the intended pipeline and label all numbers ``provided-track.''} +No transcript, target-of-speech label, close microphone, future frame, or +per-person speech label enters the deployable model. \todo{Add an active-speaker +or event auxiliary loss only if it improves the frozen primary protocol and +its label/input boundary is fully reported.} \section{Experiments} -\label{sec:setup} +\label{sec:experiments} -\textbf{Data and splits.} -We use the EgoSpeak EasyCom protocol: Sessions 4--12 for model development and -Sessions 1--3 for final reporting. All architecture, checkpoint, threshold, -and hyperparameter choices use training-side folds only. Inputs contain the -camera stream and shared array audio available to the wearer; close-talking -microphones, per-person voice labels, and target-of-speech labels are not model -inputs. We also report aggregate activity forecasting on the EgoSpeak Ego4D -split \cite{grauman2022ego4d}. Ego4D Audio-Visual Diarization supplies -clip-local anonymous participant tracks; its per-person voice coverage on the -433-clip EgoSpeak split must be audited before reporting the same multi-other -event analysis. +\textbf{Data and split.} +We follow the EgoSpeak EasyCom split: Sessions 4--12 for training-side model +development and Sessions 1--3 for final reporting \cite{donley2021easycom}. +Thresholds, windows, checkpoints, loss weights, and early stopping are selected +without using the official test sessions. Already observed test results are +treated as exploratory internal evidence and are not included in the paper. +\todo{Report duration, frame/event counts, class prevalence, visibility and +box coverage, source discontinuities, and the frozen calibration fold.} -\textbf{Oracle information audit.} -Before selecting a learned local module, we measure what information could in -principle complement the global state. Analysis-only sidecars expose -transcript-derived speech activity no later than the current cutoff under six -conditions: local audio, visibility only, one-second-delayed activity, timely -aggregate visible activity, per-person histories, and histories with their -cross-time identity assignments shuffled. These labels never enter a -deployable model. The audit separates three hypotheses that ordinary feature -ablation conflates: the value of seeing people, the value of knowing current -visible-other activity, and the incremental value of preserving who produced -that activity. +\textbf{Baselines.} +The confirmatory comparison contains: an EgoSpeak reproduction; the frozen +Qwen2.5-Omni-3B global state $G$; matched LoRA/PEFT; $G$ plus recent local +audio; a parameter- and history-matched temporal local branch; and the proposed +audio--visible-person adapter. Qwen2.5-Omni-7B is a capacity-scaling control, +not a second paper storyline. \todo{Freeze all baseline implementations and +ensure the EgoSpeak comparison uses identical targets, split, causal validity, +and evaluation.} -\textbf{Backbone comparison.} -The initial study covers Qwen2.5-Omni-3B, Qwen2.5-Omni-7B, and -MiniCPM-o~4.5. For each backbone $m$, we first compare its frozen causal state -$G_m$, a parameter-efficiently adapted state $P_m$, and local-evidence -augmentation $G_m+L$. The primary diagnostic is the within-backbone change; -cross-backbone absolute scores are secondary because native tokenization, -pretraining, streaming interfaces, and hidden-state semantics differ. Common -local encoders test whether adaptation transfers across global backbones, -whereas native local encoders are a separate, model-specific protocol. All -rows share physical input cutoffs, targets, data splits, and evaluation. -Unsupported causal readouts are disclosed rather than replaced by offline or -future-conditioned features. +\textbf{Event protocol.} +Dense ten-horizon AP/mAP is retained for EgoSpeak comparability. The implemented +competing-risk evaluator counts each eligible aggregate-other boundary once, +ends a response window before the next other onset, and ends a yield window +before the next other offset. Pending the remaining P0 real-data audit and +scenario freeze, the intended primary behavior metric is false wearer entries per +minute of multi-other opportunity at a training-side selected wearer-onset +recall, with $R=.8$ as the headline operating point. The intended analysis also +reports onset latency and wearer-yield recall/latency, following the +event-oriented emphasis on recall, false alarms, and latency +\cite{jiang2026turnbench}. The protocol will censor a response window after an +other offset at the next other onset, and a yield window after an other onset +at the next other offset. -\todo{Implement the three backbone adapters and validate state timestamps with -an impulse/cutoff test. Pre-register one matched PEFT budget and one common -local branch. Report native-vs-common local encoders in separate table panels. -Keep future model additions outside the confirmatory comparison once this -matrix is locked.} +\todo{P0 blocker: run the tested competing-risk evaluator on real predictions, +manually audit sampled timelines, and freeze pairing before treating any event +score as a paper result. Specify overlap, +minimum duration, timeouts, duplicate detections, source boundaries, offscreen +cases, and event counts. Until this is complete, call existing values +timing-derived proxies rather than final event results.} -\textbf{Event construction.} -We derive \emph{wearer onset} events directly from the wearer activity target. -A \emph{multi-other exchange} is a wearer-silent opportunity interval in which -at least two distinct non-wearer participant IDs contribute temporally ordered -speech activity. It is an observable interaction proxy, not a claim about the -speakers' mental intent or addressee. Threshold $\tau_r$ is selected on the -development split to obtain wearer-onset recall -$r\in\{0.7,0.8,0.9\}$ and is then frozen. The primary metric is false wearer -entries per minute of multi-other opportunity at matched onset recall; the -headline operating point is $r=0.8$. We also report signed onset latency, -wearer-yield recall/latency, and the original ten-horizon per-frame mAP. - -\todo{Freeze the event pairing and competing-risk rules, including the -multi-other alternation window, minimum speech/silence, overlap, maximum onset -delay, duplicate detections, one detection per event, track visibility, source -boundaries, and group/offscreen cases. Publish train/test onset counts, -multi-other events/minutes, number of distinct visible speakers, and exclusion -counts. Manually audit at least 100 events. Treat all previously viewed -Sessions 1--3 scores as exploratory; lock the protocol and model before the -confirmatory three-seed run.} - -\textbf{Baselines and controls.} -For each backbone we compare: frozen global state $G_m$; PEFT $P_m$; $G_m$ -plus recent local audio; and $G_m$ plus pooled local audio--visual evidence. -This compact matrix tests the representation hypothesis before investing in a -complex fusion module. The first learned target after the oracle audit is a -causal estimator of timely aggregate visible-person floor evidence. Only if -person-organized features survive three seeds, visual time/person corruptions, -and the event gate do we compare a parameter-matched temporal branch without -person continuity, a separately trained per-frame/no-ID set branch, and a -track-consistent masked cross-attention branch. Provided-track and online-track -inputs are reported separately. On a frozen track-aware checkpoint, a clip-consistent -global renaming must leave outputs unchanged, whereas \emph{broken -continuity} periodically remaps IDs while preserving every crop, timestamp, -audio sample, and target. Time-shifted and removed vision provide additional -alignment controls. - -\todo{Define the broken-continuity corruption before evaluation: remap only at -pre-registered 1-s boundaries or report a fixed ID-switch-rate curve. Do not -call a clip-consistent slot permutation ``broken ID.'' Verify unchanged crops, -frame counts, audio, labels, and visibility masks. Train the no-ID/reset -baseline from scratch with matched parameters; evaluate all test-time -counterfactuals using one frozen checkpoint and one frozen threshold.} - -\textbf{Implementation and statistics.} -All prediction heads run at 5~Hz and use an 8-s causal observation window; -model-specific deviations required by a native streaming interface are stated -explicitly. In the frozen protocol the omni backbone does not update. In PEFT -and local protocols, trainable parameters and optimizer exposure are reported -separately. Every temporal state resets at non-contiguous source boundaries. -We report three seeds and mean$\pm$standard deviation. Confidence intervals for -event metrics use paired cluster bootstrap over sessions/conversation blocks -rather than treating adjacent frames as independent. Runtime includes P50/P95 -decision latency, throughput, peak memory, and trainable parameter count. - -\todo{Fill optimizer, learning rate, batch size, epochs, feature dimensions, -$L_a$, crop rate/resolution, number of heads/layers, $N_{\max}$, parameter -count, tracker timeout/ranking, hardware, and end-to-end real-time latency -after the architecture is frozen.} +\textbf{Controls and statistics.} +The main intervention compares correctly aligned local evidence with +time-shifted evidence, removed vision, global-frame visual pooling, and the +capacity-matched temporal branch. A same-rate control is required for a +dual-rate claim. Track/no-ID/broken-continuity controls are added only if a +track-aware variant first wins on training-side data. We report three seeds, +mean$\pm$sample standard deviation, and paired cluster-bootstrap intervals over +sessions or conversation blocks. \todo{Freeze seeds, bootstrap unit/count, +optimizer, epochs, hardware, throughput, P50/P95 latency, and trainable +parameters.} \section{Results and Analysis} \label{sec:results} \begin{table}[t] -\caption{Causal oracle-information screen on EasyCom development Session~12 -(seed 0, four epochs). Oracle speech activity is analysis-only. Values are -percentages under the causal-valid protocol.} -\label{tab:oracle} -\centering -\small -\begin{tabular}{lcc} -\hline -Input condition & 0.2-s mAP & Mean mAP\\ -\hline -Global + local audio & 72.58 & 58.77\\ -Visibility only & 71.95 & 58.86\\ -Activity delayed by 1 s & 72.02 & 58.90\\ -Aggregate visible activity & \textbf{79.55} & \textbf{61.68}\\ -Per-person activity histories & 78.80 & 61.11\\ -Identity-shuffled histories & 78.86 & 61.19\\ -\hline -\end{tabular} -\end{table} - -Table~\ref{tab:oracle} establishes an information opportunity, not a deployable -result. Timely aggregate activity among visible non-wearers provides a large -gain, while visibility alone and stale activity do not. However, preserving -person-resolved history gives no advantage over destroying its cross-time -identity assignment, and both trail the aggregate representation. The learned -method must therefore first recover useful local floor evidence; person-track -continuity is retained only if subsequent no-ID and broken-track controls -separate. - -\begin{table}[t] -\caption{Exploratory Qwen2.5-Omni-3B local-branch evidence on EasyCom test -Sessions 1--3. Values are percentages from one seed, not a cross-backbone or -confirmatory result.} -\label{tab:preliminary} -\centering -\small -\resizebox{\columnwidth}{!}{% -\begin{tabular}{lccc} -\hline -Model & 0.2-s mAP & 0.2-s wearer AP & Mean mAP\\ -\hline -Qwen-3B $G$+local A & 81.72 & 85.07 & 65.11\\ -Qwen-3B $G$+local AV-pool & 82.03 & 85.09 & 65.40\\ -\hline -$\Delta$ visual & +0.30 & +0.02 & +0.29\\ -\hline -\end{tabular}} -\end{table} - -Table~\ref{tab:preliminary} shows why aggregate mAP alone is insufficient. -Pooled vision gives a small gain, while immediate wearer AP is nearly -unchanged. This is compatible with either useful generic local context or -regularization; it is not evidence of person grounding. Only matched-recall -event evaluation and visual-alignment controls can distinguish these cases. -A separate Qwen-3B dense-cache diagnostic gave 58.16 mean mAP for its frozen -readout and 56.20 for one LoRA recipe. Because that diagnostic used a different -feature protocol, it is not combined numerically with -Table~\ref{tab:preliminary}; it only shows why a fair, protocol-matched PEFT -study remains necessary. - -\begin{table}[t] -\caption{Planned cross-backbone EasyCom event evaluation. Lower -$\mathrm{FP}_{\mathrm{mo}}$ is better; all rows use the same -development-selected operating protocol. ``Local'' denotes the train-side -selected common local-evidence candidate.} +\caption{Confirmatory EasyCom results. Lower multi-other false entries are +better; all event rows use a training-side selected operating point.} \label{tab:main} \centering \small \resizebox{\columnwidth}{!}{% \begin{tabular}{lccccc} \hline -Backbone/protocol & $\mathrm{FP}_{\mathrm{mo}}$/min@$R=.8$ & Onset R & -Onset latency & Yield R & Mean mAP\\ +Method & 0.2-s mAP & Mean mAP & FP/min@$R=.8$ & Onset lat. & Yield R\\ \hline -Qwen-3B frozen & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\ -Qwen-3B PEFT & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\ -Qwen-3B + local & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\ -Qwen-7B frozen & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\ -Qwen-7B PEFT/local & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\ -MiniCPM-o frozen & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\ -MiniCPM-o PEFT/local & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\ +EgoSpeak reproduction & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\ +Omni-3B frozen $G$ & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\ +Omni-3B matched PEFT & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\ +$G+$local audio & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\ +$G+$matched temporal local & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\ +$G+$causal local AV (ours) & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\ +Omni-7B scaling control & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--}\\ \hline \end{tabular}} \end{table} -\todo{Report Table~\ref{tab:main} as mean$\pm$standard deviation over three -seeds, plus 95\% paired cluster-bootstrap confidence intervals for frozen vs -PEFT and frozen vs local within each backbone. A general omni-representation -claim requires a consistent pattern across at least two backbone families; an -effect confined to Qwen must be stated as Qwen-specific. The behavior claim is -supported only if multi-other FP/min decreases at matched wearer-onset recall -without worse onset latency or wearer-yield recall. If the selected local -candidate uses tracks, also report its paired intervals against no-ID and -temporal-capacity controls.} +\textbf{Information opportunity.} +On development Session~12, an analysis-only seed-0 oracle supplied perfectly +timed visible-other speech activity through an explicit non-deployable path. +Aggregate visible activity improved 0.2-s mAP by 6.97 points and ten-horizon +mean mAP by 2.91 points over matched global-plus-local-audio. Visibility-only +and one-second-delayed activity remained near audio, while per-person histories +did not outperform identity-shuffled histories. This motivates timely local +floor evidence, not participant identity. \todo{After seeds 1--2, report the +interval or remove the exact oracle numbers from the four-page paper.} + +\textbf{Main result.} +Table~\ref{tab:main} should establish one result: \todo{the selected local +adapter reduces multi-other false entry at matched valid-onset recall, retains +onset latency/yield behavior, improves dense forecasting, and recovers a +meaningful fraction of the oracle opportunity}. The 7B row should determine +whether \todo{capacity alone closes the gap or the adaptation structure remains +useful at larger scale}. No conclusion is drawn across all Omni families from +Qwen-only evidence. \begin{table}[t] -\caption{Conditional track-continuity counterfactual on the best track-aware -model. This table is retained only if train-side selection chooses such a -candidate; all rows reuse one frozen checkpoint.} -\label{tab:counterfactual} +\caption{Mechanism controls for the selected Omni-3B local adapter.} +\label{tab:ablation} \centering \small -\resizebox{\columnwidth}{!}{% \begin{tabular}{lccc} \hline -Track/visual evidence & $\mathrm{FP}_{\mathrm{mo}}$/min & Onset recall & 0.2-s mAP\\ +Local condition & Mean mAP & FP/min@$R=.8$ & Yield R\\ \hline -Correct tracks & \todo{--} & \todo{--} & \todo{--}\\ -Global ID rename & \todo{--} & \todo{--} & \todo{--}\\ -Broken continuity & \todo{--} & \todo{--} & \todo{--}\\ -Time shifted & \todo{--} & \todo{--} & \todo{--}\\ -Removed & \todo{--} & \todo{--} & \todo{--}\\ +Correct causal local AV & \todo{--} & \todo{--} & \todo{--}\\ +Remove local vision & \todo{--} & \todo{--} & \todo{--}\\ +Time-shift local vision & \todo{--} & \todo{--} & \todo{--}\\ +Global/full-frame visual pool & \todo{--} & \todo{--} & \todo{--}\\ +Same-rate/capacity control & \todo{--} & \todo{--} & \todo{--}\\ \hline -\end{tabular}} +\end{tabular} \end{table} -If a track-aware candidate is selected, Table~\ref{tab:counterfactual} is the -continuity test. Global renaming must be -numerically invariant because IDs carry no semantics. If the model exploits -person history, correct tracks should outperform broken continuity even though -both contain exactly the same per-frame people. If broken, time-shifted, and -removed variants perform similarly to correct tracks, the defensible -conclusion is additional capacity or generic visual context, not useful -person-continuous reasoning. +\textbf{Mechanism and limitations.} +Table~\ref{tab:ablation} should test whether gains require correctly timed local +visual evidence rather than extra parameters, regularization, or generic scene +context. \todo{Report paired intervals and the supported interpretation. If +correct and shifted/removed vision do not separate, drop the audiovisual +evidence-routing claim. If dense mAP improves but event behavior does not, do +not claim improved floor control. If legitimate onset/yield degrades, stop the +adapter rather than describing lower false entry as success.} -\todo{Add one compact error analysis split by model family, track visibility, -number of visible people, overlap, track age/fragmentation, and -backchannel/interruption proxy. Include event counts with every rate. Move -rejected PEFT recipes and fusion candidates to supplementary material unless -they directly explain the selected method.} - -\textbf{Qualitative evidence.} -For a multi-other exchange, visualize annotation-derived per-person activity -as an analysis reference, provided track IDs when applicable, wearer ground -truth, the frozen-global and selected-local wearer-entry scores, and the single -frozen threshold on one physical time axis. If the selected candidate is -track-aware, also show its broken-continuity score. A useful -example shows the frozen baseline rising at an other-speaker boundary while -the selected adapter remains below threshold; a companion positive example -must show that a real wearer onset is not suppressed. - -\todo{Do not hand-pick only successful clips. Before viewing the final test -plots, freeze a deterministic selection rule and show: largest paired -improvement, near-median improvement, one failure, and one correctly retained -wearer onset. Include full local timelines and source frame times, not cropped -screenshots without context. State that examples illustrate the aggregate -event result and cannot establish it alone.} +\todo{Add one compact, pre-selected qualitative timeline only if space permits: +one multi-other false-entry improvement paired with one preserved real wearer +onset or one failure. Examples illustrate aggregate rates and never establish +the claim by themselves.} \section{Conclusion} \label{sec:conclusion} -We study when an egocentric agent should enter or yield a multiparty floor, not -merely whether someone will speak. The working decomposition is global Omni -context plus causal local audiovisual floor evidence. The current -Qwen2.5-Omni-3B oracle audit supports the second term: timely aggregate -visible-other activity is informative beyond the global state and recent -mixed audio. It does not support the stronger claim that participant identity -or track continuity adds value, because per-person histories do not beat an -identity-shuffled control. Dense activity forecasting supplies a comparable -target, while matched-recall event evaluation must determine whether a learned -local estimator actually prevents intrusions during multi-other exchanges -without suppressing valid wearer entry or yield. \todo{After the controlled -study, state which learned local variant was selected, how much of the oracle -gap it recovered, whether the pattern generalized beyond Qwen, the -quantitative false-entry and onset/yield results, and the main deployment -limitation. If no learned variant passes these tests, conclude with the -information audit rather than claiming a track-based solution.} +This study asks whether a general-purpose Omni representation can be adapted +for causal floor forecasting of an invisible egocentric wearer. The candidate +method separates long-context Omni understanding from short-window local floor +evidence without assuming persistent speaker identity. \todo{State the final +learned-method improvement, matched-recall false-entry result, onset/yield +non-degradation, causal-control evidence, and main limitation. If no learned +variant passes these tests, reframe the submission as an information audit or +do not make the method claim.} \bibliographystyle{IEEEbib} \bibliography{references} diff --git a/reports/omni_turn_research_direction_2026-08-26.md b/reports/omni_turn_research_direction_2026-08-26.md new file mode 100644 index 0000000..8bfe3e2 --- /dev/null +++ b/reports/omni_turn_research_direction_2026-08-26.md @@ -0,0 +1,353 @@ +# Frozen Omni turn anticipation: evidence, trends, and next direction + +Date: **2026-08-26** + +This memo combines the current EasyCom experiments, a read-only audit of the +local EasyCom/Ego4D/EgoSpeak copies under `/Volumes/MX500B/Code`, and recent +work on streaming multimodal interaction. Nothing on the NTFS volume was +modified. + +## Executive decision + +The most defensible ICASSP direction is **not** to continue scaling the +Qwen2.5-Omni backbone or to fine-tune more of it. The evidence instead supports: + +> **A frozen causal Omni backbone with a fine-grained, transition-aware +> multimodal decision readout for egocentric future-speaker anticipation.** + +The current 4-epoch last-four-layer concat-MLP reaches `60.87144` mean mAP, +slightly above EgoSpeak's reported EasyCom GRU A+V result (`60.6`), while using +the Omni Transformer itself for temporal modeling and no additional RNN. It is +not yet a clean paper result: it is single-seed and the epoch budget was chosen +after looking at Sessions 1--3. Its main scientific value is the diagnosis: + +1. useful turn information is distributed across the upper Omni layers; +2. a final-layer query alone under-exposes that information; +3. unconstrained high-capacity readout training overfits the nine training + conversations very quickly; +4. larger 7B features and Thinker LoRA do not solve the problem. + +The next contribution should therefore target **how to expose and supervise +the pretrained representation**, not how to make the backbone larger. + +## Current result ladder + +Selected EasyCom full-timeline results on Sessions 1--3: + +| System | 0.2s mAP | Mean mAP | Interpretation | +|---|---:|---:|---| +| EgoSpeak Transformer A+V | 73.8 | 58.7 | best reported EgoSpeak 0.2s result | +| EgoSpeak GRU A+V | 73.5 | 60.6 | best reported EgoSpeak mean result | +| Frozen Omni-3B, final-layer query | 71.99768 | 58.51937 | initial 8s reference | +| Frozen Omni-3B, last-4 scalar mixture | **73.45085** | 60.53423 | recovers the short-horizon deficit | +| Frozen Omni-3B, last-4 concat-MLP, 4 epochs | 73.42020 | **60.87144** | exploratory project best | +| Same concat-MLP, 12 epochs | 72.44507 | 57.98809 | clear overfitting | +| Same concat-MLP, 50 epochs | 66.54645 | 53.03515 | severe overfitting | +| Frozen Omni-7B, reported run | 69.80133 | 57.67367 | larger backbone is worse | + +The concat-MLP result is archived in +[`results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/`](../results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/README.md). + +Compared with the scalar mixture, concat-MLP gains only `+0.33721` mean mAP +and loses `0.03065` at 0.2 seconds. The added capacity mainly helps around +1.2--1.6 seconds. This is useful but too small to justify a 16.8M-parameter +fusion module without a cleaner validation protocol and stronger regularization. + +## What the local data actually imply + +### EasyCom: many frames, few independent conversations + +The EgoSpeak EasyCom targets contain: + +| Split | Sessions | 5Hz steps | Duration | Background | Wearer | Others | +|---|---:|---:|---:|---:|---:|---:| +| Train (4--12) | 9 | 72,405 | 4.0225h | 22.57% | 24.93% | 60.50% | +| Test (1--3) | 3 | 23,048 | 1.2804h | 22.91% | 26.74% | 58.71% | +| Total | 12 | 95,453 | 5.3029h | -- | -- | -- | + +Wearer and other-speaker labels overlap in about `8.08%` of all steps, so this +is intrinsically a multi-label conversational-state problem even though the +EgoSpeak implementation applies a three-way softmax objective. + +This exposes an immediate objective mismatch. The observed target rows contain +exactly four conversational states: + +1. `[1, 0, 0]`: silence/background; +2. `[0, 1, 0]`: wearer only; +3. `[0, 0, 1]`: other people only; +4. `[0, 1, 1]`: wearer and others overlap. + +A three-way softmax cannot assign high probability to both active-speaker +classes. In the overlap state, the LSTR loss used by the current project sums +`-log p_wearer - log p_others`, forcing the two positive classes to compete. +This is especially relevant because wearer AP is the class that collapses most +under overfitting. + +A structured four-state head is a cleaner formulation: predict a softmax over +`silence / wearer-only / others-only / overlap`, then expose evaluator scores +as `p_background=p_silence`, `p_wearer=p_wearer-only+p_overlap`, and +`p_others=p_others-only+p_overlap`. This retains a normalized state model while +matching the multi-positive EgoSpeak labels exactly. + +The important limitation is the effective sample size. Adjacent 200ms anchors +are strongly correlated, and there are only nine independent training +sessions. Session-level variation is large: + +- wearer prevalence ranges from `10.37%` to `37.70%`; +- other-speaker prevalence ranges from `47.71%` to `77.17%`; +- overlap ranges from `1.42%` to `20.35%`; +- label-change frequency ranges from about `26` to `35` events per minute. + +This explains why a 16.8M concat projection can reduce training loss from +`0.806973` to `0.300277` while test mean mAP falls by `7.83629` points. The head +can learn conversation/session priors rather than a transferable switching +rule. + +The data are nevertheless rich in transitions. Depending on the session, +roughly `54%`--`69%` of anchors have at least one label change in their next +two seconds. The next method should not simply oversample all transition-near +frames; it should explicitly model **persistence versus switching**. + +### Existing annotations support stronger auxiliary supervision + +The raw EasyCom copy contains 323 transcript JSON files with: + +- 6,981 utterances and about 52,171 transcribed words; +- speaker ID and frame-accurate start/end timestamps; +- Target-of-Speech labels on 6,963 utterances; +- 6DoF tracked participant pose at the original 20 FPS; +- participant-linked head and face boxes. + +These annotations permit auxiliary targets without new manual labeling: + +- current conversational state; +- keep/switch state between adjacent 200ms steps; +- time-to-next-change bins; +- utterance completion/onset proximity; +- addressee type (group versus individual); +- head-orientation change and target-of-speech prediction. + +Transcript text should be used as **training-only privileged information** or a +distillation target if the final online model must consume raw audio/video only. +That would demonstrate a semantic advantage without giving Omni unavailable +future text at test time. + +### Ego4D is the more convincing generalization test + +The split used by EgoSpeak's reported experiment contains 346 train and 87 +test clips, almost all 5 minutes long: + +| Split | Clips | 5Hz steps | Duration | Background | Wearer | Others | Overlap | +|---|---:|---:|---:|---:|---:|---:|---:| +| Train | 346 | 518,881 | 28.83h | 43.31% | 23.24% | 40.53% | 5.58% | +| Test | 87 | 130,472 | 7.25h | 42.41% | 26.30% | 38.81% | 5.63% | + +This provides far more independent clips and substantially less split shift +than EasyCom at the session level. A method that improves only EasyCom may be +learning its fixed around-table conversational geometry. A method that also +improves Ego4D supports the stronger claim that the readout exposes general +Omni interaction cues. + +## Research trend and what transfers to this project + +### 1. Explicit decision heads are replacing implicit generation triggers + +[ROMA](https://arxiv.org/abs/2601.10323) processes aligned streaming +audio-video units, preserves 40ms audio temporal position IDs, and places a +lightweight speak head parallel to the LM head. It forms the head input as a +learned mixture of the last four layers. This strongly validates the project's +last-four-layer query experiment, while also suggesting that the low-capacity +scalar mixture is a more natural primary model than concat-MLP. + +[Proact-VL](https://arxiv.org/abs/2603.03447) inserts a dedicated FLAG token, +reads it with a small gated MLP, and separates the decision of when to respond +from response generation. [JoyAI-VL-Interaction](https://arxiv.org/abs/2606.14777) +similarly makes silence an explicit learned action rather than an external +heuristic. The shared trend is a small, explicit decision interface on top of +a general multimodal backbone. + +### 2. Transition supervision is becoming central + +Proact-VL argues that the important imbalance is not merely class frequency, +but **state transition versus state persistence**. It upweights switch steps +and adds two stability terms: local probability consistency during persistent +states and global speaking-rate calibration. This maps directly onto the +EasyCom structure and is the highest-priority idea for improving 0.2-second +prediction. + +The [BadCat unit-based agent](https://arxiv.org/abs/2601.20230) reaches a +similar conclusion from a systems perspective: it represents dialogue control +as `keep-listen`, `listen-to-speak`, `keep-speak`, and `speak-to-listen` actions. +That is a strong conceptual justification for adding a keep/switch auxiliary +head rather than another generic hidden layer. + +### 3. Frozen backbones plus task-specific fusion are credible + +[MM-When2Speak](https://arxiv.org/abs/2505.14654) formulates timing as dense +response-type classification and uses adaptive attention over visual, audio, +and textual tokens. Its response fusion stage freezes the LLM and modality +encoders and optimizes a lightweight attention module. This is closely aligned +with our empirical finding that frozen Omni is more stable than Thinker LoRA. + +The important next ablation is therefore not another larger query MLP. It is a +comparison between: + +1. final query state; +2. learned last-four query mixture; +3. recent 200ms audio-token pooling; +4. recent 200ms audio-video token attention; +5. query plus recent-token pool. + +For token pooling, only payload tokens in the causal interval should be used, +selected by TMRoPE time IDs and excluding padding/template tokens. + +### 4. Fine temporal cadence is a differentiator + +Most proactive video systems make decisions at one-second cadence. EgoSpeak +and this project operate at 200ms. That is not merely an implementation detail: +it creates a credible ICASSP contribution around fine-grained conversational +switch anticipation, especially because Omni audio tokens retain finer 40ms +temporal positions. + +The paper should emphasize that a two-second multimodal ingestion unit does +not force a two-second decision rate. Causal readout at each 200ms anchor can +still exploit fine-grained audio timing, provided the state/readout does not +collapse the current-tail tokens. + +## Recommended paper contribution + +Working framing: + +> **Fine-Grained Causal Readout of Frozen Omni Models for Egocentric Turn +> Anticipation** + +Possible three-part contribution: + +1. **Streaming formulation.** Raw egocentric audio/video is encoded by a + frozen Omni Transformer with bounded causal memory, producing decisions at + 5Hz without an external RNN. +2. **Hierarchical temporal readout.** A parameter-efficient upper-layer query + mixture is fused with recent TMRoPE-localized audio/video payload tokens, + exposing both semantic context and immediate prosodic/nonverbal cues. +3. **Structured transition-aware decoding.** A four-state overlap-aware output + is marginalized to EgoSpeak's three AP classes, while existing frame labels + generate current, keep/switch, and time-to-change supervision with + persistence consistency and session-aware validation. + +The negative results become useful evidence rather than discarded runs: + +- 7B is worse than 3B; +- Thinker LoRA causes representation drift and worse test mAP; +- concat-MLP helps briefly but collapses with longer training; +- a tiny last-four mixture recovers almost all of the useful gain. + +Together they support the thesis that **the bottleneck is task-aligned causal +readout and supervision, not foundation-model capacity**. + +## Prioritized experiment plan + +### P0: make the current gain publishable + +1. Train on Sessions 4--11 and use Session 12 only as development data. +2. Use one fixed 12-epoch schedule and save every epoch. Do not compare runs + whose cosine schedule changes with the requested epoch count. +3. Select epoch and threshold using a predeclared development metric. +4. Retrain on Sessions 4--12 for the selected epoch count, then evaluate + Sessions 1--3 once. +5. Run at least three seeds; EgoSpeak reports five-seed summary statistics. +6. Match single-query, fixed mean, scalar mixture, and concat-MLP schedules. + +Primary selection metric should be `0.5 * mAP@0.2s + 0.5 * mean_mAP`, or both +metrics should be reported as a Pareto pair. Selecting only mean mAP can hide +the exact responsiveness regression that matters most to this project. + +Before changing the representation, run an identical-output-objective +ablation on the frozen final-layer and scalar-mixture features: + +| Output formulation | Training loss | Evaluation scores | +|---|---|---| +| Official control | current three-logit LSTR MCE | three-way softmax | +| Independent multi-label | BCE on wearer/others | sigmoid scores; background derived from neither | +| Structured state | four-way CE on silence/wearer/others/overlap | three marginals | + +The structured state variant is preferred because it represents overlap +without abandoning a coherent normalized conversational state. If it improves +the scalar-mixture model, carry it into every later experiment. + +### P1: low-capacity temporal readout + +Run these in order: + +| Readout | Approx. new fusion parameters | Purpose | +|---|---:|---| +| Last-4 scalar mixture | 4 | robust reference | +| Feature-wise last-4 gate | 8,192 | per-channel layer choice | +| Low-rank concat, rank 128 | about 1.31M | cross-layer interaction | +| Low-rank concat, rank 256 | about 2.62M | capacity ablation | +| Current concat-MLP | about 16.8M | upper-capacity control | + +Initialize feature-wise/low-rank fusion as a residual around the scalar mean, +so the first optimizer step preserves the strong pretrained readout. Use +weight decay, dropout, and dev early stopping, but do not make generic +regularization the main novelty. + +### P1: transition-aware supervision + +Convert each target vector to the corresponding four-state variable `q_t`, +then derive: + +- `current_t = q_t`; +- `switch_{t,h} = 1[q_{t+h} != q_t]` for short horizons; +- `time_to_change_t` capped at 2 seconds; +- `persistence_t = 1 - switch_{t,1}`. + +Train the existing ten-horizon head jointly with current/switch/time-to-change +heads. Apply larger weight only to true switch steps, plus a consistency loss +when the ground-truth state persists. This should be evaluated first on the +camera-wearer AP at 0.2 seconds, where representation drift and head +overfitting have repeatedly caused the largest damage. + +### P2: expose recent token evidence + +Add a readout query that attends only to payload tokens from `[t-0.2s, t]`, +identified through TMRoPE positions. Compare audio-only recent tokens, AV +recent tokens, and query-plus-token fusion. Keep the backbone frozen. + +This is the most distinctive method contribution, but it should follow P0/P1 +because token masking and timestamp correctness are easier to get wrong than +layer fusion. + +### P2: use existing semantics without test leakage + +Use EasyCom transcription and Target-of-Speech only during training: + +- distill utterance-completion likelihood into the AV state; +- predict group-versus-individual addressee; +- predict a coarse semantic continuation state such as incomplete/complete; +- drop the privileged text branch at inference. + +This makes the claimed "Omni semantic advantage" testable. Merely replacing +wav2vec/ResNet with an Omni hidden state does not demonstrate semantic use. + +### P3: generalization and ablations + +- EasyCom audio-only, video-only, and AV under identical readout capacity; +- Ego4D 346/87 split with the same method and no EasyCom-specific tuning; +- 4s, 8s, and 20s memory with fixed 5Hz output; +- 2 FPS versus 5 FPS video ablation; +- runtime, memory, and causal verification; +- per-phase and transition-conditioned AP in addition to EgoSpeak Table 2. + +## Stop conditions + +Do not spend another full training cycle on: + +- full Thinker LoRA on EasyCom alone; +- Qwen2.5-Omni-7B without a readout change; +- 25--50 epoch high-capacity heads without development early stopping; +- more overlapping-window extraction as the main architecture; +- prompt wording as the primary method. + +Those directions are weakly supported by the current evidence and make the +paper harder to interpret. The clean next experiment is the matched P0 readout +ablation followed by transition-aware supervision. diff --git a/reports/roma_negative_transfer_and_future_floor_projection_2026-08-27.md b/reports/roma_negative_transfer_and_future_floor_projection_2026-08-27.md new file mode 100644 index 0000000..feb7859 --- /dev/null +++ b/reports/roma_negative_transfer_and_future_floor_projection_2026-08-27.md @@ -0,0 +1,280 @@ +# ROMA negative transfer and the next ICASSP direction + +Date: **2026-08-27** + +This memo combines the newly archived frozen-ROMA EasyCom results with recent +work on predictive turn-taking and proactive streaming multimodal agents. The +main conclusion is deliberately narrow: + +> The next useful experiment is not another backbone or a larger readout. It +> is a **low-capacity multi-timescale causal readout plus structured future +> floor projection** at 5 Hz. + +## Evidence from the current project + +| System | 0.2s mAP | Mean mAP | What it establishes | +|---|---:|---:|---| +| Frozen Qwen-3B, final query, 8s | 71.99768 | 58.51937 | strong simple baseline | +| Frozen Qwen-3B, last-4 scalar mixture | 73.45085 | 60.53423 | upper-layer information is useful | +| Frozen Qwen-3B, last-4 concat, 4ep | 73.42020 | 60.87144 | small longer-horizon gain, high overfit risk | +| Frozen Qwen-7B, reported run | 69.80133 | 57.67367 | capacity alone is not the answer | +| Frozen ROMA, last-4 fixed mean, 4ep | 69.21863 | 58.90402 | best ROMA control | +| Frozen ROMA, last-4 concat, 4ep | 68.65486 | 56.52425 | wide fusion overfits strongly | + +With the same last-4 concat design and epoch count, ROMA loses `4.76534` at +0.2 seconds and `4.34719` in mean mAP relative to Qwen-3B. This is a strong +diagnostic but not a parameter-matched attribution test: ROMA's wider hidden +state makes its concat fusion 51.4M parameters instead of about 16.8M. The +fixed mean removes that fusion-capacity confound and the worst overfitting, but +an exact Qwen-3B fixed-mean control is still missing. + +This does **not** prove that ROMA is a worse streaming model. ROMA is adapted +for one-second multimodal units and proactive alert/narration supervision, +whereas this project extracts it with an eight-second Qwen-style replay and +asks for 200 ms future-speaker anticipation. The result instead shows a task +alignment mismatch: stronger generic proactive behavior does not guarantee a +better representation for fine-grained floor transfer. + +## Strongly related work and what transfers + +### EgoSpeak: the exact benchmark contract + +[EgoSpeak (Findings of NAACL 2025)](https://github.com/jun297/EgoSpeak) +defines the exact problem and supplies LSTR and MiniROAD baselines. Its main +strength is the 5 Hz future-speaker evaluation on EasyCom and Ego4D. Its +limitation for the present question is architectural: pretrained ResNet and +wav2vec features are followed by a task-specific temporal model, so it does +not test whether an Omni Transformer already contains the necessary causal +state. + +**Transfer:** preserve its split, ten horizons, target alignment, causality, +and AP evaluator exactly. The paper contribution must lie above this unchanged +contract. + +### MM-VAP and MuVAP: model the future activity trajectory, not ten isolated labels + +[MM-VAP (Findings of ACL 2025)](https://aclanthology.org/2025.findings-acl.12/) +adds facial expression, head pose, and gaze to an audio VAP model and reports +84% versus 79% hold/shift accuracy. Its ablation identifies facial expression +as the most useful visual cue. + +[MuVAP (arXiv 2026)](https://arxiv.org/abs/2606.16731) is even closer to this +project: it is causal, multimodal, multiparty, single-camera, and single-audio +stream. It predicts a near-future joint voice-activity window and introduces +**Role-Relative Projection**, mapping arbitrary speakers to current versus +next floor-holder roles to avoid combinatorial growth. + +The original VAP insight is crucial: near-future speaking states are jointly +structured. A backchannel, overlap, hold, and switch are different trajectories +through the next two seconds, not ten unrelated classification targets. + +**Transfer:** replace the independent `10 x 3` output with a structured future +floor decoder. EasyCom naturally supplies four states at each horizon: +`silence`, `wearer-only`, `others-only`, and `overlap`. A small temporal decoder +can model their dependencies and marginalize back to EgoSpeak's three AP +scores. No new annotation is required. + +### Interspeech 2025 gaze study: global video tokens may hide the useful cue + +[Gaze-Enhanced Multimodal Turn-Taking Prediction in Triadic Conversations +(Interspeech 2025)](https://www.isca-archive.org/interspeech_2025/heo25_interspeech.html) +shows that gaze from one participant already improves prediction, and multiple +participants improve it further when gaze is grounded with speaker location. + +**Transfer:** EasyCom already provides face/head boxes and tracked poses. Use +them to build participant- or role-conditioned visual pooling rather than +expecting a global video query state to preserve small facial and gaze changes. +This is more defensible than increasing the global MLP. + +### ROMA: aligned streaming units and a small explicit decision head + +[ROMA](https://arxiv.org/abs/2601.10323) uses one-second synchronized +audio-video units, chunked TMRoPE, 40 ms audio temporal positions, and a +two-layer speak head fed by a learned mixture of the final four layers. It also +uses a two-stage curriculum: streaming-format alignment followed by proactive +timing training. + +**Transfer:** keep the explicit lightweight decision interface and last-layer +mixture. More importantly, retain the 40 ms audio time IDs when selecting the +recent 200 ms payload. The present negative result says that importing ROMA's +checkpoint alone is insufficient; its useful idea is the interface and +alignment structure. + +### Proact-VL: transition imbalance is not ordinary class imbalance + +[Proact-VL (2026 preprint)](https://arxiv.org/abs/2603.03447) reads a dedicated +`FLAG` token with a small gated MLP. It weights rare speaking/silence state +transitions more heavily, smooths probabilities only during persistent states, +and regularizes the global speaking rate. + +**Transfer:** the project's previous transition auxiliary head is not the same +idea. It asks an extra branch to predict a transition but leaves the primary +future-activity probabilities unconstrained. A stronger experiment should: + +1. upweight the main state loss at true hold/switch boundaries; +2. penalize probability jitter only inside persistence regions; +3. calibrate predicted per-class activity rates on the training split. + +This directly shapes the output trajectory and is more likely to improve the +0.2-second boundary behavior than another auxiliary classifier. + +### VideoLLM-online and Dispider: separate high-rate perception/decision from generation + +[VideoLLM-online (CVPR 2024)](https://showlab.github.io/videollm-online/) +trains temporally aligned streaming dialogue and parallelizes video encoding, +LLM forwarding, and generation; it reports 5--10 FPS on an RTX 3090 and +10--15 FPS on an A100. + +[Dispider (CVPR 2025)](https://github.com/Mark12Ding/Dispider) disentangles +perception, decision, and reaction. A compact module continuously decides +`wait/respond`, while generation runs asynchronously and does not block new +perception. + +**Transfer:** for the benchmark, keep only frozen Omni perception plus the 5 Hz +floor controller. For a later interactive demo, run text/speech generation +asynchronously after a trigger. Do not train the heavy generator merely to +improve a 30-logit anticipation benchmark. + +## Proposed method: Multi-timescale Future Floor Projection + +At every 200 ms prediction time `t`, form two causal summaries: + +1. **semantic context** `q_t`: a fixed or learned low-capacity mixture of the + final four query states over the bounded eight-second stream; +2. **immediate interaction cue** `r_t`: attention pooling over only audio/video + payload tokens whose TMRoPE timestamps fall in `(t-0.2s, t]`. + +Exclude template tokens, padding, and tokens later than `t`. Keep audio and +video pools separate first, then fuse them through a small gate: + +```text +q_t (long semantic state) -----------+ + +--> gated low-rank fusion --> future-floor decoder +a_t (recent 200 ms audio tokens) ----+ +v_t (current/role visual tokens) ----+ +``` + +The future-floor decoder predicts a sequence of ten four-state distributions, +not ten independent three-class logits. A minimal version is: + +```text +fused state -> 10 learned horizon queries -> 2-layer temporal Transformer + -> [10, 4] state logits +``` + +Convert it to the official evaluator scores with: + +```text +p(background) = p(silence) +p(wearer) = p(wearer-only) + p(overlap) +p(others) = p(others-only) + p(overlap) +``` + +This design joins three complementary signals: + +- Omni upper layers provide long semantic context; +- TMRoPE-local token pooling exposes recent prosody and motion; +- VAP-style structured projection models persistence, handover, overlap, and + backchannel trajectories across the two-second future. + +## Highest-value experiment sequence + +### P0: close attribution gaps before adding a method + +1. Run Qwen-3B last-4 **fixed mean**, four epochs, same three seeds and schedule + as the existing readouts. +2. Run original Qwen2.5-Omni-7B versus ROMA with the same last-four fixed-mean + features and head. This isolates ROMA adaptation from model size. +3. Stop ROMA if the matched comparison is not positive. Do not spend more + budget tuning its 51.4M concat head. + +### P1: test the central readout hypothesis + +Use frozen Qwen-3B and the same dense 5 Hz anchors: + +| Variant | Long state | Recent 200ms payload | Future decoder | +|---|---|---|---| +| A | final query | none | current `10 x 3` | +| B | last-4 mixture | none | current `10 x 3` | +| C | last-4 mixture | audio pool | current `10 x 3` | +| D | last-4 mixture | audio + role visual pool | current `10 x 3` | +| E | last-4 mixture | audio + role visual pool | structured `[10,4]` | + +Variant C is the most informative first implementation. If C cannot improve +0.2-second wearer/background AP, the hypothesis that the single query hides +recent prosody is weakened before more complicated visual work. + +### P2: add trajectory-aware loss only after P1 + +For E, compare: + +- ordinary four-state CE; +- boundary-weighted CE; +- boundary-weighted CE plus persistence-only smoothness; +- the above plus class activity-rate calibration. + +Keep loss weights small and predeclare them on a development session. Report +the main activity mAP and separate diagnostics on transition-near versus +persistence anchors. + +### P3: use EasyCom annotations without new manual labels + +Participant head/face boxes and tracked poses can supervise role-conditioned +visual pooling. Speech transcripts and Target-of-Speech labels should be +training-only privileged signals or distillation targets, not test-time input. +This gives a principled semantic/visual benefit while preserving the raw AV +online setting. + +### P4: require Ego4D confirmation + +EasyCom has only nine training conversations. A method selected on EasyCom +must be tested on the much larger Ego4D split. A gain on both datasets supports +a representation/readout claim; a gain only on EasyCom may reflect its fixed +around-table geometry. + +## Evaluation contract for a publishable result + +- Primary: 0.2-second mAP, because this is the current deficit and the claimed + fine-grained contribution. +- Secondary: mean mAP over all ten horizons. +- Diagnostics: per-class AP, transition-near/persistence mAP, overlap-state AP, + calibration, and end-to-end 5 Hz latency. +- Selection: Sessions 4--11 train and Session 12 development, then retrain on + Sessions 4--12 for the fixed budget and evaluate Sessions 1--3 once. +- Statistics: at least three seeds; five if matching EgoSpeak's reporting. +- Causality: no token, frame, transcript, or normalization statistic after + prediction time `t`. + +## Paper angle + +A defensible ICASSP framing is: + +> **Fine-Grained Multimodal Future Floor Projection from Frozen Omni Models** + +Possible contributions: + +1. a strictly causal 5 Hz egocentric audio-video anticipation formulation on + top of a frozen Omni Transformer; +2. a multi-timescale readout that combines upper-layer semantic context with + TMRoPE-local recent payload cues; +3. an overlap-aware structured future-floor decoder with transition/persistence + supervision; +4. controlled evidence that larger backbones, ROMA transfer, and Thinker LoRA + do not solve the fine-grained task by themselves. + +This is stronger than claiming a new head. The scientific claim is that +general streaming Omni states must be **read and supervised at the temporal +structure of conversational floor transfer**. + +## Stop conditions + +- Stop backbone scaling unless the exact same readout/schedule improves both + 0.2-second and mean mAP. +- Stop high-capacity concat fusion; the current data already show severe + session-level overfitting. +- Stop transition auxiliary branches that do not change the primary activity + trajectory. +- Stop tuning on Sessions 1--3; use them once after the configuration is fixed. + +The archived ROMA tables and provenance are in +[`results/easycom_roma_frozen_last4_2026-08-27/`](../results/easycom_roma_frozen_last4_2026-08-27/README.md). diff --git a/results/README.md b/results/README.md index 7a6ff9d..7dbef74 100644 --- a/results/README.md +++ b/results/README.md @@ -6,6 +6,7 @@ readers to the right comparison. | Result record | Question answered | |---|---| +| [easycom_oracle_pooled_wearer_audit_2026-09-10/](easycom_oracle_pooled_wearer_audit_2026-09-10/README.md) | Fixed-four-epoch three-seed oracle test results and the limited wearer gains of historical pooled AV | | [`easycom_open_stream_vs_causal_prefix_2026-08-19/`](easycom_open_stream_vs_causal_prefix_2026-08-19/README.md) | How prepared-media open-stream and causal-prefix features compare | | [`easycom_open_stream_8s_vs_20s_2026-08-23/`](easycom_open_stream_8s_vs_20s_2026-08-23/README.md) | Frozen open-stream context-length comparison | | [`easycom_sparse_supervision_vs_lora_2026-08-24/`](easycom_sparse_supervision_vs_lora_2026-08-24/README.md) | Sparse supervision versus LoRA comparison | diff --git a/results/easycom_dense_last8_qvo_frozen_head_epoch3_2026-08-25/README.md b/results/easycom_dense_last8_qvo_frozen_head_epoch3_2026-08-25/README.md new file mode 100644 index 0000000..d1a7dc0 --- /dev/null +++ b/results/easycom_dense_last8_qvo_frozen_head_epoch3_2026-08-25/README.md @@ -0,0 +1,159 @@ +# EasyCom dense last-8 Thinker LoRA with a frozen head + +Recorded on **2026-08-25** from user-reported remote evaluation output. This is +a single-run diagnostic result, not a multi-seed paper number. + +## Experiment contract + +- Model: Qwen2.5-Omni-3B Thinker. +- Inputs: dense cached AV phases, 2 FPS video, raw-audio tower embeddings, and + an 8-second causal memory budget. +- Prediction/label rate: 5 Hz with ten horizons from 0.2 to 2.0 seconds. +- LoRA scope: `q_proj`, `v_proj`, and `o_proj` in Thinker layers 28--35 only. +- LoRA configuration: rank 8, alpha 16, dropout 0.05, learning rate `5e-6`, + weight decay 0.01. +- Head: the trained frozen 8-second head, loaded from + `checkpoints/easycom_open_stream_2fps_8s_head_seed0.pth` and kept frozen. +- LoRA training: dense Sessions 4--11, global batch 16, five continuous epochs. +- Development selection: Session 12, primary metric 0.2-second full-timeline + mAP and mean mAP as tie-breaker. +- Formal evaluation: epoch 3 was first evaluated on official EasyCom test + Sessions 3, 2, and 1 after development selection. Epoch 1 was subsequently + evaluated as a diagnostic before fixing the one-epoch full-training recipe. +- Selected checkpoint: epoch 3, step 11910. + +The selected remote checkpoint was: + +```text +checkpoints/omni_turn_lora_dense_8s_last8_qvo_frozen_head_epoch5/ + checkpoint-step-11910/ +``` + +The diagnostic epoch-1 checkpoint was: + +```text +checkpoints/omni_turn_lora_dense_8s_last8_qvo_frozen_head_epoch5/ + checkpoint-step-3970/ +``` + +## Development selection on Session 12 + +All values are percentages. + +| Epoch | Step | 0.2s mAP | Mean mAP | +|---:|---:|---:|---:| +| 1 | 3970 | 70.80918 | 63.06487 | +| 2 | 7940 | 70.96876 | **63.09318** | +| 3 | 11910 | **70.99153** | 63.01294 | +| 4 | 15880 | 70.87364 | 62.85465 | +| 5 | 19850 | 70.83634 | 62.81744 | + +Epoch 3 was selected by the preregistered 0.2-second criterion. The five-run +range is only 0.18235 points at 0.2 seconds and 0.27574 mean points, indicating +that the restricted low-learning-rate LoRA is much more stable than the prior +full-Thinker sparse-LoRA runs. Epochs 4--5 nevertheless show mild overfitting. + +## Official Sessions 1--3 results + +Epoch 1 and epoch 3 are close at 0.2 seconds, but epoch 1 preserves the longer +horizons better and therefore has the higher mean mAP. + +| Checkpoint | 0.2s mAP | Mean mAP | 0.2s delta vs frozen | Mean delta vs frozen | +|---|---:|---:|---:|---:| +| Frozen dense control | **71.69695** | **58.16091** | -- | -- | +| LoRA epoch 1 / step 3970 | 71.59585 | 58.01500 | -0.10110 | -0.14591 | +| LoRA epoch 3 / step 11910 | 71.65000 | 57.88833 | -0.04695 | -0.27258 | + +Epoch 1 is only `0.05415` below epoch 3 at 0.2 seconds and is `0.12667` +higher in mean mAP. Relative to epoch 3, it improves every horizon from 0.6 +through 2.0 seconds. This supports fixing one epoch and retraining once on all +official training Sessions 4--12, without development checkpoint selection. + +### Epoch 1 / step 3970 + +| Horizon | LoRA mAP | Frozen dense mAP | Delta | Background AP | Wearer AP | Others AP | +|---:|---:|---:|---:|---:|---:|---:| +| 0.2s | 71.59585 | 71.69695 | -0.10110 | 54.40604 | 76.39165 | 83.98987 | +| 0.4s | 66.37987 | 66.46468 | -0.08481 | 47.56313 | 70.58803 | 80.98845 | +| 0.6s | 61.91962 | 62.03503 | -0.11541 | 42.34205 | 64.79174 | 78.62507 | +| 0.8s | 58.62537 | 58.77409 | -0.14872 | 39.04030 | 59.81340 | 77.02241 | +| 1.0s | 56.30013 | 56.43606 | -0.13593 | 37.19759 | 55.95535 | 75.74744 | +| 1.2s | 54.86526 | 55.00306 | -0.13780 | 36.31219 | 53.41217 | 74.87142 | +| 1.4s | 53.59386 | 53.73114 | -0.13728 | 35.39635 | 51.13887 | 74.24635 | +| 1.6s | 52.86682 | 53.04042 | -0.17360 | 35.19829 | 49.67869 | 73.72348 | +| 1.8s | 52.23019 | 52.42454 | -0.19435 | 34.71486 | 48.59750 | 73.37822 | +| 2.0s | 51.77300 | 52.00315 | -0.23015 | 34.68133 | 47.63528 | 73.00237 | +| **Mean** | **58.01500** | **58.16091** | **-0.14591** | **39.68521** | **57.80027** | **76.55951** | + +### Epoch 3 / step 11910 + +The fair control is the frozen Thinker evaluated from the same dense AV cache +path with the same frozen head. `Delta` is selected LoRA minus that control. + +| Horizon | LoRA mAP | Frozen dense mAP | Delta | Background AP | Wearer AP | Others AP | +|---:|---:|---:|---:|---:|---:|---:| +| 0.2s | 71.65000 | 71.69695 | -0.04695 | 54.59245 | 76.13410 | 84.22345 | +| 0.4s | 66.38669 | 66.46468 | -0.07799 | 47.57127 | 70.36283 | 81.22596 | +| 0.6s | 61.87698 | 62.03503 | -0.15805 | 42.33591 | 64.49864 | 78.79640 | +| 0.8s | 58.51356 | 58.77409 | -0.26053 | 38.97583 | 59.42986 | 77.13498 | +| 1.0s | 56.13447 | 56.43606 | -0.30159 | 37.07515 | 55.55303 | 75.77523 | +| 1.2s | 54.69993 | 55.00306 | -0.30313 | 36.20847 | 53.01781 | 74.87352 | +| 1.4s | 53.41760 | 53.73114 | -0.31354 | 35.30111 | 50.71685 | 74.23484 | +| 1.6s | 52.64243 | 53.04042 | -0.39799 | 35.00690 | 49.21422 | 73.70616 | +| 1.8s | 52.00887 | 52.42454 | -0.41567 | 34.55009 | 48.10873 | 73.36779 | +| 2.0s | 51.55275 | 52.00315 | -0.45040 | 34.55502 | 47.13447 | 72.96877 | +| **Mean** | **57.88833** | **58.16091** | **-0.27258** | **39.61722** | **57.41705** | **76.63071** | + +The class-mean comparison is: + +| Class | LoRA mean AP | Frozen dense mean AP | Delta | +|---|---:|---:|---:| +| Background | 39.61722 | 39.68434 | -0.06712 | +| Camera wearer speaking | 57.41705 | 58.04423 | **-0.62718** | +| Other people speaking | 76.63071 | 76.75417 | -0.12346 | + +At 0.2 seconds specifically, background and other-person AP increase by +`+0.10051` and `+0.09021`, but wearer AP falls by `-0.33157`; the net mAP is +therefore `-0.04695`. The negative gap grows almost monotonically with horizon +and reaches `-0.45040` at 2.0 seconds. + +## Interpretation + +1. **Restricted LoRA nearly eliminates the short-horizon collapse but does not + create positive transfer.** The prior frozen-head initialized sparse LoRA + scored 71.08589 at 0.2 seconds and 56.19608 mean; the new recipe improves + those by `+0.56411` and `+1.69225`, respectively, but remains below the + frozen dense control. +2. **The remaining negative transfer is horizon-dependent.** The 0.2-second + representation is essentially preserved, while longer anticipation is + progressively damaged. This is inconsistent with a simple global + calibration error. +3. **Camera-wearer speech is the main failure mode.** Its average AP decreases + by 0.62718 points, much more than background or other-person speech. +4. **More epochs are not justified.** Epoch 1 is already within 0.05415 points + of epoch 3 at 0.2 seconds and is 0.12667 points better in mean mAP on the + official test sessions. The remaining official-split experiment therefore + fixes one epoch and trains on Sessions 4--12 without checkpoint selection. +5. **This checkpoint is an ablation, not a headline improvement.** It does not + beat frozen Omni, the 58.51937 open-stream frozen-8s result, or EgoSpeak GRU + A+V at 60.6 mean mAP. + +The next defensible post-training diagnostic is adapter scaling/interpolation +on a clean development protocol. It should not be tuned from the already-seen +Sessions 1--3 result. For publication, retrain the frozen head without the +chosen development session; the reused frozen head was originally trained on +the official training split and may already have seen Session 12. + +## Provenance and limitations + +- Source: user-reported remote `eval_omni_turn_lora.py` console tables. +- Raw remote `metrics.json` archived here: no. +- Strict causal-valid table supplied: no. +- Exact remote git revision confirmed: no; the relevant implementation commits + include `7acab44`, `7fdda29`, and `aa4278e`. +- Multi-seed: no. +- Test-use caveat: epoch 1 was inspected after the epoch-3 test result; their + comparison is diagnostic rather than untouched test-set model selection. +- Development caveat: the frozen head may have been trained with Session 12; + this is adequate for diagnosis but is not a clean publication dev split. +- Publication status: negative-transfer ablation only. diff --git a/results/easycom_dense_last8_qvo_frozen_head_epoch3_2026-08-25/summary.json b/results/easycom_dense_last8_qvo_frozen_head_epoch3_2026-08-25/summary.json new file mode 100644 index 0000000..2b0c636 --- /dev/null +++ b/results/easycom_dense_last8_qvo_frozen_head_epoch3_2026-08-25/summary.json @@ -0,0 +1,93 @@ +{ + "schema_version": 1, + "recorded_date": "2026-08-25", + "status": "single_run_negative_transfer_ablation", + "dataset": "EasyCom", + "prediction_step_seconds": 0.2, + "video_fps": 2, + "context_seconds": 8, + "train_sessions": [4, 5, 6, 7, 8, 9, 10, 11], + "development_sessions": [12], + "test_sessions": [3, 2, 1], + "configuration": { + "model": "Qwen2.5-Omni-3B Thinker", + "anchor_policy": "dense_5hz", + "lora_layers": [28, 29, 30, 31, 32, 33, 34, 35], + "lora_target_modules": ["q_proj", "v_proj", "o_proj"], + "lora_rank": 8, + "lora_alpha": 16, + "lora_dropout": 0.05, + "learning_rate": 0.000005, + "weight_decay": 0.01, + "head_frozen": true, + "epochs": 5, + "selected_epoch": 3, + "selected_step": 11910 + }, + "development_metrics_percent": [ + {"epoch": 1, "step": 3970, "horizon_0.2_map": 70.80918, "mean_map": 63.06487}, + {"epoch": 2, "step": 7940, "horizon_0.2_map": 70.96876, "mean_map": 63.09318}, + {"epoch": 3, "step": 11910, "horizon_0.2_map": 70.99153, "mean_map": 63.01294}, + {"epoch": 4, "step": 15880, "horizon_0.2_map": 70.87364, "mean_map": 62.85465}, + {"epoch": 5, "step": 19850, "horizon_0.2_map": 70.83634, "mean_map": 62.81744} + ], + "epoch1_test_metrics_percent": { + "protocol": "egospeak_full_timeline", + "num_timesteps": 23048, + "step": 3970, + "horizon_seconds": [0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0], + "horizon_map": [71.59585, 66.37987, 61.91962, 58.62537, 56.30013, 54.86526, 53.59386, 52.86682, 52.23019, 51.77300], + "mean_map": 58.01500, + "background_ap": [54.40604, 47.56313, 42.34205, 39.04030, 37.19759, 36.31219, 35.39635, 35.19829, 34.71486, 34.68133], + "camera_wearer_speaking_ap": [76.39165, 70.58803, 64.79174, 59.81340, 55.95535, 53.41217, 51.13887, 49.67869, 48.59750, 47.63528], + "other_people_speaking_ap": [83.98987, 80.98845, 78.62507, 77.02241, 75.74744, 74.87142, 74.24635, 73.72348, 73.37822, 73.00237] + }, + "test_metrics_percent": { + "protocol": "egospeak_full_timeline", + "num_timesteps": 23048, + "horizon_seconds": [0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0], + "horizon_map": [71.65000, 66.38669, 61.87698, 58.51356, 56.13447, 54.69993, 53.41760, 52.64243, 52.00887, 51.55275], + "mean_map": 57.88833, + "background_ap": [54.59245, 47.57127, 42.33591, 38.97583, 37.07515, 36.20847, 35.30111, 35.00690, 34.55009, 34.55502], + "camera_wearer_speaking_ap": [76.13410, 70.36283, 64.49864, 59.42986, 55.55303, 53.01781, 50.71685, 49.21422, 48.10873, 47.13447], + "other_people_speaking_ap": [84.22345, 81.22596, 78.79640, 77.13498, 75.77523, 74.87352, 74.23484, 73.70616, 73.36779, 72.96877] + }, + "frozen_dense_control_percent": { + "horizon_map": [71.69695, 66.46468, 62.03503, 58.77409, 56.43606, 55.00306, 53.73114, 53.04042, 52.42454, 52.00315], + "mean_map": 58.16091, + "class_mean_ap": { + "background": 39.68434, + "camera_wearer_speaking": 58.04423, + "other_people_speaking": 76.75417 + } + }, + "selected_lora_minus_frozen_dense_percent_points": { + "horizon_map": [-0.04695, -0.07799, -0.15805, -0.26053, -0.30159, -0.30313, -0.31354, -0.39799, -0.41567, -0.45040], + "mean_map": -0.27258, + "class_mean_ap": { + "background": -0.06712, + "camera_wearer_speaking": -0.62718, + "other_people_speaking": -0.12346 + } + }, + "epoch1_lora_minus_frozen_dense_percent_points": { + "horizon_map": [-0.10110, -0.08481, -0.11541, -0.14872, -0.13593, -0.13780, -0.13728, -0.17360, -0.19435, -0.23015], + "mean_map": -0.14591, + "class_mean_ap": { + "background": 0.00087, + "camera_wearer_speaking": -0.24396, + "other_people_speaking": -0.19466 + } + }, + "epoch1_minus_epoch3_percent_points": { + "horizon_0.2_map": -0.05415, + "mean_map": 0.12667 + }, + "provenance": { + "source": "User-reported remote evaluator console tables", + "raw_metrics_json_archived": false, + "strict_causal_metrics_supplied": false, + "multi_seed": false, + "clean_development_head_split": false + } +} diff --git a/results/easycom_dense_last8_qvo_full_train_epoch1_2026-08-25/README.md b/results/easycom_dense_last8_qvo_full_train_epoch1_2026-08-25/README.md new file mode 100644 index 0000000..ac5bb2a --- /dev/null +++ b/results/easycom_dense_last8_qvo_full_train_epoch1_2026-08-25/README.md @@ -0,0 +1,84 @@ +# EasyCom dense last-8 LoRA, full official train split, one epoch + +Recorded on **2026-08-25** from user-reported remote evaluator output. This is +a single-seed diagnostic experiment. + +## Experiment contract + +- Backbone: Qwen2.5-Omni-3B Thinker. +- Input representation: dense cached AV phases, 2 FPS video, 5 Hz prediction, + and 8 seconds of causal open-stream memory. +- Head: pretrained frozen 8-second head from + `checkpoints/easycom_open_stream_2fps_8s_head_seed0.pth`. +- LoRA: `q_proj`, `v_proj`, and `o_proj` in Thinker layers 28--35, rank 8, + alpha 16, dropout 0.05, learning rate `5e-6`, and weight decay 0.01. +- Training: all official EasyCom train Sessions 4--12, one fixed epoch, 4526 + updates, global batch 16, and no development checkpoint selection. +- Evaluation: official test Sessions 3, 2, and 1 using the EgoSpeak Table 2 + full-timeline protocol. +- Training runtime: 1475.936 seconds on eight GPUs, 3.067 updates/second, with + 8.86 GiB as the maximum per-rank CUDA allocation. +- Cached loss: 1.272857 before training and 1.269106 after training. + +The evaluated checkpoint was: + +```text +checkpoints/ + omni_turn_lora_dense_8s_last8_qvo_frozen_head_train4to12_epoch1_ddp8/ +``` + +## Official Sessions 1--3 result + +| Horizon | LoRA mAP | Frozen dense mAP | Delta | Background AP | Wearer AP | Others AP | +|---:|---:|---:|---:|---:|---:|---:| +| 0.2s | 71.62100 | 71.69695 | -0.07595 | 54.41759 | 76.41473 | 84.03068 | +| 0.4s | 66.38400 | 66.46468 | -0.08068 | 47.56390 | 70.58441 | 81.00369 | +| 0.6s | 61.93324 | 62.03503 | -0.10179 | 42.35133 | 64.80023 | 78.64817 | +| 0.8s | 58.64198 | 58.77409 | -0.13211 | 39.04739 | 59.81664 | 77.06191 | +| 1.0s | 56.30504 | 56.43606 | -0.13102 | 37.17699 | 55.95400 | 75.78412 | +| 1.2s | 54.87690 | 55.00306 | -0.12616 | 36.30712 | 53.40678 | 74.91682 | +| 1.4s | 53.60541 | 53.73114 | -0.12573 | 35.40362 | 51.12561 | 74.28699 | +| 1.6s | 52.89234 | 53.04042 | -0.14808 | 35.22795 | 49.68491 | 73.76416 | +| 1.8s | 52.26201 | 52.42454 | -0.16253 | 34.76888 | 48.59595 | 73.42119 | +| 2.0s | 51.82720 | 52.00315 | -0.17595 | 34.76777 | 47.63587 | 73.07796 | +| **Mean** | **58.03491** | **58.16091** | **-0.12600** | **39.70325** | **57.80191** | **76.59957** | + +## Controlled comparisons + +| Checkpoint | Train sessions | Epoch | 0.2s mAP | Mean mAP | Mean delta vs frozen | +|---|---|---:|---:|---:|---:| +| Frozen dense control | 4--12 head | -- | **71.69695** | **58.16091** | -- | +| Restricted LoRA | 4--12 | 1 | 71.62100 | 58.03491 | -0.12600 | +| Restricted LoRA | 4--11 | 1 | 71.59585 | 58.01500 | -0.14591 | +| Restricted LoRA | 4--11 | 3 | 71.65000 | 57.88833 | -0.27258 | + +Adding Session 12 to the one-epoch LoRA training improves 0.2-second mAP by +`0.02515` and mean mAP by `0.01991` relative to the Sessions 4--11 epoch-1 +checkpoint. All ten horizons improve, but only by `0.00413--0.05420` points. +This is too small to explain or reverse the negative transfer. + +Relative to the frozen dense control, background mean AP is unchanged +(`+0.01891`), while camera-wearer and other-person mean AP remain lower by +`-0.24232` and `-0.15460`. The final mean gap is `-0.12600`, less than half the +epoch-3 gap but still negative. + +## Interpretation + +1. Using the complete official training split helps consistently but only + marginally; withholding Session 12 was not the cause of the regression. +2. One epoch is preferable to three for this restricted adapter because it + preserves longer-horizon representations better. +3. The adapter still does not improve the frozen Omni representation. This + experiment should be reported as a negative-transfer ablation rather than + the main result. +4. The next primary comparison should use a stronger frozen backbone, such as + Qwen2.5-Omni-7B, without LoRA. + +## Provenance and limitations + +- Source: user-reported `eval_omni_turn_lora.py` console output. +- Raw remote `metrics.json` archived here: no. +- Strict causal-valid metrics supplied: no. +- Multi-seed: no. +- Test sessions had already been inspected in earlier LoRA diagnostics, so + this is not an untouched test-set selection result. diff --git a/results/easycom_dense_last8_qvo_full_train_epoch1_2026-08-25/summary.json b/results/easycom_dense_last8_qvo_full_train_epoch1_2026-08-25/summary.json new file mode 100644 index 0000000..bcef5f7 --- /dev/null +++ b/results/easycom_dense_last8_qvo_full_train_epoch1_2026-08-25/summary.json @@ -0,0 +1,76 @@ +{ + "schema_version": 1, + "recorded_date": "2026-08-25", + "status": "single_run_negative_transfer_ablation", + "dataset": "EasyCom", + "model": "Qwen2.5-Omni-3B Thinker", + "prediction_step_seconds": 0.2, + "video_fps": 2, + "context_seconds": 8, + "train_sessions": [7, 11, 10, 6, 9, 4, 8, 5, 12], + "development_sessions": [], + "test_sessions": [3, 2, 1], + "configuration": { + "anchor_policy": "dense_5hz", + "lora_layers": [28, 29, 30, 31, 32, 33, 34, 35], + "lora_target_modules": ["q_proj", "v_proj", "o_proj"], + "lora_rank": 8, + "lora_alpha": 16, + "lora_dropout": 0.05, + "learning_rate": 0.000005, + "weight_decay": 0.01, + "head_frozen": true, + "epochs": 1, + "updates": 4526, + "global_batch_size": 16, + "num_gpus": 8, + "training_seconds": 1475.936, + "updates_per_second": 3.067, + "peak_cuda_allocated_gib_max_across_ranks": 8.86, + "initial_cached_loss": 1.272857, + "final_cached_loss": 1.269106 + }, + "test_metrics_percent": { + "protocol": "egospeak_full_timeline", + "num_timesteps": 23048, + "horizon_seconds": [0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0], + "horizon_map": [71.62100, 66.38400, 61.93324, 58.64198, 56.30504, 54.87690, 53.60541, 52.89234, 52.26201, 51.82720], + "mean_map": 58.03491, + "background_ap": [54.41759, 47.56390, 42.35133, 39.04739, 37.17699, 36.30712, 35.40362, 35.22795, 34.76888, 34.76777], + "camera_wearer_speaking_ap": [76.41473, 70.58441, 64.80023, 59.81664, 55.95400, 53.40678, 51.12561, 49.68491, 48.59595, 47.63587], + "other_people_speaking_ap": [84.03068, 81.00369, 78.64817, 77.06191, 75.78412, 74.91682, 74.28699, 73.76416, 73.42119, 73.07796], + "class_mean_ap": { + "background": 39.70325, + "camera_wearer_speaking": 57.80191, + "other_people_speaking": 76.59957 + } + }, + "frozen_dense_control_percent": { + "horizon_map": [71.69695, 66.46468, 62.03503, 58.77409, 56.43606, 55.00306, 53.73114, 53.04042, 52.42454, 52.00315], + "mean_map": 58.16091, + "class_mean_ap": { + "background": 39.68434, + "camera_wearer_speaking": 58.04423, + "other_people_speaking": 76.75417 + } + }, + "lora_minus_frozen_dense_percent_points": { + "horizon_map": [-0.07595, -0.08068, -0.10179, -0.13211, -0.13102, -0.12616, -0.12573, -0.14808, -0.16253, -0.17595], + "mean_map": -0.12600, + "class_mean_ap": { + "background": 0.01891, + "camera_wearer_speaking": -0.24232, + "other_people_speaking": -0.15460 + } + }, + "full_train_epoch1_minus_train4to11_epoch1_percent_points": { + "horizon_map": [0.02515, 0.00413, 0.01362, 0.01661, 0.00491, 0.01164, 0.01155, 0.02552, 0.03182, 0.05420], + "mean_map": 0.01991 + }, + "provenance": { + "source": "User-reported remote evaluator console table", + "raw_metrics_json_archived": false, + "strict_causal_metrics_supplied": false, + "multi_seed": false + } +} diff --git a/results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/README.md b/results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/README.md new file mode 100644 index 0000000..1dfb79b --- /dev/null +++ b/results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/README.md @@ -0,0 +1,121 @@ +# EasyCom last-4 concat-MLP epoch-budget sweep + +Recorded on **2026-08-26** from three user-reported remote runs. This is an +exploratory single-seed capacity/overfitting diagnostic, not a clean +test-independent checkpoint-selection result. + +## Shared experiment contract + +- Dataset: EasyCom. +- Backbone: frozen Qwen2.5-Omni-3B Thinker. +- Features: final four query states from 8-second open-stream replay, 2 FPS + video, and 5 Hz prediction timestamps. +- Fusion: concatenate `[4, 2048]` to 8192 dimensions, then + `LayerNorm -> Linear(8192, 2048) -> GELU` before the unchanged `10 x 3` + anticipation MLP. +- Training examples: dense 5 Hz anchors from the official training sessions. +- Evaluation: EgoSpeak full-timeline AP/mAP on Sessions 3, 2, and 1. +- The three runs differ in total training budget: 4, 12, or 50 epochs. + +## Main result + +| Total epochs | Final train loss | 0.2s mAP | Mean mAP | Delta mean from 4 epochs | +|---:|---:|---:|---:|---:| +| **4** | 0.806973 | **73.42020** | **60.87144** | -- | +| 12 | 0.661751 | 72.44507 | 57.98809 | -2.88335 | +| 50 | **0.300277** | 66.54645 | 53.03515 | -7.83629 | + +Training loss decreases monotonically while test mAP collapses. This is strong +evidence that the 16.8-million-parameter concat fusion overfits EasyCom's small, +highly correlated session-level training set when trained for conventional +50-epoch head budgets. + +## Per-horizon mAP + +| Horizon | 4 epochs | 12 epochs | 50 epochs | 4ep minus 12ep | 4ep minus 50ep | +|---:|---:|---:|---:|---:|---:| +| 0.2s | 73.42020 | 72.44507 | 66.54645 | +0.97513 | +6.87375 | +| 0.4s | 68.60449 | 66.95546 | 60.89720 | +1.64903 | +7.70729 | +| 0.6s | 64.61782 | 62.31847 | 56.41553 | +2.29935 | +8.20229 | +| 0.8s | 61.53765 | 58.54713 | 53.45517 | +2.99052 | +8.08248 | +| 1.0s | 59.41820 | 56.08389 | 51.56044 | +3.33431 | +7.85776 | +| 1.2s | 58.10069 | 54.58857 | 50.11650 | +3.51212 | +7.98419 | +| 1.4s | 57.24684 | 53.57918 | 48.95956 | +3.66766 | +8.28728 | +| 1.6s | 56.19807 | 52.66949 | 48.10094 | +3.52858 | +8.09713 | +| 1.8s | 55.18182 | 51.73658 | 47.49916 | +3.44524 | +7.68266 | +| 2.0s | 54.38864 | 50.95706 | 46.80053 | +3.43158 | +7.58811 | +| **Mean** | **60.87144** | **57.98809** | **53.03515** | **+2.88335** | **+7.83629** | + +Longer horizons deteriorate earlier and more strongly than 0.2 seconds. The +12-epoch model loses `0.97513` at 0.2 seconds but more than 3.3 points from +1.0 through 2.0 seconds. The model is therefore not merely becoming globally +miscalibrated; it increasingly memorizes short-term training correlations that +do not support future-speaker anticipation. + +## Average class AP over ten horizons + +| Epoch budget | Background | Camera wearer | Other people | +|---:|---:|---:|---:| +| 4 | **42.12966** | **62.08079** | **78.40387** | +| 12 | 40.15120 | 56.70353 | 77.10954 | +| 50 | 34.28629 | 51.12830 | 73.69085 | +| 4ep minus 50ep | +7.84337 | **+10.95249** | +4.71302 | + +The camera-wearer class is most sensitive to overfitting. This matches the +project's earlier LoRA diagnosis: wearer prediction repeatedly suffers first +when representation/head capacity is increased without enough independent +conversation sessions. + +## Comparison with the lower-capacity readouts + +| Readout | Training budget | 0.2s mAP | Mean mAP | +|---|---:|---:|---:| +| Single final-layer query | prior run | 71.99768 | 58.51937 | +| Last-4 learned scalar mixture | 25 epochs | **73.45085** | 60.53423 | +| Last-4 concat-MLP | 4 epochs | 73.42020 | **60.87144** | + +Relative to scalar mixture, the 4-epoch concat-MLP is essentially tied at +0.2 seconds (`-0.03065`) and improves mean mAP by `+0.33721`. Its additional +gain is concentrated around 1.2--1.6 seconds, peaking at `+0.80974` at 1.4 +seconds. The higher-capacity fusion therefore helps longer-term anticipation, +but does not improve the primary 0.2-second score beyond the scalar mixture. + +## Validity caveats + +1. The three commands wrote to the same misleadingly named checkpoint and + result paths containing `25ep`. Later runs therefore overwrite earlier + artifacts unless they were copied elsewhere. The 4-epoch model must be + rerun with a unique path if it was not preserved. +2. Each total-epoch setting creates a different cosine schedule. These are + training-budget comparisons, not checkpoints sampled from one identical + optimization trajectory. +3. The supplied excerpts do not contain complete optimizer/scheduler configs. + Confirm them from the remote checkpoint JSON before a paper table. +4. The 4-epoch budget was identified after evaluating Sessions 1--3. It is a + diagnostic result and cannot be presented as test-independent model + selection. +5. Only full-timeline metrics were supplied; strict causal-valid metrics and + remote `metrics.json` are not archived here. +6. All values are single-seed results. + +## Clean follow-up protocol + +- Use Sessions 4--11 for training and Session 12 for model selection. +- Save every epoch from one fixed 12-epoch schedule and select using a declared + development metric (0.2-second mAP for responsiveness, or mean mAP for Table + 2 overall performance). +- Freeze the selected epoch count before retraining on Sessions 4--12 and + evaluating Sessions 1--3 once. +- Compare single-layer, fixed last-4 mean, scalar mixture, and concat-MLP with + identical seeds and schedules. +- For the concat model, test a smaller bottleneck or low-rank fusion before + adding stronger generic regularization; this preserves the cross-layer idea + while matching EasyCom's effective sample size. + +Machine-readable values are stored in [`summary.json`](summary.json) and +[`metrics.csv`](metrics.csv). A cleaned transcription of the supplied console +tables is in [`console_transcription.txt`](console_transcription.txt). + +The data audit, recent-work comparison, paper framing, and prioritized next +experiments are documented in +[`reports/omni_turn_research_direction_2026-08-26.md`](../../reports/omni_turn_research_direction_2026-08-26.md). diff --git a/results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/console_transcription.txt b/results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/console_transcription.txt new file mode 100644 index 0000000..56c4b4e --- /dev/null +++ b/results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/console_transcription.txt @@ -0,0 +1,41 @@ +4 epochs +final train loss: 0.806973 +0.2s mAP=73.42020 background=56.72497 wearer=77.87909 others=85.65653 +0.4s mAP=68.60449 background=49.88131 wearer=73.26704 others=82.66511 +0.6s mAP=64.61782 background=44.66684 wearer=68.57660 others=80.61002 +0.8s mAP=61.53765 background=41.31402 wearer=64.21804 others=79.08089 +1.0s mAP=59.41820 background=39.34993 wearer=61.35721 others=77.54745 +1.2s mAP=58.10069 background=38.91573 wearer=58.45949 others=76.92686 +1.4s mAP=57.24684 background=38.68165 wearer=57.00022 others=76.05865 +1.6s mAP=56.19807 background=37.71307 wearer=55.21953 others=75.66162 +1.8s mAP=55.18182 background=36.99897 wearer=53.45848 others=75.08801 +2.0s mAP=54.38864 background=37.05013 wearer=51.37220 others=74.74359 +mean mAP=60.87144 + +12 epochs +final train loss: 0.661751 +0.2s mAP=72.44507 background=55.62292 wearer=76.67027 others=85.04201 +0.4s mAP=66.95546 background=48.28782 wearer=70.71200 others=81.86658 +0.6s mAP=62.31847 background=42.64250 wearer=64.97839 others=79.33452 +0.8s mAP=58.54713 background=38.96725 wearer=59.03811 others=77.63603 +1.0s mAP=56.08389 background=37.02472 wearer=55.22792 others=75.99903 +1.2s mAP=54.58857 background=36.78301 wearer=51.71494 others=75.26775 +1.4s mAP=53.57918 background=36.51419 wearer=49.39741 others=74.82594 +1.6s mAP=52.66949 background=35.89852 wearer=48.01650 others=74.09344 +1.8s mAP=51.73658 background=35.09050 wearer=46.39426 others=73.72498 +2.0s mAP=50.95706 background=34.68059 wearer=44.88548 others=73.30511 +mean mAP=57.98809 + +50 epochs +final train loss: 0.300277 +0.2s mAP=66.54645 background=46.77405 wearer=71.62111 others=81.24419 +0.4s mAP=60.89720 background=39.76093 wearer=64.88690 others=78.04378 +0.6s mAP=56.41553 background=35.39797 wearer=58.39212 others=75.45650 +0.8s mAP=53.45517 background=33.13157 wearer=53.17203 others=74.06192 +1.0s mAP=51.56044 background=32.04965 wearer=49.57623 others=73.05543 +1.2s mAP=50.11650 background=31.56647 wearer=46.40684 others=72.37618 +1.4s mAP=48.95956 background=31.27035 wearer=44.07047 others=71.53785 +1.6s mAP=48.10094 background=31.17011 wearer=42.20836 others=70.92435 +1.8s mAP=47.49916 background=31.06982 wearer=41.03746 others=70.39020 +2.0s mAP=46.80053 background=30.67198 wearer=39.91152 others=69.81809 +mean mAP=53.03515 diff --git a/results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/metrics.csv b/results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/metrics.csv new file mode 100644 index 0000000..5694364 --- /dev/null +++ b/results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/metrics.csv @@ -0,0 +1,31 @@ +epochs,horizon_seconds,map,background_ap,camera_wearer_ap,other_people_ap,train_loss +4,0.2,73.42020,56.72497,77.87909,85.65653,0.806973 +4,0.4,68.60449,49.88131,73.26704,82.66511,0.806973 +4,0.6,64.61782,44.66684,68.57660,80.61002,0.806973 +4,0.8,61.53765,41.31402,64.21804,79.08089,0.806973 +4,1.0,59.41820,39.34993,61.35721,77.54745,0.806973 +4,1.2,58.10069,38.91573,58.45949,76.92686,0.806973 +4,1.4,57.24684,38.68165,57.00022,76.05865,0.806973 +4,1.6,56.19807,37.71307,55.21953,75.66162,0.806973 +4,1.8,55.18182,36.99897,53.45848,75.08801,0.806973 +4,2.0,54.38864,37.05013,51.37220,74.74359,0.806973 +12,0.2,72.44507,55.62292,76.67027,85.04201,0.661751 +12,0.4,66.95546,48.28782,70.71200,81.86658,0.661751 +12,0.6,62.31847,42.64250,64.97839,79.33452,0.661751 +12,0.8,58.54713,38.96725,59.03811,77.63603,0.661751 +12,1.0,56.08389,37.02472,55.22792,75.99903,0.661751 +12,1.2,54.58857,36.78301,51.71494,75.26775,0.661751 +12,1.4,53.57918,36.51419,49.39741,74.82594,0.661751 +12,1.6,52.66949,35.89852,48.01650,74.09344,0.661751 +12,1.8,51.73658,35.09050,46.39426,73.72498,0.661751 +12,2.0,50.95706,34.68059,44.88548,73.30511,0.661751 +50,0.2,66.54645,46.77405,71.62111,81.24419,0.300277 +50,0.4,60.89720,39.76093,64.88690,78.04378,0.300277 +50,0.6,56.41553,35.39797,58.39212,75.45650,0.300277 +50,0.8,53.45517,33.13157,53.17203,74.06192,0.300277 +50,1.0,51.56044,32.04965,49.57623,73.05543,0.300277 +50,1.2,50.11650,31.56647,46.40684,72.37618,0.300277 +50,1.4,48.95956,31.27035,44.07047,71.53785,0.300277 +50,1.6,48.10094,31.17011,42.20836,70.92435,0.300277 +50,1.8,47.49916,31.06982,41.03746,70.39020,0.300277 +50,2.0,46.80053,30.67198,39.91152,69.81809,0.300277 diff --git a/results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/summary.json b/results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/summary.json new file mode 100644 index 0000000..c91cc98 --- /dev/null +++ b/results/easycom_frozen_3b_8s_last4_concat_mlp_epoch_budget_2026-08-26/summary.json @@ -0,0 +1,73 @@ +{ + "schema_version": 1, + "recorded_date": "2026-08-26", + "status": "single_seed_test_inspected_diagnostic", + "dataset": "EasyCom", + "model": { + "backbone": "Qwen2.5-Omni-3B", + "thinker_frozen": true, + "memory_seconds": 8, + "streaming_mode": "open-stream-replay", + "video_fps": 2, + "prediction_hz": 5, + "readout": "last_4_query_concat_mlp", + "fusion_parameters_approx": 16795648 + }, + "runs": [ + { + "epochs": 4, + "final_train_loss": 0.806973, + "horizon_0.2_map": 73.42020, + "mean_map": 60.87144, + "mean_class_ap": { + "background": 42.12966, + "camera_wearer": 62.08079, + "other_people": 78.40387 + } + }, + { + "epochs": 12, + "final_train_loss": 0.661751, + "horizon_0.2_map": 72.44507, + "mean_map": 57.98809, + "mean_class_ap": { + "background": 40.15120, + "camera_wearer": 56.70353, + "other_people": 77.10954 + } + }, + { + "epochs": 50, + "final_train_loss": 0.300277, + "horizon_0.2_map": 66.54645, + "mean_map": 53.03515, + "mean_class_ap": { + "background": 34.28629, + "camera_wearer": 51.12830, + "other_people": 73.69085 + } + } + ], + "comparisons_percent": { + "epoch4_minus_epoch12_mean_map": 2.88335, + "epoch4_minus_epoch50_mean_map": 7.83629, + "epoch4_minus_scalar_mixture_mean_map": 0.33721, + "epoch4_minus_scalar_mixture_horizon_0.2_map": -0.03065, + "epoch4_minus_single_final_layer_mean_map": 2.35207, + "epoch4_minus_single_final_layer_horizon_0.2_map": 1.42252 + }, + "validity": { + "test_sessions": [3, 2, 1], + "test_was_used_to_compare_epoch_budgets": true, + "same_output_path_was_reused": true, + "cosine_schedule_changes_with_total_epochs": true, + "strict_causal_valid_reported": false, + "multi_seed": false + }, + "provenance": { + "source": "user-reported remote console tables", + "shared_remote_checkpoint": "checkpoints/easycom_frozen_3b_8s_last4_concat_mlp_25ep.pth", + "shared_remote_result_dir": "results/easycom_frozen_3b_8s_last4_concat_mlp_25ep", + "complete_remote_checkpoint_json_archived": false + } +} diff --git a/results/easycom_oracle_pooled_wearer_audit_2026-09-10/README.md b/results/easycom_oracle_pooled_wearer_audit_2026-09-10/README.md new file mode 100644 index 0000000..b6dd032 --- /dev/null +++ b/results/easycom_oracle_pooled_wearer_audit_2026-09-10/README.md @@ -0,0 +1,71 @@ +# Pooled AV 与 oracle 的 wearer 增益审计 + +日期：2026-09-10。来源：用户粘贴的远程三种子控制台结果，以及已有的 +[pooled/encoder-only 记录](../easycom_encoder_only_1s_4ep_2026-08-28/README.md)。 +本记录没有重新运行训练或读取远程 checkpoint/predictions。 + +## 协议与来源 + +新 oracle：Sessions 4--12 全训练，固定 4 epochs，保存末轮，无 dev checkpoint +selection；Sessions 1--3 测试；seeds 0/1/2；`causal_valid_only`。 +命令入口：`bash run_easycom_oracle_information.sh final 0 1 2`。 +预测根目录：`results/easycom_oracle_information_4ep_train4_12_fixed4_test/`； +checkpoint 按 runner 默认应为 +`checkpoints/easycom_oracle_information_4ep_train4_12_fixed4/_seed.pth`， +但实际环境覆盖值、checkpoint hash 和运行代码 revision 未随日志提供，尚未核验。 +运行契约见 [oracle 手册](../../docs/easycom_oracle_information.md)。 + +`metrics.csv` 保存用户提供的全部 18 行数值，单位均为 AP/mAP 百分点。 +BG/wearer/others AP 三列仅对应 0.2 秒；mean mAP 是三个类别、十个 horizon 的平均， +不能当成十 horizon wearer AP。 + +## 当前观测 + +三种子均值差值（百分点）： + +| 比较 | 0.2s mAP | 0.2s wearer AP | 十 horizon mean mAP | +|---|---:|---:|---:| +| aggregate-visible − audio | +4.82609 | +0.57206 | +1.77462 | +| delayed − audio | +0.08185 | +0.00647 | +0.16443 | +| visibility-only − audio | +0.07598 | −0.06329 | +0.04018 | +| per-person − shuffled-id | +0.34857 | +0.03092 | +0.20013 | +| aggregate-visible − per-person | +0.42975 | +0.22818 | +0.29128 | + +aggregate 的 0.2s 宏平均增益主要来自 BG（+10.45109）和 others（+3.45514）， +wearer 增益为 +0.57206。三种子方向一致，但 seed 方差不能替代跨 session 泛化的 +置信区间，也不能据此声称事件级改善。 + +## 与历史 pooled AV 的关系 + +历史 seed-0 `causal_valid_only`：G+A wearer AP 85.06554，G+A+V 85.08583， +局部视觉的 0.2s wearer 增量仅 +0.02029；mean mAP 增量 +0.29083。 +历史完整曲线采用另一协议 `egospeak_full_timeline`： + +- G+A+V − G+A：十 horizon wearer AP 平均 +0.06078，mean mAP +0.29073。 +- A+V − A：十 horizon wearer AP 平均 −0.99993；0.2s wearer AP −1.40150。 + +因此普通 pooled V 尚无稳定改善 wearer 的证据；encoder-only A+V 还出现 wearer +退步。历史单种子与新 oracle 三种子不是一组完全匹配实验，不能用它们直接计算 +oracle-gap recovery 或判定两个方法的统计显著差异。 + +## 可以和不可以得出的结论 + +1. 已支持：在当前实验设置下，真实的可见 others 活动标注具有互补预测价值； + aggregate 条件优于 visibility-only，收益不能仅由人物可见性控制解释。 +2. 及时性有证据：per-person 优于同结构 delayed 条件；aggregate 与 delayed 同时 + 改变了聚合方式和时间，不能单独作为纯时间干预对照。 +3. 身份连续性：per-person 对 shuffled-id 有小幅同向增益，不能继续称为完全持平或 + 断言 identity 无用；但 wearer 增量仅 +0.03，且 aggregate 更好，不支持其必要性。 +4. 尚未证明：增益来自真实 RGB 可恢复的信息，或 pooling 是失败原因。Oracle 使用 + 说话标注，可能提供当前视觉/音频前端根本无法可靠估计的信号；信息缺失与访问困难 + 目前也不能区分。 +5. 尚未证明：更少抢话、addressee 理解或更好的 entry/yield。短期活动延续也能改善 + BG/others AP，需要直接分析已有 wearer predictions 的事件表现。 +6. Oracle 是特权信息参照，不是数学性能上界；learned latent 可以携带额外线索而超过它。 + +## 对后续研究的含义 + +优先验证从真实因果 AV 估计的局部活动是否对 wearer 提供增量，保留简单 aggregate +对照；在证据出现前，不预设 track、pooling 修复或层间注入必然有效。 +新 oracle 尚缺十 horizon wearer AP 曲线与事件结果，不能仅凭 mean mAP 批准强论文主张。 +Sessions 1--3 已被查看；后续选择不能依赖这次 test 表，使用训练侧验证或预先固定配置。 diff --git a/results/easycom_oracle_pooled_wearer_audit_2026-09-10/metrics.csv b/results/easycom_oracle_pooled_wearer_audit_2026-09-10/metrics.csv new file mode 100644 index 0000000..5478220 --- /dev/null +++ b/results/easycom_oracle_pooled_wearer_audit_2026-09-10/metrics.csv @@ -0,0 +1,19 @@ +condition,seed,map_0p2,bg_ap_0p2,wearer_ap_0p2,others_ap_0p2,mean_map +aggregate-visible,0,86.70353,78.68879,85.97697,95.44481,66.97086 +aggregate-visible,1,86.60725,78.67625,85.68389,95.46161,66.95553 +aggregate-visible,2,86.77584,79.03497,85.84636,95.44621,66.92697 +audio,0,81.82018,68.32374,85.13784,91.99897,65.11495 +audio,1,81.91804,68.45640,85.31319,91.98453,65.23095 +audio,2,81.87011,68.26659,85.34002,92.00373,65.18361 +delayed,0,81.98017,68.44804,85.42083,92.07165,65.36472 +delayed,1,81.85499,68.35216,85.15280,92.06000,65.37526 +delayed,2,82.01874,68.81110,85.23682,92.00832,65.28282 +per-person,0,86.47493,78.23009,85.85070,95.34399,66.79960 +per-person,1,86.14132,77.84909,85.42206,95.15281,66.64966 +per-person,2,86.18113,77.91935,85.54992,95.07412,66.53026 +shuffled-id,0,86.01447,77.05511,85.83326,95.15503,66.54441 +shuffled-id,1,85.85603,77.03031,85.34384,95.19396,66.47007 +shuffled-id,2,85.88116,77.04115,85.55283,95.04951,66.36466 +visibility-only,0,81.99974,68.56855,85.35703,92.07364,65.23904 +visibility-only,1,81.83181,68.36851,85.07484,92.05209,65.24955 +visibility-only,2,82.00473,68.85219,85.16930,91.99268,65.16145 diff --git a/results/easycom_roma_frozen_last4_2026-08-27/README.md b/results/easycom_roma_frozen_last4_2026-08-27/README.md new file mode 100644 index 0000000..65463f8 --- /dev/null +++ b/results/easycom_roma_frozen_last4_2026-08-27/README.md @@ -0,0 +1,123 @@ +# EasyCom frozen-ROMA last-four-layer readout study + +Recorded on **2026-08-27** from three user-reported remote evaluations. These +are single-seed diagnostic results on official EasyCom test Sessions 1--3, not +test-independent paper numbers. + +## Shared experiment contract + +- Dataset: EasyCom; EgoSpeak full-timeline evaluator on Sessions 1--3. +- Backbone checkpoint: ROMA, based on Qwen2.5-Omni-7B. +- Backbone use: frozen representation extraction; no end-to-end ROMA update. +- Streaming path: this project's 8-second Qwen-style open-stream replay, 2 FPS + video, one prediction every 200 ms. +- Saved state: final four query states, shape `[T, 4, 3584]`. +- Prediction target: ten future horizons from 0.2 to 2.0 seconds, three + EgoSpeak AP classes. + +ROMA's released model is trained with one-second aligned streaming units, but +these features were extracted with this project's matched Qwen open-stream +protocol. The result therefore tests the **frozen ROMA representation under +our protocol**; it does not reproduce ROMA's own streaming training recipe. + +## Main result + +| Readout | Epochs | Approx. fusion parameters | 0.2s mAP | Mean mAP | +|---|---:|---:|---:|---:| +| Last-4 concat-MLP | 1 | 51.4M | 68.27145 | 58.89219 | +| Last-4 concat-MLP | 4 | 51.4M | 68.65486 | 56.52425 | +| Last-4 fixed mean | 4 | 0 | **69.21863** | **58.90402** | + +The fixed uniform mean is the strongest ROMA readout. It is effectively tied +with the one-epoch concat model in mean mAP (`+0.01183`) while gaining +`+0.94718` at 0.2 seconds. The 51.4M-parameter fusion is unnecessary here. + +## Per-horizon mAP + +| Horizon | Concat 1ep | Concat 4ep | Fixed mean 4ep | +|---:|---:|---:|---:| +| 0.2s | 68.27145 | 68.65486 | **69.21863** | +| 0.4s | 65.29126 | 64.32941 | **65.67454** | +| 0.6s | 62.15965 | 60.41845 | **62.52327** | +| 0.8s | **59.95476** | 57.35198 | 59.92499 | +| 1.0s | **58.25407** | 55.17851 | 58.14859 | +| 1.2s | **56.83849** | 53.70716 | 56.61984 | +| 1.4s | 55.65764 | 52.59370 | **55.68495** | +| 1.6s | **54.91836** | 51.83772 | 54.52641 | +| 1.8s | **54.20725** | 50.70519 | 53.67463 | +| 2.0s | **53.36895** | 50.46558 | 53.04431 | +| **Mean** | 58.89219 | 56.52425 | **58.90402** | + +## Mean class AP over ten horizons + +| Readout | Background | Camera wearer | Other people | +|---|---:|---:|---:| +| Concat 1ep | 42.17735 | **57.38537** | 77.11384 | +| Concat 4ep | 40.19574 | 52.75190 | 76.62513 | +| Fixed mean 4ep | **42.19246** | 57.26979 | **77.24980** | + +Extending concat training from one to four epochs raises 0.2-second mAP by +only `+0.38341` but reduces mean mAP by `-2.36794`. Camera-wearer mean AP drops +by `-4.63347`, much more than background (`-1.98161`) or other people +(`-0.48871`). This is the same overfitting pattern observed with the 3B +concat-MLP, amplified by ROMA's wider 3584-dimensional state. + +Replacing concat-MLP with a fixed mean at four epochs restores `+4.51789` +camera-wearer AP and `+2.37977` mean mAP. This isolates readout capacity as the +proximate cause of the four-epoch collapse. + +## Comparison with the strongest frozen Qwen2.5-Omni-3B controls + +| Frozen representation/readout | 0.2s mAP | Mean mAP | +|---|---:|---:| +| Qwen-3B final-layer query, 8s | 71.99768 | 58.51937 | +| Qwen-3B last-4 scalar mixture | **73.45085** | 60.53423 | +| Qwen-3B last-4 concat-MLP, 4ep | 73.42020 | **60.87144** | +| ROMA last-4 fixed mean, 4ep | 69.21863 | 58.90402 | +| ROMA last-4 concat-MLP, 4ep | 68.65486 | 56.52425 | + +The same concat design and epoch count gives a strong diagnostic: ROMA is +`-4.76534` at 0.2 seconds and `-4.34719` in mean mAP relative to Qwen-3B. +It is not a parameter-matched attribution test, because the 3584-dimensional +ROMA fusion has 51.4M parameters versus about 16.8M for Qwen-3B. Together with +the fixed-mean result, it shows that ROMA's proactive streaming adaptation is +not automatically useful for 5 Hz egocentric future-speaker anticipation, but +does not yet isolate checkpoint adaptation from width and protocol. + +The fixed-mean ROMA result is not directly architecture-matched to the 3B +scalar-mixture/concat models. An exact Qwen-3B fixed-mean four-epoch control is +still required. A second clean attribution test is ROMA versus its original +Qwen2.5-Omni-7B base using the same last-four stack and fixed-mean head. + +## Interpretation and decision + +1. **Do not continue tuning the ROMA concat head.** Its capacity is much too + high for nine EasyCom training conversations. +2. **Do not conclude that ROMA itself is universally worse.** The experiment + changes checkpoint family and uses a protocol different from ROMA's native + one-second training units. +3. **The practical EasyCom backbone remains frozen Qwen2.5-Omni-3B.** Its + last-four readouts are stronger, cheaper, and already validated throughout + the current pipeline. +4. **The next method should change readout structure and supervision.** The + strongest hypothesis is a low-capacity semantic query state fused with + recent 200 ms payload tokens, followed by a structured future-activity + projection rather than ten independent horizon classifiers. + +## Validity caveats + +- Only user-reported full-timeline metrics were available; strict + causal-valid tables and remote `metrics.json` files were not archived. +- Results are single-seed and Sessions 1--3 were inspected repeatedly. +- Exact optimizer settings and the remote checkpoint filenames for all three + runs were not supplied here. +- The train split is assumed to follow the project's Sessions 4--12 contract; + verify each remote command before using the values in a paper table. +- ROMA's official one-second streaming-unit path was not reproduced. + +Machine-readable values are in [`summary.json`](summary.json), and a formatted +workbook with per-horizon metrics and comparison deltas is in +[`metrics.xlsx`](metrics.xlsx). + +The literature-backed diagnosis and proposed ICASSP research plan are in +[`reports/roma_negative_transfer_and_future_floor_projection_2026-08-27.md`](../../reports/roma_negative_transfer_and_future_floor_projection_2026-08-27.md). diff --git a/results/easycom_roma_frozen_last4_2026-08-27/metrics.xlsx b/results/easycom_roma_frozen_last4_2026-08-27/metrics.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..503c2e81c12d6867dc0ef176cb99fee837df5d8a GIT binary patch literal 9181 zcma)C1yEc|vtHcYCAcPNa1Re0077U`5-a_t3z>t@6eh!EC7H7E!}O|-M~&Z zreLrQo4cKD{dBc*4+l2lrgpVGqw?w`gX#?1Gh=b&?9S?msJTU=B~18i3RM+7&T1E- zhn~teci@%}Yn|&HQVQ+CG$R4I+6Coe4IK2&+FiblSs8AqpP~brf~3S+x>Ouq_C##s0# zj6Yt)>b$DQmbO!Q!JVf+^mRXnQJ%PxsWjMh*P^Yf0=s#Zp70J4-*+j}Xx-!%LFG3i z4K08+6X5^=B{l#6<8LE5yLi}|JO47#p5B^s9tZZ5cIwq-#h#>So%V9|bFSOzl$$L> z)u5MnxD34%>qdTXFz}`2Mc8%UI~ful_8K$R5F{zGw*hkJu`;4vyejATRN}(4&Xnx7 zRi^89l)@sSy|*}TX}>qWi*xa{d{ETN(!UJ$al!UDwz|1{s3v z*X${4NS)Isx`oF}J;=`(#jOQ9wnUuWKuF*|PHXN4l85$Dx!-h4gl$gAk5PMo$HHLY zpe5bF2+!C1#29u!B)YK-b}p`gqTMD?+W_Q;rniPrejH+t`Nq5rPp{tdBc~|QmB(k@ zXkFBVS_#TVot4SE;>om5+J)Cep{p=4`jol56wQ(1;m;_knG-hVsq0`EhzUcNHrbO1 zXAMfzRo}AH41RDnjF`$Mj(>4y+5I8%*%}G&doc>dkmxxozcJPRp#6)iDJjYfjBTN4 zUQ(V=Fn!0wB^UKrB6F^@Y@<`K+t;!kCmQMlPG%p)Wq;P%_xkh(AMR;=xQ7_u^fO8p zb?D5iDH6nUM!thw5u*s0^~ysqIw3!7*{ZLVfw87jO#1FOTX>U{|_(sjP%)nd|;(%pQ z03Z6)ojHLbHDvk;CTyy8fx2n+E6#FVwt7|F12e*Pi;C=uZMI65-O1fG>m701S&zeX$*R=~zPkTlkM|f#;wT%7Q7N6?tz^Q8 zC5mSwN3()B-kV&;>_E2cK-Pj#FPxvD3env%83Qkecfu+>?B! zy1Yt`?)MYxclW*1htY+Dy6-a7ck%vqUl)+MojLnoPtIRX+@`(al)-`jlR)MigU|Wf zJ0R|WJ>|63ry;CCs#j6i6mrp79p_Z`u#r35t|v$erV~rjU#hwn8t*efm7$LMlh~=+ zh?Ce+;lwlC@NeV1zjd9C@nlAz@5$zKyr$Lxs(3#=px5~*Gkxg621?lOH;?C|@5LGs zkV)MN$e|`nc2%M#A8`f=Db8dbB%3sOVsawbt<#GO}M1^ z@{A!hN*Cr#)1pXz8-O%Gp9L8$fLiR;47lQ&H#{`VnBIY2Ov(5ePKVCR*urDpX&t@C z(Gz02z)((y!;}rSe5p<*iMr(cK&1g5?^2i-Mb_?AUYlt*RIf~`2G9e%z+-nXxSAN2 zJ7fk3BS@bpsZ{4CNf1^^=6#HB;pj?8oAdpbzK+)>KLEmpkw3DVH68zGU+H|Ss1{3! zbQy3=x-J=XZrnU=tqJ?wGwSw*oWncd#@+?EOACRZW?03zB&-ob-dKZ|iOzkj>LzqX zQI>-L^~JMSy)m#?FqYAWEwIrll+<>If`|rd=%0B$r+U-jDGnyM(;mbrIJQ#p8=6S$ zO3Bn0E&|cINxLaH%&W-032F}4D|Ja4`w!_)i*pdnr62RW5y?!jd#nvT{>b-M$ln=L z+O5k0{{x6*mu0Uc1lO$wb0T|kY6=6g$aoa%=2U~d@v|y+vk2RO8LLYpXpebo)VgI4 zhmMI2O^fj$>6(Qe(?Z=1=;LbhhKX~+I5NGpKQX@8;<)(&U0SQa{n3P8oP)YxwL}45 zmvRLr_aUv}#gDfAv&Rt@-pB}~!CF*C_WCa`G(oUEKVV5qSJOtO4thkS$r?q^I6JHYh}4#c^i$LvpNHeWyS{UPt|dtgG>@{kvn zU2w`yvO3O-J@mrcFZJ7yX2B81Yv0+<`j02`8NRse4x)&|p9{Xf-t?PQe^DxYTU~sl zvZq8^E02NfW`&R zAQLBZpt_5bmA&O(J0C)qDryfEI$*L?_R(@xv_Ez^TY9+;m4Pq;Bc|gv+0MyXj8w0M z&ixJ$4LpCH^rCBFd;kf$BB;)-?lO>C2AaCQmRdt|)Y<^Y51pO_)fS)cNVFmCNutF$ z_Glx6cl8m-EBt;}3wBaDAQlk-z$OI%i2qjWSGeK~GB)>i;GEk3Qm05|_&{l0-Uu|~Wxbmj*FfM}VGn~pn{qrJ z!Vk>n&!4p;-RLEeu{|#-=;?E1xP?7v4$$0-Y_jJs1$D6~ElNGHZ8CYW(z*R?!tc9N z=z6uTQxSYMIg#0MFwjw%F?n(`Gb&pRlJUa_VUX#i9EDM8tJu07^>>R0c_}#?^7?0{ z6g|HxVbz#g*uSmTDP4=&!$Ug_zb4h;(l=D-D}8F5xZq1CR2JkH=($#*4Doulde7Ok z(e=P3e8t!qgh}B|CXxxj5_11;c2N}cS`0Bp$Xysl^xne1pVPUd$R62OD(`SMm7ez^ zGi#QZe_`#sDj1mH4Kb6WPiK6;5;@%HWtbbaX%w*Z#!q-PH(uTx?9O@;-yZPF)8;~t zy7=fiBYQqKYc=iuTbh?^^ctC|vm^^wFJZ{yH1=DAehVcFHrxU?nZ^v!M5^gn=^*$A z%_=*T26&x(hdY zUcTO!lC~zHtQggE<|hX8tra#4SBwl(;3iHA;AOEzKYpak#EtLfxy054xoz;YnjI zQxB;CecX35_cWHUj3{tI+ElM7T73p6?5kBzD{Ll@NsTG-#9X6Zx`iIVR)2mt#DUjy zRQ29F`Ses4+Hb2@!{I9X?xC}&Bkttv{N&I+?ZHhpy@KQMN1#Ax7C$wAbQEvetE@Qg zWA*#qF4bLx-cw(R{9=a4YN|t1j)%d^?M}3)*;15Qg?P66S=C+RlfHJhwd5_Sw;`0Z zg<_^76hC))Hb5f|HLN{Yi?~?cB#6GYqrQa77xWB=;a5o*S^nSYIGorS94u1{bU&=P zJnU_$)&^YJK!>r80S~#qvtfWn0d(A@%)@aFfagq;>Jz z+RTqcseZGIb;aahS64X;yaBZQ-X2EYo%RejATV{FJ=0D+bG{IsRi~?=sOTn;@)gba zFkgYIKW}g>>ZzVO?eIKE8;yR&5~7>eN{IA5nhjXO)>}4`=Qep$Pa@>;Mjy_O#*5;n z4n=}JTsei4kIKR>5~vHbOab;@=Fy?Hqu2=_e0jDcWS5<@jZ`k<|Lh6Ij;%f7)nN<_ zjG7}1u3bPtj``&^ZAMSV^FTg>cT*VKQDKuCRAH1`-&q1Lc_85ZY0|6D`jKW%3_Wzb z@`C3F>%|V$qv0jqNHsRABD^b`t(sTLpm8af8n!t%jf+nulfLA*!oKwFv0D?g94q%f z*F>h`ZrFz#&ZSH?)4=rk*b^}hrsK9qmm$~0dN6uHEDg4Z&59Rjo)rYvm1^ZnXvGn> zRKC(`DB}%8kPIX29n`(T`M~G&;a~3H_QmGR-kds2o>c_VwUu!gazE4~JMsXMVv z8eaHOEv!Z_F|2vMA&MPG`FOBOJQg_>%-PH>qivvZPNZI5TAbx;)YL>TKm=LYgQ!j$ zWit!%$#9vum_)KgP0cYc&z~rd2t9OcX|PwkvfZK>ZbjlyVl z{Muh+grB&pYMB)EzD=7 z79ACC32O075!Q`m+Q)0x1+6xf>p#Ax#86Ufh|P}0JSb___wBgWu5|ciCi2;N-9K)d zcC;2ZM2O5a&Yd~7d@jj2&0AAn(>FQ+XINJY8n286<8~X`5*-_mY+PI{JXMswf2h^& zd;I8|%5yviJ~pWTkeJ6}dUOtXprv@cJ-YZ55ktwg`Bo8Yse`w#q*-s(ZW(;t)2e5* zQJ0|hHNeZeql^#9SQi|AD(HJuv&cq)B}O@4dSjvX+=!6xc&=pm*);3DewJgPF@E17_#{3^IfmmYkJ>>cUr;z%-7@A zOz^irIk>&~XnlhM%Jy%_x`+rG>yXn-Q=8+AIhKET82MzXd!8I|vzD^aPFT|%9RFc! zVCn2J|D7vdnzCF2S?cBj-rfg#?1CI^;?Zy&RD`EuX1YwO%JSPu2D(-{vnQI8B7#Zu`pf8ru7M3+aOZ}Xj`c7U)Oa_92n#vXK zG~O}(jQvyIF{%v2m3NkRxQJv7#R#G^>asf0*=@aMc(~&UDBWFHqHp$M$!#YVcNH^) z#~C`-7_e%F78!(RQaBcwUUu8q;^C4H*7n+4#5zQmak%q-_~Zo7TVZLtteD}NsdVp} z;To$Hp^TvDfFc=1X9DajWuRM(*H+%~AjBPy$9fJr!2NPju!wxytdMdz9p!wuem0Ha zbeJ<0Hr&U}(LrsW|Aamn&CluExIOHj-V8mg(?hon)fTM#pHwHjCu({UN6HNgjV zEhpfsb(*`Pq9hlTSqe`{baPrwAR3xFXZEPXwsWO*R5452qlrg;`;qBq0D7(+GsRd| zioYPvuP0Q~&)ASj%oz}P>E7J;&I~tPNNCXc9zbI&8+OKfT{w5rA4_hfX`z$UQzq{s zdVv~lmaqN^-cW+mlpKK;ZnborV;em!$6MmLlukl}=#|`<9lA6X1Dc4MlKc_Zqqlt3 ztWpUPil3PrHie3V2_!g^8|B#A9{fhQ&`w+(73i$A6k`zUC>u<5ZRj@bb|@D#nnwM__oRi^W`QMs#3{F>`{muM8 zKX6Jgn&!h|xoNNa%i(aT&+s?{z`B_*2mwvm5DMe{aN(+j8oZ3vHHqE>M>&PfUpl@0 z+P{7J1)sJ*9=UxcT*M|glNEd@dtB0c$=B<(jv?Fh) zC3a9C;W%c4VQ2Af4X8Mlp}`7usGQyLt*%+FS0k@UoLUrci1V)no|@-1D%uA~;OE0f z=*i2|>YOpp?Kn)inW@n!%7`7)b%df)v?8mh7QfF~jE94WpMkrD}zZzuu-M zATCWyZIRq&A)t|oLnGY~%`oY;!u_azpP3~Qh(I#3st#9rG7zmSf)$o&SWa20SzS)p zQa5Ai9Wx_UA(`&RiQ($mJ)6<2z@HAO3->}*kK~u8L5^zVCB6)QcYz3osshJ+SbjE@ zzGCp(jL`3%Vu%GE;sp`V%*S1yI8egU8pd7~la++AD9)3deMb58R}Ty4fFPcpI?^(N z1qJ?P@_l1vP;lrRT~H%-nn|z6*O#hEV|cpw)zew_`wHcsPYaPFr-w=OJW{p#_3*B7 zBYDGJWVu~G5r<9C%V}NYk8*62SI=dUT$k}A(KF%Q9Tl!V{u;1)61`K7a>6XERip0e z%O7+}o<&5swqe12ZFwXznT1q#{`p?ngWGbX6F$n$D^SDl&6@K=dNKtMxHZ|kK+guN zdiy5uY0H=o?8VN@2x!LRtG_p#2*>rtCw1MHmwsM@>Q!^ZRd#2hCDM@2lww?Wv+Lbi zdk7VgjW~mwvn<7SRO}-4{$!aw*bQtuY=Klb{c5I=2MEIGsY5K|QDy4_oyxiQ?|{N_ z2n!PX@s7}Gqve)jk0du)LUfkey~+d|+wcQ8^*(&7v{BkSn!0-jso!=&c|Abbf%^Tj zhFM0DDwnRERj4PFYd7=kDm=T+^c`e%_3Bg?^82JfGSqP_YXG+O8wIAB(uuawwd5Z_ zXGb+bSyJDN7c%^=1&Wq^e(|1DKeo4P*30Klo7Y)+2f!T-$7_)4CnI%(zM)N@QOD4h zJJ-hI@SBMhYc;lGrL~`ypBFXE9aqYZE!G}A+F{2Bq8EL90-;hfd>np{mkrj*a}@>y zZp}uv(QP#mY@gkK{#YApW==moPjC{<@+3R4bO(C=SR!-jJ(@^48|?k|weg9kUD&&S z2FF2fyJd~7qLe*i-?MFnDC4LkD=|=CmwQ0m;d@&-7^}lk*Sm71VO_MXxtOrDB&^a! zSAP!T`)k_Qk!$pi=1`*&1 z@6;g_BS&F?=MX(Eew-~?I)GYTdDdG&=%htYRede(Le5o^N{@`oIgo7f+$9ulS!_T> zY7q(ZX#r@@(Vfn3VicO#NV#ZN!6K*u$1X^G?^VFprx_&@7cgqQ_>Q+&MLRU9^p%km zRvkrI**}+T6DDe{8!?ZfSAn;#S9sn<)%Odm1IS z3C6zuE&%sTY-5btUu>Rf$9LP0jsAAv+>Mw+$cN9ZY zypS+OtlxDBahQ&C*k;ftcgjkz-DbEqRkIAKBzzlM;=zzCIZGKnWTK$!7fK^9U z{#WXg1tgidyIxlj`Uj_{OLVCPTuY(lxa}P(=!mW5W??S|)ubW9%#gKPWA>j6!Cx{( z-l_73B3W*q>9v-167+9@Y9}UQOHO*}VhkCX`B{%Y`IZcC+OY0KI99p&h7{UN)xc^~ zxn1u}Ii6tOHSN$u-c5moL&xFQ>j`xOxGsdA9NYbk_7^C(k`=rrUN(5rbu?QsSc25L z=vb3f`WgaiD(-%aS5K%-#cX#DHFUf>t)!qof~{1oYm5&_cre zYp#MA8Y^?LpTvkvo@R1S6qM{UUvA3b6gz=gyn*X;lI-3IuJ zEUn%ezGi{}0Dj$Q(Cnj;lew)kJKL}4c}s_|7<9>^ay55ywgTIW&~UPG&`_D%n}LDQ ze5nWx#KnS@pGHhnSk2tl#0C0>Gsw!pnF=am?<_(Ca&d7GWM_9agYpt4&TL=@b9?AN z7GNhk6BlUJ$&%f{#LULT(wzMz2L~@Zw8eigXnx0}((rIF|NkI@EiA0e%%#9)5Ib{w zmwy3f|NRXbDh(4SOLG?y8umX}Vj3zrpa_kcIj;Z@ugOa;E?u3<@wUQgc_3rx*0$-y1%l_f7|pwIq1LaJ2s>6HxXSbwT>sY{Sn1_8H22$ zkjUZ05qBy5I79a*?ZcgL{D#p6Z_7~n35f)Gs}1%T%5bM# zCE^0lLE@D`u=5tO|c|a5R z;H1RN5w)+7w>juW7>S-$Rag3Xm2#HPB~UcVK+?JznK&LwnkUkL>%{KO=8c8)N($9O7cZst2&{ocxB~i9#z$|!t8^o9pGK*ZZeW6 za05`8dReEFVgq7U*LTc0>h6i~>v#bCZfTzk^ax6*rOlz1{>6d4l?2V~^w$?vUdLi+yQx9uud{Nz_JNN49$(Iv z-b)!u>g~rAd@XvzQad501-vnmVw+&g`#P2C2TkF?z~aOG_mdM)qyF_P2>4a};URwt z|9PU|Z&3iCAb=h^NKoN_&KLYC`X`(DkLWIRY@woma+`kw{7LQn13(PTN&Ra3Pon2f zgg;rBe-JAFM)+4w=1-JAvta+AK%gg4{?~y2$?W`<6Z;e8&sg$5P@n|Szo7gTS^k6a z?<@O1D6G$ZTj&1~{Pz{?A3)D?;fBNS?I0uyf>a8jY2v8mj06>LaG0?+1 J{M5f{{{zETb{PNw literal 0 HcmV?d00001 diff --git a/results/easycom_roma_frozen_last4_2026-08-27/summary.json b/results/easycom_roma_frozen_last4_2026-08-27/summary.json new file mode 100644 index 0000000..9b633a6 --- /dev/null +++ b/results/easycom_roma_frozen_last4_2026-08-27/summary.json @@ -0,0 +1,121 @@ +{ + "schema_version": 1, + "recorded_date": "2026-08-27", + "status": "single_seed_user_reported_test_inspected_diagnostic", + "dataset": "EasyCom", + "evaluation": { + "protocol": "EgoSpeak full timeline", + "test_sessions": [1, 2, 3], + "prediction_hz": 5, + "future_horizons_seconds": [0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0], + "classes": ["background", "camera_wearer_speaking", "other_people_speaking"], + "n": 23048 + }, + "representation": { + "checkpoint_family": "ROMA based on Qwen2.5-Omni-7B", + "thinker_frozen": true, + "streaming_mode": "open-stream-replay", + "memory_seconds": 8, + "video_fps": 2, + "state": "last_four_query_states", + "state_shape": [4, 3584], + "native_roma_one_second_streaming_recipe_reproduced": false + }, + "runs": [ + { + "id": "roma_last4_concat_mlp_1ep", + "layer_fusion": "concat-mlp", + "epochs": 1, + "fusion_parameters_approx": 51412480, + "reported_mean_map": 58.89219, + "mean_class_ap": { + "background": 42.17735, + "camera_wearer_speaking": 57.38537, + "other_people_speaking": 77.11384 + }, + "per_horizon": [ + {"seconds": 0.2, "background": 52.51021, "camera_wearer_speaking": 70.83688, "other_people_speaking": 81.46725, "map": 68.27145}, + {"seconds": 0.4, "background": 48.71387, "camera_wearer_speaking": 67.22686, "other_people_speaking": 79.93305, "map": 65.29126}, + {"seconds": 0.6, "background": 45.13005, "camera_wearer_speaking": 62.85818, "other_people_speaking": 78.49071, "map": 62.15965}, + {"seconds": 0.8, "background": 42.40835, "camera_wearer_speaking": 59.85171, "other_people_speaking": 77.60421, "map": 59.95476}, + {"seconds": 1.0, "background": 41.04191, "camera_wearer_speaking": 56.82736, "other_people_speaking": 76.89293, "map": 58.25407}, + {"seconds": 1.2, "background": 39.74672, "camera_wearer_speaking": 54.55271, "other_people_speaking": 76.21605, "map": 56.83849}, + {"seconds": 1.4, "background": 38.80110, "camera_wearer_speaking": 52.54133, "other_people_speaking": 75.63049, "map": 55.65764}, + {"seconds": 1.6, "background": 38.10108, "camera_wearer_speaking": 51.26856, "other_people_speaking": 75.38544, "map": 54.91836}, + {"seconds": 1.8, "background": 37.64821, "camera_wearer_speaking": 49.94810, "other_people_speaking": 75.02545, "map": 54.20725}, + {"seconds": 2.0, "background": 37.67202, "camera_wearer_speaking": 47.94197, "other_people_speaking": 74.49285, "map": 53.36895} + ] + }, + { + "id": "roma_last4_concat_mlp_4ep", + "layer_fusion": "concat-mlp", + "epochs": 4, + "fusion_parameters_approx": 51412480, + "reported_mean_map": 56.52425, + "mean_class_ap": { + "background": 40.19574, + "camera_wearer_speaking": 52.75190, + "other_people_speaking": 76.62513 + }, + "per_horizon": [ + {"seconds": 0.2, "background": 52.12125, "camera_wearer_speaking": 71.51813, "other_people_speaking": 82.32518, "map": 68.65486}, + {"seconds": 0.4, "background": 47.23227, "camera_wearer_speaking": 65.34609, "other_people_speaking": 80.40987, "map": 64.32941}, + {"seconds": 0.6, "background": 42.97315, "camera_wearer_speaking": 59.62350, "other_people_speaking": 78.65869, "map": 60.41845}, + {"seconds": 0.8, "background": 39.98155, "camera_wearer_speaking": 54.81910, "other_people_speaking": 77.25528, "map": 57.35198}, + {"seconds": 1.0, "background": 38.30512, "camera_wearer_speaking": 50.99168, "other_people_speaking": 76.23874, "map": 55.17851}, + {"seconds": 1.2, "background": 37.25808, "camera_wearer_speaking": 48.42603, "other_people_speaking": 75.43737, "map": 53.70716}, + {"seconds": 1.4, "background": 36.84681, "camera_wearer_speaking": 46.09807, "other_people_speaking": 74.83621, "map": 52.59370}, + {"seconds": 1.6, "background": 36.30368, "camera_wearer_speaking": 44.70198, "other_people_speaking": 74.50749, "map": 51.83772}, + {"seconds": 1.8, "background": 35.37777, "camera_wearer_speaking": 43.20047, "other_people_speaking": 73.53734, "map": 50.70519}, + {"seconds": 2.0, "background": 35.55772, "camera_wearer_speaking": 42.79393, "other_people_speaking": 73.04510, "map": 50.46558} + ] + }, + { + "id": "roma_last4_fixed_mean_4ep", + "layer_fusion": "fixed-mean", + "epochs": 4, + "fusion_parameters_approx": 0, + "reported_mean_map": 58.90402, + "mean_class_ap": { + "background": 42.19246, + "camera_wearer_speaking": 57.26979, + "other_people_speaking": 77.24980 + }, + "per_horizon": [ + {"seconds": 0.2, "background": 53.55709, "camera_wearer_speaking": 72.09031, "other_people_speaking": 82.00848, "map": 69.21863}, + {"seconds": 0.4, "background": 49.15182, "camera_wearer_speaking": 67.68575, "other_people_speaking": 80.18605, "map": 65.67454}, + {"seconds": 0.6, "background": 45.55752, "camera_wearer_speaking": 63.19862, "other_people_speaking": 78.81368, "map": 62.52327}, + {"seconds": 0.8, "background": 42.39609, "camera_wearer_speaking": 59.81524, "other_people_speaking": 77.56363, "map": 59.92499}, + {"seconds": 1.0, "background": 41.02210, "camera_wearer_speaking": 56.50674, "other_people_speaking": 76.91692, "map": 58.14859}, + {"seconds": 1.2, "background": 39.45131, "camera_wearer_speaking": 53.93851, "other_people_speaking": 76.46970, "map": 56.61984}, + {"seconds": 1.4, "background": 38.78924, "camera_wearer_speaking": 52.30126, "other_people_speaking": 75.96435, "map": 55.68495}, + {"seconds": 1.6, "background": 38.08597, "camera_wearer_speaking": 50.22392, "other_people_speaking": 75.26934, "map": 54.52641}, + {"seconds": 1.8, "background": 37.22956, "camera_wearer_speaking": 49.02115, "other_people_speaking": 74.77319, "map": 53.67463}, + {"seconds": 2.0, "background": 36.68386, "camera_wearer_speaking": 47.91641, "other_people_speaking": 74.53267, "map": 53.04431} + ] + } + ], + "comparisons_percent": { + "concat_4ep_minus_concat_1ep_horizon_0.2_map": 0.38341, + "concat_4ep_minus_concat_1ep_mean_map": -2.36794, + "fixed_mean_4ep_minus_concat_4ep_horizon_0.2_map": 0.56377, + "fixed_mean_4ep_minus_concat_4ep_mean_map": 2.37977, + "fixed_mean_4ep_minus_concat_1ep_horizon_0.2_map": 0.94718, + "fixed_mean_4ep_minus_concat_1ep_mean_map": 0.01183, + "roma_concat_4ep_minus_qwen3b_concat_4ep_horizon_0.2_map": -4.76534, + "roma_concat_4ep_minus_qwen3b_concat_4ep_mean_map": -4.34719 + }, + "validity": { + "single_seed": true, + "test_sessions_repeatedly_inspected": true, + "strict_causal_valid_reported": false, + "remote_metrics_json_archived": false, + "exact_training_commands_archived": false, + "train_sessions_assumed_from_project_protocol": [4, 5, 6, 7, 8, 9, 10, 11, 12] + }, + "provenance": { + "source": "user-reported remote console tables", + "recorded_by": "Codex", + "values_are_percent": true + } +} diff --git a/run_easycom_oracle_information.sh b/run_easycom_oracle_information.sh index 8fae88f..1dc0fc2 100755 --- a/run_easycom_oracle_information.sh +++ b/run_easycom_oracle_information.sh @@ -146,11 +146,11 @@ screen_variants() { --protocol causal_valid_only } -train_all_fixed_and_test() { +configure_final_protocol() { if [[ "${EPOCHS}" != "4" ]]; then - echo "The final protocol requires EPOCHS=4, got ${EPOCHS}" >&2 - exit 2 + echo "Final protocol: overriding inherited EPOCHS=${EPOCHS} with EPOCHS=4" >&2 fi + EPOCHS=4 normalized_train=() for session in "${train_sessions[@]}"; do @@ -166,8 +166,6 @@ train_all_fixed_and_test() { dev_sessions=() CHECKPOINT_ROOT="${FINAL_CHECKPOINT_ROOT:-${CHECKPOINT_ROOT}_train4_12_fixed4}" RESULT_ROOT="${FINAL_RESULT_ROOT:-${RESULT_ROOT}_train4_12_fixed4}" - train_variants - test_selected } test_selected() { @@ -219,8 +217,10 @@ case "${stage}" in screen_variants ;; final) + configure_final_protocol prepare_oracles - train_all_fixed_and_test + train_variants + test_selected ;; test) test_selected diff --git a/test_evaluate_floor_state_changes.py b/test_evaluate_floor_state_changes.py index f8bd747..eb2fa28 100644 --- a/test_evaluate_floor_state_changes.py +++ b/test_evaluate_floor_state_changes.py @@ -1,8 +1,13 @@ from __future__ import annotations +import io +import json +import sys import tempfile import unittest +from contextlib import redirect_stdout from pathlib import Path +from unittest import mock import numpy as np @@ -12,7 +17,9 @@ from evaluate_floor_state_changes import ( evaluate_event_horizon, evaluate_horizon, load_arrays, + main, parse_runs, + print_event_timeline_examples, validate_recalls, ) @@ -22,6 +29,385 @@ def activity(wearer: int, others: int) -> list[int]: class FloorStateMetricTests(unittest.TestCase): + def test_response_window_stops_before_next_other_onset(self) -> None: + target = np.asarray( + [ + activity(0, 1), # session 0: other offset anchor + activity(0, 0), + activity(0, 1), # competing other onset; window ends before here + activity(1, 1), # must not count as response to row-1 offset + activity(0, 1), # session 1: other offset anchor + activity(0, 0), + activity(1, 0), # valid response before any competing onset + activity(1, 1), # later competitor; response has already won + ], + dtype=np.float32, + ) + prediction = np.full( + (len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32 + ) + prediction[1, 1, 0] = 0.1 + prediction[2, 1, 1] = 0.2 + prediction[3, 1, 2] = 0.99 # excluded by competing-risk censoring + prediction[5, 1, 0] = 0.2 + prediction[6, 1, 1] = 0.9 + prediction[7, 1, 2] = 0.9 + valid = np.ones((len(target), NUM_HORIZONS), dtype=bool) + session_ids = np.asarray([0, 0, 0, 0, 1, 1, 1, 1], dtype=np.int32) + + result = evaluate_event_horizon( + target, + prediction, + valid, + session_ids, + 2, + [0.8], + midturn_pause_max_steps=10, + ) + response = result["other_offset_to_wearer_response_event"] + self.assertEqual(response["num_positive"], 1) + self.assertEqual(response["num_negative"], 1) + self.assertEqual(response["num_competing_risk_censored"], 1) + self.assertEqual(response["num_short_resume_hard_negative"], 1) + self.assertAlmostEqual(response["average_precision"], 1.0) + self.assertAlmostEqual(response["min_effective_window_seconds"], 0.2) + self.assertAlmostEqual(response["max_effective_window_seconds"], 0.4) + + def test_yield_window_stops_before_next_other_offset(self) -> None: + target = np.asarray( + [ + activity(1, 0), # session 0: other onset anchor + activity(1, 1), + activity(1, 0), # competing other offset; window ends before here + activity(0, 0), # must not count as yield to row-1 onset + activity(1, 0), # session 1: other onset anchor + activity(1, 1), + activity(0, 1), # valid yield while other remains active + activity(0, 0), # later competitor; yield has already won + ], + dtype=np.float32, + ) + prediction = np.full( + (len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32 + ) + prediction[1, 1, 0] = 0.9 + prediction[2, 1, 1] = 0.2 + prediction[3, 1, 2] = 0.01 # excluded by competing-risk censoring + prediction[5, 1, 0] = 0.8 + prediction[6, 1, 1] = 0.1 + prediction[7, 1, 2] = 0.1 + valid = np.ones((len(target), NUM_HORIZONS), dtype=bool) + session_ids = np.asarray([0, 0, 0, 0, 1, 1, 1, 1], dtype=np.int32) + + result = evaluate_event_horizon( + target, + prediction, + valid, + session_ids, + 2, + [0.8], + midturn_pause_max_steps=10, + ) + yield_metric = result["other_onset_to_wearer_yield_event"] + self.assertEqual(yield_metric["num_positive"], 1) + self.assertEqual(yield_metric["num_negative"], 1) + self.assertEqual(yield_metric["num_competing_risk_censored"], 1) + self.assertAlmostEqual(yield_metric["average_precision"], 1.0) + point = yield_metric["operating_points"][0] + self.assertEqual(point["negative_false_positive_count"], 0) + + def test_observed_response_before_recording_end_is_not_right_censored(self) -> None: + target = np.asarray( + [ + activity(0, 1), + activity(0, 0), + activity(1, 0), # observed response, but nominal 0.6-s end is absent + ], + dtype=np.float32, + ) + prediction = np.full( + (len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32 + ) + valid = np.ones((len(target), NUM_HORIZONS), dtype=bool) + session_ids = np.zeros(len(target), dtype=np.int32) + result = evaluate_event_horizon( + target, + prediction, + valid, + session_ids, + 2, + [0.8], + midturn_pause_max_steps=10, + ) + response = result["other_offset_to_wearer_response_event"] + self.assertEqual(response["num_positive"], 1) + self.assertEqual(response["num_right_censored"], 0) + + def test_unresolved_event_at_recording_end_is_right_censored(self) -> None: + target = np.asarray( + [ + activity(0, 1), + activity(0, 0), + activity(0, 0), + ], + dtype=np.float32, + ) + prediction = np.full( + (len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32 + ) + valid = np.ones((len(target), NUM_HORIZONS), dtype=bool) + session_ids = np.zeros(len(target), dtype=np.int32) + result = evaluate_event_horizon( + target, + prediction, + valid, + session_ids, + 2, + [0.8], + midturn_pause_max_steps=10, + ) + response = result["other_offset_to_wearer_response_event"] + self.assertEqual(response["num_evaluated"], 0) + self.assertEqual(response["num_right_censored"], 1) + + def test_simultaneous_wearer_entry_at_competing_onset_is_excluded(self) -> None: + target = np.asarray( + [ + activity(0, 1), + activity(0, 0), + activity(1, 1), # wearer and others restart simultaneously + activity(1, 1), + ], + dtype=np.float32, + ) + prediction = np.full( + (len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32 + ) + valid = np.ones((len(target), NUM_HORIZONS), dtype=bool) + session_ids = np.zeros(len(target), dtype=np.int32) + result = evaluate_event_horizon( + target, + prediction, + valid, + session_ids, + 2, + [0.8], + midturn_pause_max_steps=10, + ) + response = result["other_offset_to_wearer_response_event"] + self.assertEqual(response["num_positive"], 0) + self.assertEqual(response["num_negative"], 1) + self.assertEqual(response["num_competing_risk_censored"], 1) + self.assertEqual(response["num_short_resume_hard_negative"], 0) + + def test_timeline_marks_competing_boundary_as_excluded(self) -> None: + target = np.asarray( + [ + activity(0, 1), + activity(0, 0), + activity(0, 1), + activity(1, 1), + ], + dtype=np.float32, + ) + prediction = np.full( + (len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32 + ) + valid = np.ones((len(target), NUM_HORIZONS), dtype=bool) + session_ids = np.zeros(len(target), dtype=np.int32) + stream = io.StringIO() + with redirect_stdout(stream): + print_event_timeline_examples( + target, + prediction, + valid, + session_ids, + ["Session_1"], + examples_per_type=1, + horizon_steps=3, + ) + output = stream.getvalue() + self.assertIn("status=competing-censored negative", output) + self.assertIn("C/excluded", output) + + def test_fixed_calibration_threshold_is_not_reselected(self) -> None: + target = np.asarray( + [ + activity(0, 1), + activity(0, 0), + activity(1, 0), + activity(1, 0), + ], + dtype=np.float32, + ) + prediction = np.full( + (len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32 + ) + valid = np.ones((len(target), NUM_HORIZONS), dtype=bool) + session_ids = np.zeros(len(target), dtype=np.int32) + result = evaluate_event_horizon( + target, + prediction, + valid, + session_ids, + 1, + [0.8], + midturn_pause_max_steps=10, + operating_thresholds={ + "response": {0.8: 0.75}, + "yield": {0.8: 0.25}, + }, + threshold_source="calibration_sessions:Session_12", + ) + point = result["other_offset_to_wearer_response_event"][ + "operating_points" + ][0] + self.assertEqual(point["threshold"], 0.75) + self.assertEqual( + point["threshold_source"], "calibration_sessions:Session_12" + ) + self.assertEqual(point["achieved_positive_recall"], 0.0) + + def test_event_score_uses_full_observable_window(self) -> None: + target = np.asarray( + [ + activity(0, 1), + activity(0, 0), + activity(1, 0), # response occurs here + activity(1, 0), + ], + dtype=np.float32, + ) + prediction = np.full( + (len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32 + ) + prediction[1, 1, 0] = 0.1 + prediction[2, 1, 1] = 0.2 + prediction[3, 1, 2] = 0.9 # still part of the by-horizon event score + valid = np.ones((len(target), NUM_HORIZONS), dtype=bool) + session_ids = np.zeros(len(target), dtype=np.int32) + result = evaluate_event_horizon( + target, + prediction, + valid, + session_ids, + 2, + [0.8], + midturn_pause_max_steps=10, + ) + response = result["other_offset_to_wearer_response_event"] + self.assertAlmostEqual(response["max_effective_window_seconds"], 0.6) + self.assertAlmostEqual(response["operating_points"][0]["threshold"], 0.9) + + def test_main_calibrates_on_disjoint_training_session(self) -> None: + target = np.asarray( + [ + activity(0, 1), + activity(0, 0), # response anchor + activity(1, 0), # response + activity(1, 0), + activity(1, 1), # yield anchor + activity(0, 1), # yield + activity(0, 1), + activity(0, 0), # response anchor + activity(0, 0), + activity(0, 1), # competing onset, no response + activity(1, 0), # response anchor and response + activity(1, 0), + activity(1, 1), # yield anchor + activity(1, 1), + activity(1, 0), # competing offset, no yield + ] + + [activity(0, 1)] * 15, + dtype=np.float32, + ) + prediction = np.full( + (len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32 + ) + valid = np.ones((len(target), NUM_HORIZONS), dtype=bool) + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + target_root = root / "target" + run_root = root / "run" + (run_root / "pred").mkdir(parents=True) + (run_root / "valid").mkdir(parents=True) + target_root.mkdir() + for session in ("Session_1", "Session_12"): + np.save(target_root / f"{session}.npy", target) + np.save(run_root / "pred" / f"{session}.npy", prediction) + np.save(run_root / "valid" / f"{session}.npy", valid) + + output = root / "report.json" + argv = [ + "evaluate_floor_state_changes.py", + "--target-root", + str(target_root), + "--run", + f"model={run_root}", + "--sessions", + "Session_1", + "--calibration-sessions", + "Session_12", + "--matched-recalls", + "0.8", + "--output", + str(output), + ] + with mock.patch.object(sys, "argv", argv), redirect_stdout(io.StringIO()): + main() + + payload = json.loads(output.read_text()) + self.assertEqual( + payload["protocol"], "wearer_floor_event_competing_risk_v2" + ) + self.assertEqual(payload["calibration_sessions"], ["Session_12"]) + point = payload["runs"]["model"]["event_horizons"][1][ + "other_offset_to_wearer_response_event" + ]["operating_points"][0] + self.assertEqual( + point["threshold_source"], + "calibration_sessions:Session_12", + ) + framewise_point = payload["runs"]["model"]["horizons"][1][ + "eot_response_vs_other_hold_proxy" + ]["operating_points"][0] + self.assertEqual( + framewise_point["threshold_source"], + "calibration_sessions:Session_12", + ) + self.assertTrue(output.with_name("report_events.csv").is_file()) + + def test_main_refuses_official_test_without_calibration(self) -> None: + argv = [ + "evaluate_floor_state_changes.py", + "--target-root", + "unused", + "--run", + "model=unused", + "--sessions", + "Session_1", + ] + with mock.patch.object(sys, "argv", argv): + with self.assertRaisesRegex(ValueError, "requires --calibration-sessions"): + main() + + def test_main_refuses_test_session_as_official_calibration(self) -> None: + argv = [ + "evaluate_floor_state_changes.py", + "--target-root", + "unused", + "--run", + "model=unused", + "--sessions", + "Session_1", + "--calibration-sessions", + "Session_2", + ] + with mock.patch.object(sys, "argv", argv): + with self.assertRaisesRegex(ValueError, "Sessions 4--12"): + main() + def test_event_level_counts_each_boundary_once(self) -> None: target = np.asarray( [