From 5923b89d607fa3ef0d0763960138800562826968 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Wed, 26 Aug 2026 17:18:50 +0800 Subject: [PATCH] Add transition-aware R1 and R2 objectives --- README.md | 112 +++++++++++ easycom_head.py | 177 +++++++++++++++++- infer_turn_head.py | 146 ++++++++++++++- literature_notes/01_qwen2_5_omni.md | 14 ++ literature_notes/02_egospeak.md | 14 ++ literature_notes/03_lstr.md | 14 ++ literature_notes/04_miniroad.md | 14 ++ literature_notes/05_roma.md | 15 ++ literature_notes/06_duplexpo.md | 15 ++ literature_notes/07_aspirin.md | 14 ++ literature_notes/08_soulx_duplug.md | 14 ++ literature_notes/09_salmonn_omni.md | 14 ++ literature_notes/10_tempo_r0.md | 14 ++ literature_notes/11_videotg_r1.md | 14 ++ literature_notes/12_tar_tvg.md | 14 ++ literature_notes/13_video_r2.md | 14 ++ .../14_prompt_guided_turn_taking.md | 14 ++ literature_notes/15_multilingual_vap.md | 14 ++ .../16_multimodal_turn_taking_backchannel.md | 14 ++ .../17_action_sensitivity_learning.md | 14 ++ .../18_llm_enhanced_dialogue_management.md | 14 ++ ...gning_spoken_dialogue_from_interactions.md | 14 ++ literature_notes/20_spoken_dialogue_rlaif.md | 14 ++ literature_notes/README.md | 52 +++++ test_easycom_head_objectives.py | 81 ++++++++ train_omni_turn_lora.py | 1 + train_turn_head.py | 150 +++++++++++++-- 27 files changed, 979 insertions(+), 22 deletions(-) create mode 100644 literature_notes/01_qwen2_5_omni.md create mode 100644 literature_notes/02_egospeak.md create mode 100644 literature_notes/03_lstr.md create mode 100644 literature_notes/04_miniroad.md create mode 100644 literature_notes/05_roma.md create mode 100644 literature_notes/06_duplexpo.md create mode 100644 literature_notes/07_aspirin.md create mode 100644 literature_notes/08_soulx_duplug.md create mode 100644 literature_notes/09_salmonn_omni.md create mode 100644 literature_notes/10_tempo_r0.md create mode 100644 literature_notes/11_videotg_r1.md create mode 100644 literature_notes/12_tar_tvg.md create mode 100644 literature_notes/13_video_r2.md create mode 100644 literature_notes/14_prompt_guided_turn_taking.md create mode 100644 literature_notes/15_multilingual_vap.md create mode 100644 literature_notes/16_multimodal_turn_taking_backchannel.md create mode 100644 literature_notes/17_action_sensitivity_learning.md create mode 100644 literature_notes/18_llm_enhanced_dialogue_management.md create mode 100644 literature_notes/19_aligning_spoken_dialogue_from_interactions.md create mode 100644 literature_notes/20_spoken_dialogue_rlaif.md create mode 100644 literature_notes/README.md diff --git a/README.md b/README.md index 44822df..ec463b6 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,9 @@ This project tests whether **Qwen2.5-Omni-3B can replace both the handcrafted audio/visual feature extractors and the downstream RNN/Transformer used by EgoSpeak** for egocentric turn-taking anticipation. +The persistent paper index and Chinese summaries are maintained in +[literature_notes](literature_notes/README.md). + The main experiment is deliberately simple: ```text @@ -160,6 +163,115 @@ reducing background AP. The full protocol, per-seed summary, seed-0 horizon curves, and machine-readable data are archived in [`results/easycom_loss_structure_4ep_2026-08-26/`](results/easycom_loss_structure_4ep_2026-08-26/README.md). +#### R1/R2 transition-aware head ablation + +The transition target for current state t and future horizon h is one exactly +when the complete three-label activity row changes between t+h-1 and t+h. +This includes onset, offset, speaker handover, and entering or leaving overlap. +It never uses media or labels after the prediction target t+h. + +Both variants share the activity MLP trunk and add a small binary 10-horizon +transition readout: + +- R1 supervised uses + L = L_activity + 0.2 L_transition, + where L_transition is BCEWithLogits with positive weight 3. +- R2 expected reward uses + L = L_activity + 0.2 L_transition - 0.1 E[R_transition]. + For transition probability p and target z, correctness reward is + z p + (1-z)(1-p). Its expectation is computed exactly, weighted with the + same positive weight 3, and normalized by total weight. There is no sampled + policy gradient, so this is the low-variance control rather than a claim of + interactive reinforcement learning. + +Run the fixed four-epoch, no-development-set comparison with the same schedule +as the loss-structure ablation: + +~~~bash +FEATURE_ROOT=/data1/yfzx/wangx/Datasets/easycom/omni_features/open_stream_2fps_8s_last4_stack +TARGET_ROOT=/data1/yfzx/wangx/Datasets/easycom/easycom_target_frame + +for seed in 0 1 2 +do + CUDA_VISIBLE_DEVICES=0 \ + OMP_NUM_THREADS=4 \ + MKL_NUM_THREADS=4 \ + OPENBLAS_NUM_THREADS=4 \ + python train_turn_head.py \ + --feature-root "$FEATURE_ROOT" \ + --target-root "$TARGET_ROOT" \ + --output "checkpoints/easycom-transition-r1-4ep-seed-$seed.pth" \ + --sessions 4 5 6 7 8 9 10 11 12 \ + --epochs 4 \ + --batch-size 512 \ + --learning-rate 7e-5 \ + --weight-decay 5e-5 \ + --warmup-epochs 1.6 \ + --warmup-factor 0.3 \ + --dropout 0.2 \ + --loss lstr-mce \ + --transition-objective r1-supervised \ + --transition-loss-weight 0.2 \ + --transition-pos-weight 3 \ + --layer-fusion concat-mlp \ + --expected-mode open-stream-replay \ + --seed "$seed" \ + --device cuda:0 + + CUDA_VISIBLE_DEVICES=0 \ + OMP_NUM_THREADS=4 \ + MKL_NUM_THREADS=4 \ + OPENBLAS_NUM_THREADS=4 \ + python train_turn_head.py \ + --feature-root "$FEATURE_ROOT" \ + --target-root "$TARGET_ROOT" \ + --output "checkpoints/easycom-transition-r2-4ep-seed-$seed.pth" \ + --sessions 4 5 6 7 8 9 10 11 12 \ + --epochs 4 \ + --batch-size 512 \ + --learning-rate 7e-5 \ + --weight-decay 5e-5 \ + --warmup-epochs 1.6 \ + --warmup-factor 0.3 \ + --dropout 0.2 \ + --loss lstr-mce \ + --transition-objective r2-expected-reward \ + --transition-loss-weight 0.2 \ + --transition-pos-weight 3 \ + --transition-reward-weight 0.1 \ + --layer-fusion concat-mlp \ + --expected-mode open-stream-replay \ + --seed "$seed" \ + --device cuda:0 +done +~~~ + +Evaluate all six checkpoints. In addition to the unchanged EgoSpeak activity +metrics, inference writes transition_pred arrays and prints transition AP, +precision, recall, and F1 at threshold 0.5: + +~~~bash +for variant in r1 r2 +do + for seed in 0 1 2 + do + CUDA_VISIBLE_DEVICES=0 python infer_turn_head.py \ + --feature-root "$FEATURE_ROOT" \ + --target-root "$TARGET_ROOT" \ + --checkpoint "checkpoints/easycom-transition-$variant-4ep-seed-$seed.pth" \ + --output-dir "results/easycom-transition-$variant-4ep-seed-$seed" \ + --expected-mode open-stream-replay \ + --batch-size 1024 \ + --device cuda:0 + done +done +~~~ + +Compare R1 and R2 against the existing three-seed LSTR-MCE baseline under the +same four-epoch budget. The primary paper outcome remains activity mAP; the +transition metrics diagnose whether an activity improvement is genuinely +associated with better boundary prediction. + ## Current Ego4D context-length results The following single-seed snapshot was recorded on 2026-08-22. Both variants diff --git a/easycom_head.py b/easycom_head.py index ba2b2b9..e735009 100644 --- a/easycom_head.py +++ b/easycom_head.py @@ -46,6 +46,11 @@ LOSS_NAMES = ( "independent-bce", "four-state-ce", ) +TRANSITION_OBJECTIVE_NAMES = ( + "none", + "r1-supervised", + "r2-expected-reward", +) DATASET_NAMES = ("easycom", "ego4d") LAYER_FUSION_MODES = ( "single", @@ -68,6 +73,7 @@ class TurnPredictionHead(nn.Module): num_input_layers: int = 1, learnable_layer_mixture: bool = False, layer_fusion: str | None = None, + transition_auxiliary: bool = False, ) -> None: super().__init__() if num_input_layers <= 0: @@ -101,6 +107,7 @@ class TurnPredictionHead(nn.Module): self.num_input_layers = num_input_layers self.layer_fusion = layer_fusion self.learnable_layer_mixture = layer_fusion == "scalar-mixture" + self.transition_auxiliary = transition_auxiliary if self.learnable_layer_mixture: # Zero initialization exactly reproduces a uniform layer mean at # step zero while allowing training to select useful query layers. @@ -123,8 +130,15 @@ class TurnPredictionHead(nn.Module): nn.Dropout(dropout), nn.Linear(hidden_size // 2, num_horizons * num_classes), ) + self.transition_head: nn.Module | None + if transition_auxiliary: + # Share the activity MLP trunk and add only a small binary readout + # for each future horizon. + self.transition_head = nn.Linear(hidden_size // 2, num_horizons) + else: + self.transition_head = None - def forward(self, states: torch.Tensor) -> torch.Tensor: + def _fuse_layers(self, states: torch.Tensor) -> torch.Tensor: if states.shape[-1] != self.hidden_size: raise ValueError( f"Expected hidden size {self.hidden_size}, got {states.shape[-1]}" @@ -150,11 +164,40 @@ class TurnPredictionHead(nn.Module): assert self.layer_fusion_net is not None states = states.flatten(start_dim=-2) states = self.layer_fusion_net(states) - logits = self.net(states) + return states + + def _shared_features(self, states: torch.Tensor) -> torch.Tensor: + """Return the activity MLP representation shared by both readouts.""" + + states = self._fuse_layers(states) + return self.net[:-1](states) + + def forward(self, states: torch.Tensor) -> torch.Tensor: + features = self._shared_features(states) + logits = self.net[-1](features) return logits.view( - *states.shape[:-1], self.num_horizons, self.num_classes + *features.shape[:-1], self.num_horizons, self.num_classes ) + def forward_with_transition( + self, + states: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Return activity logits and binary transition logits.""" + + if self.transition_head is None: + raise RuntimeError( + "This head was created without transition_auxiliary=True" + ) + features = self._shared_features(states) + activity_logits = self.net[-1](features).view( + *features.shape[:-1], + self.num_horizons, + self.num_classes, + ) + transition_logits = self.transition_head(features) + return activity_logits, transition_logits + def layer_mixture_weights(self) -> torch.Tensor | None: """Return normalized oldest-to-newest layer weights, if enabled.""" @@ -209,6 +252,133 @@ def activity_targets_to_four_states(targets: torch.Tensor) -> torch.LongTensor: return (targets[..., 1] + 2 * targets[..., 2]).long() +def transition_targets_from_activity( + activity_sequence: torch.Tensor, +) -> torch.Tensor: + """Mark every activity-state change between adjacent 5 Hz target rows. + + The input ends in [time, 3] and the output ends in [time - 1]. + A silence/onset, offset, speaker handover, or overlap-state change is one + positive transition. + """ + + activity_sequence = _validated_binary_activity_targets(activity_sequence) + if activity_sequence.ndim < 2 or activity_sequence.shape[-2] < 2: + raise ValueError( + "Activity sequence must contain at least two target rows, got " + f"{tuple(activity_sequence.shape)}" + ) + changed = activity_sequence[..., 1:, :] != activity_sequence[..., :-1, :] + return changed.any(dim=-1).to(activity_sequence.dtype) + + +def weighted_transition_bce_loss( + logits: torch.Tensor, + targets: torch.Tensor, + *, + pos_weight: float = 3.0, +) -> torch.Tensor: + """Binary transition loss with an explicit positive-class weight.""" + + if logits.shape != targets.shape: + raise ValueError( + f"Transition logits/targets mismatch: {tuple(logits.shape)} vs " + f"{tuple(targets.shape)}" + ) + if pos_weight <= 0: + raise ValueError("Transition pos_weight must be positive") + targets = targets.to(device=logits.device, dtype=logits.dtype) + if not torch.isfinite(targets).all() or not torch.all( + (targets == 0) | (targets == 1) + ): + raise ValueError("Transition targets must contain only finite 0/1 values") + return nn.functional.binary_cross_entropy_with_logits( + logits, + targets, + pos_weight=logits.new_tensor(pos_weight), + ) + + +def expected_transition_reward( + logits: torch.Tensor, + targets: torch.Tensor, + *, + pos_weight: float = 3.0, +) -> torch.Tensor: + """Exact expected weighted correctness of a Bernoulli transition policy. + + Correct transition/non-transition actions receive reward one and incorrect + actions zero. The expectation is evaluated analytically rather than by + sampling, and normalized by the sum of example weights to remain in [0, 1]. + """ + + if logits.shape != targets.shape: + raise ValueError( + f"Transition logits/targets mismatch: {tuple(logits.shape)} vs " + f"{tuple(targets.shape)}" + ) + if pos_weight <= 0: + raise ValueError("Transition pos_weight must be positive") + targets = targets.to(device=logits.device, dtype=logits.dtype) + if not torch.isfinite(targets).all() or not torch.all( + (targets == 0) | (targets == 1) + ): + raise ValueError("Transition targets must contain only finite 0/1 values") + probability = logits.sigmoid() + probability_correct = ( + targets * probability + (1.0 - targets) * (1.0 - probability) + ) + weights = 1.0 + (pos_weight - 1.0) * targets + return (weights * probability_correct).sum() / weights.sum() + + +def transition_aware_objective( + activity_loss: torch.Tensor, + transition_logits: torch.Tensor, + transition_targets: torch.Tensor, + *, + objective: str, + transition_weight: float = 0.2, + pos_weight: float = 3.0, + reward_weight: float = 0.1, +) -> dict[str, torch.Tensor]: + """Compose the R1 or R2 transition-aware training objective.""" + + if objective not in TRANSITION_OBJECTIVE_NAMES[1:]: + raise ValueError( + f"Transition objective must be R1 or R2, got {objective!r}" + ) + if transition_weight < 0 or reward_weight < 0: + raise ValueError("Transition and reward weights must be non-negative") + transition_loss = weighted_transition_bce_loss( + transition_logits, + transition_targets, + pos_weight=pos_weight, + ) + expected_reward = expected_transition_reward( + transition_logits, + transition_targets, + pos_weight=pos_weight, + ) + applied_reward = ( + expected_reward + if objective == "r2-expected-reward" + else expected_reward.detach().new_zeros(()) + ) + total = ( + activity_loss + + transition_weight * transition_loss + - reward_weight * applied_reward + ) + return { + "total": total, + "activity": activity_loss, + "transition": transition_loss, + "expected_reward": expected_reward, + "applied_reward": applied_reward, + } + + def output_classes_for_loss(loss_name: str) -> int: """Return the number of head logits required by one training objective.""" @@ -609,6 +779,7 @@ def checkpoint_head_config(payload: dict[str, Any]) -> dict[str, Any]: config.setdefault("dropout", 0.1) config.setdefault("num_input_layers", 1) config.setdefault("learnable_layer_mixture", False) + config.setdefault("transition_auxiliary", False) config.setdefault( "layer_fusion", ( diff --git a/infer_turn_head.py b/infer_turn_head.py index 1af8470..ff61ed6 100644 --- a/infer_turn_head.py +++ b/infer_turn_head.py @@ -28,6 +28,7 @@ from easycom_head import ( normalize_sample_name, output_classes_for_loss, output_space_for_loss, + transition_targets_from_activity, ) @@ -112,8 +113,9 @@ def _predict_states( description: str, show_progress: bool, loss_name: str, -) -> np.ndarray: +) -> tuple[np.ndarray, np.ndarray | None]: chunks: list[torch.Tensor] = [] + transition_chunks: list[torch.Tensor] = [] starts = range(0, len(states), batch_size) progress = tqdm( starts, @@ -128,10 +130,20 @@ def _predict_states( dtype=torch.float32, non_blocking=True, ) + if head.transition_auxiliary: + activity_logits, transition_logits = head.forward_with_transition(batch) + transition_chunks.append(transition_logits.sigmoid().cpu()) + else: + activity_logits = head(batch) chunks.append( - marginal_probabilities_from_logits(head(batch), loss_name).cpu() + marginal_probabilities_from_logits(activity_logits, loss_name).cpu() ) - return torch.cat(chunks, dim=0).numpy() + transition_probabilities = ( + torch.cat(transition_chunks, dim=0).numpy() + if transition_chunks + else None + ) + return torch.cat(chunks, dim=0).numpy(), transition_probabilities def _align_to_egospeak_timeline( @@ -156,6 +168,82 @@ def _align_to_egospeak_timeline( return predictions, valid +def _align_transitions_to_timeline( + probabilities: np.ndarray, + label_indices: torch.LongTensor, + num_target_steps: int, +) -> np.ndarray: + """Align [current_state, future_horizon] transition probabilities.""" + + predictions = np.zeros( + (num_target_steps, NUM_HORIZONS), + dtype=np.float32, + ) + current = label_indices.numpy() + for horizon in range(NUM_HORIZONS): + target_indices = current + horizon + 1 + usable = (target_indices >= 0) & (target_indices < num_target_steps) + predictions[target_indices[usable], horizon] = probabilities[usable, horizon] + return predictions + + +def _evaluate_transitions( + session_predictions: dict[str, np.ndarray], + session_valid: dict[str, np.ndarray], + session_targets: dict[str, np.ndarray], +) -> dict[str, Any]: + """Evaluate auxiliary transition probabilities on causal-valid rows.""" + + from sklearn.metrics import average_precision_score + + horizons = [] + for horizon in range(NUM_HORIZONS): + scores = np.concatenate( + [ + session_predictions[name][session_valid[name][:, horizon], horizon] + for name in session_predictions + ] + ) + targets = np.concatenate( + [ + session_targets[name][session_valid[name][:, horizon]] + for name in session_predictions + ] + ) + predicted = scores >= 0.5 + positive = targets == 1 + true_positive = int(np.logical_and(predicted, positive).sum()) + false_positive = int(np.logical_and(predicted, ~positive).sum()) + false_negative = int(np.logical_and(~predicted, positive).sum()) + precision = true_positive / max(1, true_positive + false_positive) + recall = true_positive / max(1, true_positive + false_negative) + f1 = 2.0 * precision * recall / max(1e-12, precision + recall) + horizons.append( + { + "horizon": horizon + 1, + "seconds": (horizon + 1) * STEP_SECONDS, + "num_steps": int(len(scores)), + "positive_rate": float(positive.mean()), + "average_precision": float( + average_precision_score(targets, scores) + ), + "precision_at_0.5": precision, + "recall_at_0.5": recall, + "f1_at_0.5": f1, + } + ) + return { + "definition": ( + "A positive at target row u means the three-label activity state " + "changed between rows u-1 and u." + ), + "horizons": horizons, + "mean_average_precision": float( + np.mean([item["average_precision"] for item in horizons]) + ), + } + + def _evaluate( session_predictions: dict[str, np.ndarray], session_valid: dict[str, np.ndarray], @@ -278,6 +366,11 @@ def main() -> None: session_predictions: dict[str, np.ndarray] = {} session_valid: dict[str, np.ndarray] = {} session_targets: dict[str, np.ndarray] = {} + session_transition_predictions: dict[str, np.ndarray] = {} + session_transition_targets: dict[str, np.ndarray] = {} + transition_prediction_dir = args.output_dir / "transition_pred" + if head.transition_auxiliary: + transition_prediction_dir.mkdir(parents=True, exist_ok=True) print(f"device: {device}") print( @@ -302,7 +395,7 @@ def main() -> None: f"{name} has {data.num_input_layers} feature layers, checkpoint " f"expects {head_config['num_input_layers']}" ) - probabilities = _predict_states( + probabilities, transition_probabilities = _predict_states( head, data.states, device, @@ -322,12 +415,37 @@ def main() -> None: session_targets[name] = target_array np.save(prediction_dir / f"{name}.npy", predictions) np.save(valid_dir / f"{name}.npy", valid) + if transition_probabilities is not None: + transition_predictions = _align_transitions_to_timeline( + transition_probabilities, + data.label_indices, + len(data.targets), + ) + transition_targets = np.zeros(len(data.targets), dtype=np.float32) + transition_targets[1:] = ( + transition_targets_from_activity(data.targets).numpy() + ) + session_transition_predictions[name] = transition_predictions + session_transition_targets[name] = transition_targets + np.save( + transition_prediction_dir / f"{name}.npy", + transition_predictions, + ) print( f"{name}: saved pred {predictions.shape}, " f"causal-valid counts={valid.sum(axis=0).tolist()}" ) metrics = _evaluate(session_predictions, session_valid, session_targets) + transition_metrics = ( + _evaluate_transitions( + session_transition_predictions, + session_valid, + session_transition_targets, + ) + if session_transition_predictions + else None + ) metrics.update( { "sessions": sessions, @@ -346,6 +464,12 @@ def main() -> None: "checkpoint_train_sessions": checkpoint_payload.get("train_sessions"), "checkpoint_dev_sessions": checkpoint_payload.get("dev_sessions"), "layer_fusion": head.layer_fusion, + "transition_auxiliary": head.transition_auxiliary, + "transition_objective": checkpoint_payload.get( + "training_config", + {}, + ).get("transition_objective", "none"), + "transition_metrics": transition_metrics, "layer_mixture_weights_oldest_to_newest": ( head.layer_mixture_weights().detach().float().cpu().tolist() if head.layer_mixture_weights() is not None @@ -356,6 +480,20 @@ def main() -> None: metrics_path = args.output_dir / "metrics.json" metrics_path.write_text(json.dumps(metrics, indent=2) + "\n") _print_metrics(metrics) + if transition_metrics is not None: + print("\ntransition_auxiliary:") + for result in transition_metrics["horizons"]: + print( + f"{result['seconds']:.1f}s: " + f"AP={result['average_precision'] * 100:.5f}, " + f"P@0.5={result['precision_at_0.5'] * 100:.5f}, " + f"R@0.5={result['recall_at_0.5'] * 100:.5f}, " + f"F1@0.5={result['f1_at_0.5'] * 100:.5f}" + ) + print( + "10-horizon transition mean AP: " + f"{transition_metrics['mean_average_precision'] * 100:.5f}" + ) if metrics["layer_mixture_weights_oldest_to_newest"] is not None: print( "\nLayer mixture weights (oldest->newest): " diff --git a/literature_notes/01_qwen2_5_omni.md b/literature_notes/01_qwen2_5_omni.md new file mode 100644 index 0000000..de67bca --- /dev/null +++ b/literature_notes/01_qwen2_5_omni.md @@ -0,0 +1,14 @@ +# Qwen2.5-Omni Technical Report + +- 年份：2025 +- 阅读深度：深入阅读 +- 官方链接：https://arxiv.org/abs/2503.20215 + +## 摘要（中文转述） + +Qwen2.5-Omni 是端到端全模态模型，统一处理文本、图像、音频和视频输入，并生成文本与语音。其架构采用 Thinker–Talker 分工：Thinker 负责多模态理解和文本语义建模，Talker 负责流式语音生成。模型用分块式音频、视频编码和时间对齐机制组织连续媒体，借助 TMRoPE 表达跨模态时间关系；语音侧还采用滑窗式扩散生成以降低延迟。 + +## 与当前项目的关系 + +当前特征提取应优先保持 Thinker 的原生流式输入语义：按固定 AV 块提交、保持同一开放媒体 turn、让已提交块进入持久 KV。因果前缀反复重算可作为严格离线参照，但不是原作的在线执行方式。预先重编码成 2 FPS 便于构造严格网格，却可能改变原视频末尾不足完整块时的抽帧语义，因此直接解码方案必须显式验证时间戳、最终帧和目标长度。 + diff --git a/literature_notes/02_egospeak.md b/literature_notes/02_egospeak.md new file mode 100644 index 0000000..a011003 --- /dev/null +++ b/literature_notes/02_egospeak.md @@ -0,0 +1,14 @@ +# EgoSpeak: Learning When to Speak for Egocentric Conversational Agents in the Wild + +- 会议：Findings of NAACL 2025 +- 阅读深度：深入阅读 +- 官方链接：https://aclanthology.org/2025.findings-naacl.163/ + +## 摘要（中文转述） + +EgoSpeak 研究第一人称智能体在自然多人对话中“何时开口”。任务要求模型只利用当前及过去的第一人称视觉和音频，在连续、未裁剪场景中预测近期说话状态。工作引入大规模网络对话预训练，并在 EasyCom 与 Ego4D 上评估，强调多模态输入、较长上下文和在线因果约束对开口预测的重要性。 + +## 与当前项目的关系 + +它定义了本项目复现的核心协议：5 Hz 目标、多个未来预测时距、EasyCom/Ego4D 划分及 AP/mAP 评测。我们发现 EasyCom 中 wearer 与 others 可以同时为真，因此三路 softmax 的互斥输出与真实标签结构不完全一致；四状态 softmax 和独立 BCE 的消融虽更匹配标签结构，但当前四个 epoch 的收益很小，说明主要瓶颈可能不是输出空间。 + diff --git a/literature_notes/03_lstr.md b/literature_notes/03_lstr.md new file mode 100644 index 0000000..b6224f3 --- /dev/null +++ b/literature_notes/03_lstr.md @@ -0,0 +1,14 @@ +# Long Short-Term Transformer for Online Action Detection + +- 会议：NeurIPS 2021 +- 阅读深度：重点章节阅读 +- 官方链接：https://papers.nips.cc/paper/2021/hash/08b255a5d42b89b0585260b6f2360bdd-Abstract.html + +## 摘要（中文转述） + +LSTR 面向在线动作检测，把长历史与最近短期片段分开建模。长时编码器压缩较远上下文，短时解码器聚焦当前附近的细粒度变化，在不能访问未来的条件下兼顾长期依赖和实时决策。该设计成为后续在线检测和时序预测常用的建模参照。 + +## 与当前项目的关系 + +EgoSpeak-compatible 训练中的 LSTR-MCE 损失与评测习惯来自这一技术脉络，但当前项目并没有完整复现 LSTR 架构。实验中应把“采用 LSTR-MCE 损失”和“采用 LSTR 模型”明确区分。现有三种标签损失在四个 epoch 上差异有限，也提示下一步更值得比较长短期时序头、边界特征和难例采样。 + diff --git a/literature_notes/04_miniroad.md b/literature_notes/04_miniroad.md new file mode 100644 index 0000000..d7864c1 --- /dev/null +++ b/literature_notes/04_miniroad.md @@ -0,0 +1,14 @@ +# MiniROAD: Minimal RNN Framework for Online Action Detection + +- 会议：ICCV 2023 +- 阅读深度：重点章节阅读 +- 官方链接：https://openaccess.thecvf.com/content/ICCV2023/html/An_MiniROAD_Minimal_RNN_Framework_for_Online_Action_Detection_ICCV_2023_paper.html + +## 摘要（中文转述） + +MiniROAD 重新审视在线动作检测，指出递归模型表现不佳的重要原因是训练与推理过程不一致，而不一定是 RNN 本身能力不足。论文通过简单的递归结构和时间步损失设计获得较好的准确率与效率平衡，并把方法扩展到动作预判，展示轻量在线模型的竞争力。 + +## 与当前项目的关系 + +本项目已经把归一化 MiniROAD 风格损失作为消融候选。它给出的关键启示是：训练时必须模拟真正的流式状态演进；若训练头看到完整序列、推理却逐步更新，损失微调很难弥补语义错位。它也是 transition-focused 监督训练比直接采用强化学习更稳妥的依据之一。 + diff --git a/literature_notes/05_roma.md b/literature_notes/05_roma.md new file mode 100644 index 0000000..c1df4dd --- /dev/null +++ b/literature_notes/05_roma.md @@ -0,0 +1,15 @@ +# ROMA: A Real-Time Omni-Multimodal Assistant + +- 年份：2026 +- 阅读深度：深入阅读 +- 官方链接：https://arxiv.org/abs/2601.10323 +- 本地参考实现：/Users/wx44wx/Code/ROMA + +## 摘要（中文转述） + +ROMA 面向实时全模态助手，把连续音视频整理成同步的流式单元，通过分块时间位置编码和持久 KV 维持在线上下文。模型把内容理解与“是否说话”的时机判断分离，设置独立 Speak Head，并对最后若干层隐藏状态做可学习聚合。训练包含流式模板对齐和时间感知决策阶段，目标是同时支持主动响应、被动倾听和打断处理。 + +## 与当前项目的关系 + +最值得复用的不是照搬其 1 秒块，而是三个思想：最后四层可学习融合、表示头与时机头解耦、对临界转换区域增加决策监督。ROMA 的时间分辨率和任务输出与当前 5 Hz EgoSpeak 预测不同，不能直接把 1 秒决策粒度当作替代方案。本地实现还应结合论文逐项核查，不能默认所有描述组件都已完整落地。 + diff --git a/literature_notes/06_duplexpo.md b/literature_notes/06_duplexpo.md new file mode 100644 index 0000000..0dd293e --- /dev/null +++ b/literature_notes/06_duplexpo.md @@ -0,0 +1,15 @@ +# Decoupling Conversational Dynamics in Full-Duplex Spoken Models through Reinforcement Learning + +- 简称：DuplexPO +- 年份：2026 +- 阅读深度：深入阅读 +- 官方链接：https://arxiv.org/html/2607.07148 + +## 摘要（中文转述） + +DuplexPO 把全双工语音模型的内容能力与对话动态策略分开优化。方法围绕轮次开始、backchannel、打断和让出话轮等关键窗口构造训练样本，把连续决策表示成帧级策略，并设计开始时机、反馈、让步和正则等因子化奖励。训练采用带组内归一化和 KL 约束的策略优化，以改善互动时机而尽量保留已有语言能力。 + +## 与当前项目的关系 + +它是“针对 transition 做 RL”最接近的参考，但其奖励依赖智能体动作改变后续交互，而 EasyCom 冻结数据中的未来是外生的。直接在离线固定轨迹上做 RL，容易退化为昂贵的加权分类。若未来建立可交互模拟环境，可借鉴其动态关键窗口；当前更适合先把这些窗口用于监督采样、边界加权和专门评测。 + diff --git a/literature_notes/07_aspirin.md b/literature_notes/07_aspirin.md new file mode 100644 index 0000000..57eabaa --- /dev/null +++ b/literature_notes/07_aspirin.md @@ -0,0 +1,14 @@ +# ASPIRin: Action Space Projection for Interactivity-Optimized Reinforcement Learning in Full-Duplex Speech Language Models + +- 年份：2026 +- 阅读深度：深入阅读 +- 官方链接：https://arxiv.org/html/2604.10065 + +## 摘要（中文转述） + +ASPIRin 观察到，直接在完整语音 token 空间中用 GRPO 优化响应速度，会让模型利用奖励漏洞，出现重复或语义崩坏。它把庞大的生成动作空间投影为 active/inactive 两类互动状态，仅在这一低维空间优化打断和延迟等规则奖励，从而把“何时响应”与“说什么”部分解耦。 + +## 与当前项目的关系 + +如果本项目以后采用 RL，最安全的方案不是更新 Qwen Thinker 表示或完整内容头，而是冻结编码器和内容预测，只优化 transition/no-transition 或 speak/listen 的小策略头。当前三标签任务还应保留监督主损失，并用 KL 或参数冻结防止为了边界奖励牺牲整体 AP。 + diff --git a/literature_notes/08_soulx_duplug.md b/literature_notes/08_soulx_duplug.md new file mode 100644 index 0000000..be28ef7 --- /dev/null +++ b/literature_notes/08_soulx_duplug.md @@ -0,0 +1,14 @@ +# SoulX-Duplug: Plug-and-Play Streaming State Prediction Module for Realtime Full-Duplex Speech Conversation + +- 年份：2026 +- 阅读深度：深入阅读 +- 官方链接：https://arxiv.org/html/2603.14877 + +## 摘要（中文转述） + +SoulX-Duplug 提出可插拔的流式状态预测模块，以短音频块持续输入，把语音识别信息与对话状态 token 交错建模。训练联合优化 ASR 和状态预测，并采用分阶段训练减少任务冲突。系统强调低延迟状态决策，可在不重建整个对话模型的情况下提供倾听、回应或打断控制信号。 + +## 与当前项目的关系 + +这篇工作支持先做显式状态和转换监督，而不是立刻使用 RL。可在现有 5 Hz 头上增加辅助任务：当前状态、是否发生转换、距离下一次转换的时间，或者 wearer/others 的 onset 与 offset。若辅助监督能稳定改善临界窗口，再考虑把同一小头转成策略优化。 + diff --git a/literature_notes/09_salmonn_omni.md b/literature_notes/09_salmonn_omni.md new file mode 100644 index 0000000..26c9bde --- /dev/null +++ b/literature_notes/09_salmonn_omni.md @@ -0,0 +1,14 @@ +# SALMONN-omni: A Standalone Speech LLM without Codec Injection for Full-duplex Conversation + +- 年份：2025 +- 阅读深度：重点章节阅读 +- 官方链接：https://arxiv.org/html/2505.17060 + +## 摘要（中文转述） + +SALMONN-omni 构建不依赖外部 codec 注入的全双工语音语言模型，包含流式语音编码、动态思考和流式合成组件。论文针对打断和 backchannel 使用偏好优化来改善交互行为，同时保留语音理解和生成能力。 + +## 与当前项目的关系 + +其 DPO 实验说明对话转换策略可以被偏好学习调整，但训练过程可能明显不稳定：中间阶段的综合指标可能先大幅下降再恢复。因此，如果本项目做 transition 偏好优化，必须同时记录全时距 mAP、三类 AP、转换窗口指标和非转换区误报，不能只报告 onset 延迟或单一奖励。 + diff --git a/literature_notes/10_tempo_r0.md b/literature_notes/10_tempo_r0.md new file mode 100644 index 0000000..54c4406 --- /dev/null +++ b/literature_notes/10_tempo_r0.md @@ -0,0 +1,14 @@ +# Tempo-R0: A Video-MLLM for Temporal Video Grounding through Efficient Temporal Sensing Reinforcement Learning + +- 年份：2025 +- 阅读深度：重点章节阅读 +- 官方链接：https://arxiv.org/html/2507.04702 + +## 摘要（中文转述） + +Tempo-R0 面向视频时间定位，让模型自适应选择观察帧并输出带时间戳的答案。训练先进行监督微调，再使用包含格式正确性、时间区间重叠和拒答行为的可验证奖励进行策略优化。其核心价值是把时间定位转化为可以自动评分的推理行为。 + +## 与当前项目的关系 + +论文说明明确、可验证的时间奖励能够改善生成式边界定位，但它处理的是离线时间区间生成，而不是 5 Hz 在线多标签预测。可借鉴的部分是基于 onset/offset 容差、边界 IoU 和提前量的奖励设计；不可直接据此断言 GRPO 会优于当前交叉熵。 + diff --git a/literature_notes/11_videotg_r1.md b/literature_notes/11_videotg_r1.md new file mode 100644 index 0000000..8b15fea --- /dev/null +++ b/literature_notes/11_videotg_r1.md @@ -0,0 +1,14 @@ +# VideoTG-R1: Boosting Video Temporal Grounding via Curriculum Reinforcement Learning on Reflected Boundary Annotations + +- 年份：2025 +- 阅读深度：重点章节阅读 +- 官方链接：https://arxiv.org/html/2510.23397 + +## 摘要（中文转述） + +VideoTG-R1 在视频时间定位中先用边界反思模块识别含糊、不完整或不可靠的标注，再依据样本难度组织训练，并通过格式和时间区间重叠奖励进行 GRPO。论文强调经过筛选的少量高质量边界样本可以比直接使用全部数据更有效，并降低训练成本。 + +## 与当前项目的关系 + +对 EasyCom/Ego4D 最重要的启示可能是标签和边界质量，而不是 RL 本身。下一步应先检查转换附近的标注延迟、音视频同步和窗口歧义，再比较 transition oversampling、课程采样与损失重加权。只有监督式数据治理达到上限后，RL 的增量才容易解释。 + diff --git a/literature_notes/12_tar_tvg.md b/literature_notes/12_tar_tvg.md new file mode 100644 index 0000000..49e9a3b --- /dev/null +++ b/literature_notes/12_tar_tvg.md @@ -0,0 +1,14 @@ +# TAR-TVG: Enhancing VLMs with Timestamp Anchor-Constrained Reasoning for Temporal Video Grounding + +- 年份：2025 +- 阅读深度：摘要级筛选 +- 官方链接：https://arxiv.org/abs/2508.07683 + +## 摘要（中文转述） + +TAR-TVG 把时间戳锚点显式插入视频时序推理过程，并通过逐步收紧的准确性约束训练模型。整体流程结合初始策略优化产生推理轨迹、监督蒸馏以及后续强化学习，使模型在给出最终时间区间前形成受时间锚点约束的中间推理。 + +## 与当前项目的关系 + +它提示可以增加结构化的中间目标，例如“距 wearer onset 还有多少步”或“最近一次 others offset 在何时”，但当前仅做过摘要级筛选。是否适用于连续在线特征，需要进一步核查因果性、输入采样和奖励实现。 + diff --git a/literature_notes/13_video_r2.md b/literature_notes/13_video_r2.md new file mode 100644 index 0000000..f2476d7 --- /dev/null +++ b/literature_notes/13_video_r2.md @@ -0,0 +1,14 @@ +# Video-R2: Reinforcing Consistent and Grounded Reasoning in Multimodal Language Models + +- 年份：2025 +- 阅读深度：摘要级筛选 +- 官方链接：https://arxiv.org/abs/2511.23478 + +## 摘要（中文转述） + +Video-R2 关注视频多模态模型推理过程与最终回答之间的一致性，以及推理是否真正依赖视觉证据。方法结合时间感知监督微调和带时间对齐奖励的策略优化，并设计诊断指标评估推理—答案一致性与视觉依据。 + +## 与当前项目的关系 + +它可能帮助构造“预测是否由近期 AV 证据支持”的辅助评测，但任务比轮次预测更宽泛。当前只适合作为时间对齐奖励的候选文献，不能用来支持 transition RL 的直接有效性结论。 + diff --git a/literature_notes/14_prompt_guided_turn_taking.md b/literature_notes/14_prompt_guided_turn_taking.md new file mode 100644 index 0000000..e3a299e --- /dev/null +++ b/literature_notes/14_prompt_guided_turn_taking.md @@ -0,0 +1,14 @@ +# Prompt-Guided Turn-Taking Prediction + +- 会议：SIGDIAL 2025 +- 阅读深度：摘要级筛选 +- 官方链接：https://aclanthology.org/2025.sigdial-1.9/ + +## 摘要（中文转述） + +该工作把自然语言提示加入 Voice Activity Projection 模型，使轮次预测不仅拟合观察到的对话，还能根据“更快”“更平静”等交互要求改变预测倾向。模型在大规模人类对话上训练，并使用合成提示建立语音动态与控制条件之间的联系。 + +## 与当前项目的关系 + +这提供了一个区别于单纯提高 mAP 的方向：让模型的开口阈值和风格可控。对当前论文，更现实的近期实验是改变决策成本或校准阈值，画出响应积极性与误打断之间的曲线，而不是马上增加 prompt 分支。 + diff --git a/literature_notes/15_multilingual_vap.md b/literature_notes/15_multilingual_vap.md new file mode 100644 index 0000000..bec1801 --- /dev/null +++ b/literature_notes/15_multilingual_vap.md @@ -0,0 +1,14 @@ +# Multilingual Turn-taking Prediction Using Voice Activity Projection + +- 会议：LREC-COLING 2024 +- 阅读深度：摘要级筛选 +- 官方链接：https://aclanthology.org/2024.lrec-main.1036/ + +## 摘要（中文转述） + +该论文研究 Voice Activity Projection 在英语、普通话和日语对话中的跨语言能力，比较单语和多语训练，并分析音高等声学线索对轮次预测的影响。结果表明，多语言训练可以获得接近单语言模型的表现，说明部分轮次线索具有跨语言共享性。 + +## 与当前项目的关系 + +它提醒我们 EasyCom/Ego4D 上的提升可能来自可迁移的韵律线索，而不仅是视觉或语义上下文。未来可将 Qwen 隐状态与轻量 pitch、energy、VAD 特征做受控融合，检验冻结大模型是否丢失了对 transition 最敏感的低层声学信息。 + diff --git a/literature_notes/16_multimodal_turn_taking_backchannel.md b/literature_notes/16_multimodal_turn_taking_backchannel.md new file mode 100644 index 0000000..a0adbf5 --- /dev/null +++ b/literature_notes/16_multimodal_turn_taking_backchannel.md @@ -0,0 +1,14 @@ +# Predicting Turn-Taking and Backchannel in Human-Machine Conversations Using Linguistic, Acoustic, and Visual Signals + +- 会议：ACL 2025 +- 阅读深度：摘要级筛选 +- 官方链接：https://aclanthology.org/2025.acl-long.743/ + +## 摘要（中文转述） + +该工作在面对面人机对话中联合研究轮次转换和 backchannel，融合语言、声学和视觉信号，并报告多模态模型相对既有方法的改进。其任务把正常接管、简短反馈和持续倾听视为可区分的互动事件。 + +## 与当前项目的关系 + +它支持把笼统的 wearer/others 活动进一步拆成 transition 子类型进行诊断。即使训练标签暂时不变，也可以从目标序列派生 onset、offset、overlap onset 和短促 backchannel-like 片段，分别报告性能，判断模型真正改善了哪种动态。 + diff --git a/literature_notes/17_action_sensitivity_learning.md b/literature_notes/17_action_sensitivity_learning.md new file mode 100644 index 0000000..7e2e16c --- /dev/null +++ b/literature_notes/17_action_sensitivity_learning.md @@ -0,0 +1,14 @@ +# Action Sensitivity Learning for Temporal Action Localization + +- 会议：ICCV 2023 +- 阅读深度：摘要级筛选 +- 官方链接：https://openaccess.thecvf.com/content/ICCV2023/html/Shao_Action_Sensitivity_Learning_for_Temporal_Action_Localization_ICCV_2023_paper.html + +## 摘要（中文转述） + +Action Sensitivity Learning 针对时间动作定位中不同类别、不同实例对边界和上下文的敏感程度不同的问题，通过类别级和实例级敏感性建模调整梯度贡献及对比采样，使模型更关注难定位、边界模糊的样本。 + +## 与当前项目的关系 + +它提供了比 RL 更低风险的 transition 强化路径：根据距 onset/offset 的距离、转换类型和样本不确定度重加权，但仍保持标准监督学习。建议先与均匀采样做严格对照，并保证权重归一化，避免仅因提高 transition 占比而破坏整体概率校准。 + diff --git a/literature_notes/18_llm_enhanced_dialogue_management.md b/literature_notes/18_llm_enhanced_dialogue_management.md new file mode 100644 index 0000000..e28f7cb --- /dev/null +++ b/literature_notes/18_llm_enhanced_dialogue_management.md @@ -0,0 +1,14 @@ +# LLM-Enhanced Dialogue Management for Full-Duplex Spoken Dialogue Systems + +- 初版年份：2025；后续修订：2026 +- 阅读深度：摘要级筛选 +- 官方链接：https://arxiv.org/abs/2502.14145 + +## 摘要（中文转述） + +该工作用轻量语言模型构建全双工语音系统的语义式对话管理器，连续处理短语音块并输出有限的控制状态，以决定继续倾听、回应、反馈或处理打断。它强调把高层语义判断作为可插拔模块，而不是让完整生成模型独自承担所有实时控制。 + +## 与当前项目的关系 + +有限状态控制与本项目的小型 turn head 很接近。可借鉴的是把输出空间设计成具有明确动作含义的状态，并单独评估状态转换；但目前仅阅读摘要，需要进一步检查其训练标签、因果延迟和四种控制 token 的确切定义。 + diff --git a/literature_notes/19_aligning_spoken_dialogue_from_interactions.md b/literature_notes/19_aligning_spoken_dialogue_from_interactions.md new file mode 100644 index 0000000..7129af3 --- /dev/null +++ b/literature_notes/19_aligning_spoken_dialogue_from_interactions.md @@ -0,0 +1,14 @@ +# Aligning Spoken Dialogue Models from User Interactions + +- 年份：2025 +- 阅读深度：摘要级筛选 +- 官方链接：https://arxiv.org/abs/2506.21463 + +## 摘要（中文转述） + +该工作尝试从真实用户交互中自动构造语音对话偏好数据，通过时间和语言层面的变化形成大量成对样本，再进行离线偏好对齐。目标是减少完全人工标注的依赖，让模型从实际互动模式中学习更合适的响应行为。 + +## 与当前项目的关系 + +它说明 transition 策略不一定要依赖在线 RL，也可以通过构造偏好对做 DPO 类离线学习。对 EasyCom，可以围绕真实边界生成早开口、晚开口和错误打断的反事实候选；但要警惕这些候选并不改变固定视频中的后续说话者行为，因果意义仍然有限。 + diff --git a/literature_notes/20_spoken_dialogue_rlaif.md b/literature_notes/20_spoken_dialogue_rlaif.md new file mode 100644 index 0000000..4c3e7dd --- /dev/null +++ b/literature_notes/20_spoken_dialogue_rlaif.md @@ -0,0 +1,14 @@ +# Optimizing Conversational Quality in Spoken Dialogue Systems with Reinforcement Learning from AI Feedback + +- 年份：2026 +- 阅读深度：摘要级筛选 +- 官方链接：https://arxiv.org/abs/2601.19063 + +## 摘要（中文转述） + +该工作研究如何用 AI 反馈和多种奖励改善语音对话质量。系统采用块级增量解码，在话轮层面采样偏好候选，并把各块的概率汇总后进行偏好优化，试图同时改善内容质量、互动自然度和流式响应行为。 + +## 与当前项目的关系 + +其块级偏好思想与 Qwen2.5-Omni 的流式提交方式有潜在联系，但当前只做过摘要级筛选。若用于本项目，需要先确认奖励是否真正针对 turn transition、是否包含交互式环境，以及优化变量是生成内容还是独立控制策略。 + diff --git a/literature_notes/README.md b/literature_notes/README.md new file mode 100644 index 0000000..017bc83 --- /dev/null +++ b/literature_notes/README.md @@ -0,0 +1,52 @@ +# ICASSP 2027 文献笔记 + +本目录持久化记录我们在本项目讨论中实际阅读、分析或筛选过的论文。这里的“摘要”均为面向当前课题的中文转述，不是原文摘要的复制。 + +最后更新：2026-08-26 + +## 阅读深度 + +- 深入阅读：检查过方法、实验或附录，可用于设计实验和论证。 +- 重点章节阅读：阅读过与当前问题直接相关的方法或实验部分。 +- 摘要级筛选：主要依据摘要和论文主页判断相关性，只适合作为候选线索。 + +## 论文索引 + +| 编号 | 论文 | 年份 | 阅读深度 | 与当前项目的主要关系 | +|---|---|---:|---|---| +| 01 | [Qwen2.5-Omni Technical Report](01_qwen2_5_omni.md) | 2025 | 深入阅读 | 原生流式多模态语义与 Thinker 状态 | +| 02 | [EgoSpeak](02_egospeak.md) | 2025 | 深入阅读 | 任务定义、数据划分、标签和评测协议 | +| 03 | [Long Short-Term Transformer for Online Action Detection](03_lstr.md) | 2021 | 重点章节阅读 | 在线时序建模与 LSTR-MCE 基线 | +| 04 | [MiniROAD](04_miniroad.md) | 2023 | 重点章节阅读 | 在线检测、时间步损失与高效递归建模 | +| 05 | [ROMA](05_roma.md) | 2026 | 深入阅读 | 流式 AV 单元、持久 KV、Speak Head、末层融合 | +| 06 | [DuplexPO](06_duplexpo.md) | 2026 | 深入阅读 | 对话动态关键窗口和强化学习 | +| 07 | [ASPIRin](07_aspirin.md) | 2026 | 深入阅读 | 将 RL 动作空间投影到 active/inactive | +| 08 | [SoulX-Duplug](08_soulx_duplug.md) | 2026 | 深入阅读 | 显式流式状态预测和低延迟监督学习 | +| 09 | [SALMONN-omni](09_salmonn_omni.md) | 2025 | 重点章节阅读 | 全双工对话、DPO 与策略训练稳定性 | +| 10 | [Tempo-R0](10_tempo_r0.md) | 2025 | 重点章节阅读 | 可验证的时间定位奖励和 GRPO | +| 11 | [VideoTG-R1](11_videotg_r1.md) | 2025 | 重点章节阅读 | 边界反思、样本筛选与课程 RL | +| 12 | [TAR-TVG](12_tar_tvg.md) | 2025 | 摘要级筛选 | 时间戳锚点和分阶段时序推理训练 | +| 13 | [Video-R2](13_video_r2.md) | 2025 | 摘要级筛选 | 时间对齐奖励与视频推理一致性 | +| 14 | [Prompt-Guided Turn-Taking Prediction](14_prompt_guided_turn_taking.md) | 2025 | 摘要级筛选 | 可控式轮次预测 | +| 15 | [Multilingual Turn-taking Prediction Using VAP](15_multilingual_vap.md) | 2024 | 摘要级筛选 | VAP、跨语言轮次线索与泛化 | +| 16 | [Predicting Turn-Taking and Backchannel](16_multimodal_turn_taking_backchannel.md) | 2025 | 摘要级筛选 | 多模态轮次与 backchannel 分类 | +| 17 | [Action Sensitivity Learning](17_action_sensitivity_learning.md) | 2023 | 摘要级筛选 | 边界敏感重加权，作为 RL 的低风险替代 | +| 18 | [LLM-Enhanced Dialogue Management](18_llm_enhanced_dialogue_management.md) | 2025/2026 | 摘要级筛选 | 模块化流式对话控制状态 | +| 19 | [Aligning Spoken Dialogue Models from User Interactions](19_aligning_spoken_dialogue_from_interactions.md) | 2025 | 摘要级筛选 | 从用户交互构造偏好数据 | +| 20 | [Optimizing Conversational Quality with RLAIF](20_spoken_dialogue_rlaif.md) | 2026 | 摘要级筛选 | 块级语音生成与多奖励偏好优化 | + +## 当前综合判断 + +对本项目最直接的主线不是立即把三分类预测改成通用 RL，而是： + +1. 以 Qwen2.5-Omni 原生流式语义和 EgoSpeak 协议为任务基础。 +2. 从 ROMA 借鉴末层聚合、时机与内容解耦以及独立决策头。 +3. 先用 SoulX-Duplug、Action Sensitivity Learning 一类监督式状态或边界强化建立可靠增益。 +4. 若转向交互式闭环训练，再借鉴 DuplexPO 的动态关键窗口和 ASPIRin 的低维动作空间，避免对语义表示进行不受约束的 RL。 + +## 维护约定 + +- 新增论文时，创建独立 Markdown 文件，并在本索引登记阅读深度。 +- 若后来深入阅读过一篇摘要级论文，更新其阅读深度、方法细节和实验结论。 +- 实验设计引用具体数值前，应再次核对论文正文、版本和官方实现。 + diff --git a/test_easycom_head_objectives.py b/test_easycom_head_objectives.py index 523410c..e58036e 100644 --- a/test_easycom_head_objectives.py +++ b/test_easycom_head_objectives.py @@ -10,11 +10,16 @@ import torch from easycom_head import ( TurnPredictionHead, activity_targets_to_four_states, + checkpoint_head_config, checkpoint_loss_name, + expected_transition_reward, four_state_multihorizon_loss, independent_bce_multihorizon_loss, marginal_probabilities_from_logits, output_classes_for_loss, + transition_aware_objective, + transition_targets_from_activity, + weighted_transition_bce_loss, ) @@ -71,6 +76,81 @@ class ObjectiveTests(unittest.TestCase): ) self.assertEqual(tuple(head(torch.randn(3, 8)).shape), (3, 2, 4)) + def test_transition_targets_mark_adjacent_activity_changes(self) -> None: + sequence = torch.tensor( + [ + [1.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 1.0, 1.0], + [0.0, 0.0, 1.0], + ] + ) + actual = transition_targets_from_activity(sequence) + expected = torch.tensor([0.0, 1.0, 0.0, 1.0, 1.0]) + self.assertTrue(torch.equal(actual, expected)) + + def test_transition_auxiliary_head_shapes(self) -> None: + head = TurnPredictionHead( + hidden_size=8, + num_horizons=2, + dropout=0.0, + transition_auxiliary=True, + ) + activity, transition = head.forward_with_transition(torch.randn(3, 8)) + self.assertEqual(tuple(activity.shape), (3, 2, 3)) + self.assertEqual(tuple(transition.shape), (3, 2)) + self.assertEqual(tuple(head(torch.randn(3, 8)).shape), (3, 2, 3)) + + def test_transition_bce_uses_positive_weight_three(self) -> None: + logits = torch.zeros(2) + targets = torch.tensor([0.0, 1.0]) + actual = weighted_transition_bce_loss(logits, targets, pos_weight=3.0) + self.assertAlmostEqual(float(actual), 2.0 * math.log(2.0), places=6) + + def test_expected_transition_reward_is_exact_weighted_correctness(self) -> None: + logits = torch.tensor([2.0, -2.0]) + targets = torch.tensor([1.0, 0.0]) + actual = expected_transition_reward(logits, targets, pos_weight=3.0) + self.assertAlmostEqual( + float(actual), + float(torch.sigmoid(torch.tensor(2.0))), + places=6, + ) + + def test_r1_and_r2_objective_formulas(self) -> None: + activity_loss = torch.tensor(2.0) + logits = torch.zeros(2, requires_grad=True) + targets = torch.tensor([0.0, 1.0]) + transition_loss = weighted_transition_bce_loss( + logits, + targets, + pos_weight=3.0, + ) + r1 = transition_aware_objective( + activity_loss, + logits, + targets, + objective="r1-supervised", + ) + r2 = transition_aware_objective( + activity_loss, + logits, + targets, + objective="r2-expected-reward", + ) + self.assertAlmostEqual( + float(r1["total"].detach()), + 2.0 + 0.2 * float(transition_loss.detach()), + places=6, + ) + self.assertAlmostEqual( + float(r2["total"].detach()), + 2.0 + 0.2 * float(transition_loss.detach()) - 0.1 * 0.5, + places=6, + ) + def test_all_new_objectives_backpropagate_to_marginals(self) -> None: states = torch.randn(4, 8) future_targets = self.targets[:, None, :].repeat(1, 2, 1) @@ -95,6 +175,7 @@ class ObjectiveTests(unittest.TestCase): def test_legacy_checkpoint_defaults_to_lstr(self) -> None: self.assertEqual(checkpoint_loss_name({}), "lstr-mce") + self.assertFalse(checkpoint_head_config({})["transition_auxiliary"]) def test_checkpoint_rejects_loss_output_mismatch(self) -> None: payload = { diff --git a/train_omni_turn_lora.py b/train_omni_turn_lora.py index 724f2ba..d130161 100644 --- a/train_omni_turn_lora.py +++ b/train_omni_turn_lora.py @@ -982,6 +982,7 @@ def _save_weight_checkpoint( "num_input_layers": raw_head.num_input_layers, "layer_fusion": raw_head.layer_fusion, "learnable_layer_mixture": raw_head.learnable_layer_mixture, + "transition_auxiliary": raw_head.transition_auxiliary, }, "initial_head_checkpoint": ( str(args.head_checkpoint.resolve()) if args.head_checkpoint else None diff --git a/train_turn_head.py b/train_turn_head.py index c03cbf4..7ebc8d7 100644 --- a/train_turn_head.py +++ b/train_turn_head.py @@ -28,6 +28,7 @@ from easycom_head import ( NUM_HORIZONS, OFFICIAL_TRAIN_SESSIONS, STEP_SECONDS, + TRANSITION_OBJECTIVE_NAMES, TurnPredictionHead, average_precision_metrics, egospeak_multihorizon_loss, @@ -41,6 +42,8 @@ from easycom_head import ( normalize_sample_name, output_classes_for_loss, output_space_for_loss, + transition_aware_objective, + transition_targets_from_activity, validate_disjoint_sessions, ) @@ -125,6 +128,19 @@ def parse_args() -> argparse.Namespace: "silence/wearer-only/others-only/overlap softmax logits." ), ) + parser.add_argument( + "--transition-objective", + choices=TRANSITION_OBJECTIVE_NAMES, + default="none", + help=( + "none keeps the activity-only baseline; r1-supervised adds a " + "weighted binary transition head; r2-expected-reward additionally " + "maximizes exact expected transition correctness." + ), + ) + parser.add_argument("--transition-loss-weight", type=float, default=0.2) + parser.add_argument("--transition-pos-weight", type=float, default=3.0) + parser.add_argument("--transition-reward-weight", type=float, default=0.1) parser.add_argument("--device", default="cuda:0") parser.add_argument("--num-workers", type=int, default=0) parser.add_argument("--seed", type=int, default=0) @@ -154,6 +170,12 @@ def _validate_args(args: argparse.Namespace) -> None: raise ValueError("--dropout must be in [0, 1)") if args.num_workers < 0: raise ValueError("--num-workers cannot be negative") + if args.transition_loss_weight < 0: + raise ValueError("--transition-loss-weight cannot be negative") + if args.transition_pos_weight <= 0: + raise ValueError("--transition-pos-weight must be positive") + if args.transition_reward_weight < 0: + raise ValueError("--transition-reward-weight cannot be negative") if args.dataset == "ego4d" and args.manifest is None: raise ValueError("--manifest is required when --dataset ego4d") if ( @@ -188,9 +210,17 @@ def _load_examples( expected_mode: str | None, dataset: str, selected_indices: dict[str, set[int]] | None = None, -) -> tuple[torch.Tensor, torch.Tensor, int, int, list[dict[str, Any]]]: +) -> tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + int, + int, + list[dict[str, Any]], +]: all_states: list[torch.Tensor] = [] all_targets: list[torch.Tensor] = [] + all_transition_targets: list[torch.Tensor] = [] metadata: list[dict[str, Any]] = [] hidden_size: int | None = None num_input_layers: int | None = None @@ -203,18 +233,24 @@ def _load_examples( dataset=dataset, ) states, targets, future_indices = data.future_examples(NUM_HORIZONS) + current_indices = future_indices[:, 0] - 1 + activity_sequence = torch.cat( + (data.targets[current_indices, None, :], targets), + dim=1, + ) + transition_targets = transition_targets_from_activity(activity_sequence) dense_example_count = len(states) if selected_indices is not None: if name not in selected_indices: raise ValueError(f"No sparse anchor manifest was supplied for {name}") wanted = selected_indices[name] - current_indices = future_indices[:, 0] - 1 keep = torch.tensor( [int(index) in wanted for index in current_indices.tolist()], dtype=torch.bool, ) states = states[keep] targets = targets[keep] + transition_targets = transition_targets[keep] if not len(states): raise ValueError(f"Sparse anchor selection kept no examples for {name}") if hidden_size is None: @@ -233,12 +269,18 @@ def _load_examples( ) all_states.append(states) all_targets.append(targets) + all_transition_targets.append(transition_targets) metadata.append( { "session": name, "num_states": len(data.states), "num_training_examples": len(states), "num_dense_training_examples": dense_example_count, + "num_positive_transition_targets": int( + transition_targets.sum().item() + ), + "transition_target_count": int(transition_targets.numel()), + "transition_positive_rate": float(transition_targets.mean()), "supervision_policy": ( "transition-sparse-manifest" if selected_indices is not None @@ -257,6 +299,7 @@ def _load_examples( return ( torch.cat(all_states, dim=0), torch.cat(all_targets, dim=0), + torch.cat(all_transition_targets, dim=0), hidden_size, num_input_layers, metadata, @@ -372,7 +415,7 @@ def _checkpoint( history: list[dict[str, Any]], ) -> dict[str, Any]: return { - "format_version": 2, + "format_version": 3, "head": head.state_dict(), "head_config": { "hidden_size": hidden_size, @@ -382,6 +425,7 @@ def _checkpoint( "num_input_layers": num_input_layers, "layer_fusion": args.layer_fusion, "learnable_layer_mixture": args.learnable_layer_mixture, + "transition_auxiliary": head.transition_auxiliary, }, "alignment": { "state_label_offset": 0, @@ -414,6 +458,10 @@ def _checkpoint( "warmup_epochs": args.warmup_epochs, "warmup_factor": args.warmup_factor, "loss": args.loss, + "transition_objective": args.transition_objective, + "transition_loss_weight": args.transition_loss_weight, + "transition_pos_weight": args.transition_pos_weight, + "transition_reward_weight": args.transition_reward_weight, "optimizer": "Adam", "scheduler": "linear_warmup_times_cosine", "seed": args.seed, @@ -502,7 +550,14 @@ def main() -> None: print(f"device: {device}") print(f"train sessions: {', '.join(train_sessions)}") print(f"dev sessions: {', '.join(dev_sessions) if dev_sessions else 'none'}") - states, targets, hidden_size, num_input_layers, feature_metadata = _load_examples( + ( + states, + targets, + transition_targets, + hidden_size, + num_input_layers, + feature_metadata, + ) = _load_examples( args.feature_root, args.target_root, train_sessions, @@ -514,6 +569,11 @@ def main() -> None: f"training examples: {len(states)}, feature_dim={hidden_size}, " f"input_layers={num_input_layers}" ) + print( + "transition targets: " + f"{int(transition_targets.sum().item())}/{transition_targets.numel()} " + f"positive ({float(transition_targets.mean()) * 100:.3f}%)" + ) if num_input_layers > 1 and args.layer_fusion == "single": raise ValueError( f"Features contain {num_input_layers} stacked layers; pass " @@ -530,6 +590,7 @@ def main() -> None: ( dev_states, dev_targets, + _, dev_hidden_size, dev_num_input_layers, _, @@ -553,7 +614,7 @@ def main() -> None: ) print(f"development examples: {len(dev_states)}") - dataset = TensorDataset(states, targets) + dataset = TensorDataset(states, targets, transition_targets) generator = torch.Generator().manual_seed(args.seed) loader = DataLoader( dataset, @@ -573,6 +634,7 @@ def main() -> None: num_input_layers=num_input_layers, learnable_layer_mixture=args.learnable_layer_mixture, layer_fusion=args.layer_fusion, + transition_auxiliary=args.transition_objective != "none", ).to(device) trainable_parameters = sum( parameter.numel() for parameter in head.parameters() if parameter.requires_grad @@ -584,7 +646,11 @@ def main() -> None: ) print( f"loss={args.loss}, output_space={output_space_for_loss(args.loss)}, " - f"head_outputs={num_output_classes}, layer_fusion={args.layer_fusion}, " + f"transition_objective={args.transition_objective}, " + f"activity_outputs={num_output_classes}, " + "transition_outputs=" + f"{NUM_HORIZONS if args.transition_objective != 'none' else 0}, " + f"layer_fusion={args.layer_fusion}, " f"fusion_parameters={fusion_parameters:,}, " f"head_trainable_parameters={trainable_parameters:,}" ) @@ -633,6 +699,9 @@ def main() -> None: for epoch in range(1, args.epochs + 1): head.train() loss_sum = 0.0 + activity_loss_sum = 0.0 + transition_loss_sum = 0.0 + transition_reward_sum = 0.0 example_count = 0 progress = tqdm( loader, @@ -640,7 +709,7 @@ def main() -> None: unit="batch", disable=args.no_progress, ) - for state_batch, target_batch in progress: + for state_batch, target_batch, transition_target_batch in progress: state_batch = state_batch.to( device=device, dtype=torch.float32, @@ -651,25 +720,64 @@ def main() -> None: dtype=torch.float32, non_blocking=True, ) + transition_target_batch = transition_target_batch.to( + device=device, + dtype=torch.float32, + non_blocking=True, + ) optimizer.zero_grad(set_to_none=True) - logits = head(state_batch) - loss = loss_function(logits, target_batch) + if args.transition_objective == "none": + logits = head(state_batch) + activity_loss = loss_function(logits, target_batch) + loss = activity_loss + transition_loss = activity_loss.detach().new_zeros(()) + expected_reward = activity_loss.detach().new_zeros(()) + else: + logits, transition_logits = head.forward_with_transition(state_batch) + activity_loss = loss_function(logits, target_batch) + components = transition_aware_objective( + activity_loss, + transition_logits, + transition_target_batch, + objective=args.transition_objective, + transition_weight=args.transition_loss_weight, + pos_weight=args.transition_pos_weight, + reward_weight=args.transition_reward_weight, + ) + loss = components["total"] + transition_loss = components["transition"] + expected_reward = components["expected_reward"] loss.backward() optimizer.step() scheduler.step() batch_count = len(state_batch) loss_sum += float(loss.detach()) * batch_count + activity_loss_sum += float(activity_loss.detach()) * batch_count + transition_loss_sum += float(transition_loss.detach()) * batch_count + transition_reward_sum += float(expected_reward.detach()) * batch_count example_count += batch_count - progress.set_postfix( - loss=f"{float(loss.detach()):.5f}", - lr=f"{scheduler.get_last_lr()[0]:.3g}", - ) + postfix = { + "loss": f"{float(loss.detach()):.5f}", + "lr": f"{scheduler.get_last_lr()[0]:.3g}", + } + if args.transition_objective != "none": + postfix["transition"] = f"{float(transition_loss.detach()):.5f}" + postfix["reward"] = f"{float(expected_reward.detach()):.5f}" + progress.set_postfix(postfix) train_loss = loss_sum / example_count record: dict[str, Any] = { "epoch": epoch, "train_loss": train_loss, + "train_activity_loss": activity_loss_sum / example_count, "learning_rate": scheduler.get_last_lr()[0], } + if args.transition_objective != "none": + record["train_transition_loss"] = ( + transition_loss_sum / example_count + ) + record["train_expected_transition_reward"] = ( + transition_reward_sum / example_count + ) layer_weights = head.layer_mixture_weights() if layer_weights is not None: record["layer_mixture_weights_oldest_to_newest"] = ( @@ -685,6 +793,16 @@ def main() -> None: ) + "]" ) + transition_text = ( + "" + if args.transition_objective == "none" + else ( + f", activity_loss={record['train_activity_loss']:.6f}, " + f"transition_loss={record['train_transition_loss']:.6f}, " + "expected_transition_reward=" + f"{record['train_expected_transition_reward']:.6f}" + ) + ) if dev_sessions: assert dev_states is not None and dev_targets is not None @@ -700,7 +818,8 @@ def main() -> None: record["dev_valid_mean_map"] = dev_map print( f"epoch {epoch:02d}: train_loss={train_loss:.6f}, " - f"dev_valid_mean_mAP={dev_map * 100:.3f}{weight_text}" + f"dev_valid_mean_mAP={dev_map * 100:.3f}" + f"{transition_text}{weight_text}" ) if dev_map > float(best_dev_map): best_dev_map = dev_map @@ -725,7 +844,8 @@ def main() -> None: print(f"saved new best checkpoint: {args.output}") else: print( - f"epoch {epoch:02d}: train_loss={train_loss:.6f}{weight_text}" + f"epoch {epoch:02d}: train_loss={train_loss:.6f}" + f"{transition_text}{weight_text}" ) history.append(record)