From 2f9ccfad8876e5503902195c011c120affdf4b66 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Mon, 14 Sep 2026 23:43:28 +0800 Subject: [PATCH] Align behavioral event annotation protocol --- docs/easycom_event_review_tool.md | 11 +- manuscript/main/main.tex | 218 ++++++++++++++---------------- prepare_easycom_eot_int_review.py | 23 ++-- 3 files changed, 123 insertions(+), 129 deletions(-) diff --git a/docs/easycom_event_review_tool.md b/docs/easycom_event_review_tool.md index 21d1dd3..a5faa8c 100644 --- a/docs/easycom_event_review_tool.md +++ b/docs/easycom_event_review_tool.md @@ -24,8 +24,8 @@ python3 prepare_easycom_eot_int_review.py \ The generator emits two independent files: -- `easycom_eot_int_candidates.jsonl` contains EOT opportunities that pair a - non-wearer release with a later floor-level wearer `0->1` behavior anchor +- `easycom_eot_int_candidates.jsonl` contains floor-level wearer `0->1` + behavior anchors retrospectively paired with a preceding non-wearer release (`WEARER_ONSET`, `EOT_OPPORTUNITY_CANDIDATE`), only episode-ending wearer `1->0` boundaries where a new non-wearer onset satisfies `wearer_floor_start < other_onset < wearer_offset` and the same activity @@ -49,9 +49,10 @@ continues across the resumed onset conservatively preserves a new wearer-entry candidate. Every decision is stored in `temporal_evidence`, including the raw gap, feedback IDs, competing IDs, threshold, and `floor_episode_merged` flag. -After each non-wearer release, EOT construction scans forward until a -floor-level wearer onset or a competing non-wearer onset. Silence length alone -does not invalidate the opportunity. Each primary EOT candidate stores both +For each floor-level wearer onset, EOT construction searches backward for the +most recent non-wearer release and rejects the pair if wearer speech or a +competing non-wearer onset intervenes. Silence length alone does not invalidate +the opportunity. Each primary EOT candidate stores both `opportunity_anchor_seconds` and `behavior_anchor_seconds`, plus their response delay. diff --git a/manuscript/main/main.tex b/manuscript/main/main.tex index 1e85ff7..aa310b0 100644 --- a/manuscript/main/main.tex +++ b/manuscript/main/main.tex @@ -232,21 +232,52 @@ camera-wearer, and aggregate non-wearer activity. Second, the original participant timeline is retained to derive committed interaction events for behavioral evaluation. -An \emph{EOT-response} is an observed wearer take-turn after a non-wearer floor -release; an \emph{INT-yield} is an observed wearer floor release after a -non-wearer takeover during wearer speech. +\textbf{Behavioral event annotation.} +We use floor-level wearer transitions as high-recall anchors for positive-event +annotation. Each wearer onset is examined retrospectively for a preceding +non-wearer floor release and labeled as an EOT-response only when the subsequent +wearer speech constitutes a genuine floor take. Similarly, each wearer offset +is examined for a new competing non-wearer onset during the current wearer turn +and labeled as INT-yield only when the competing speaker genuinely takes over +the floor. We use permissive temporal retrieval and let an LLM jointly inspect +participant-resolved timing and transcripts to provide a semantic pre-label, +followed by mandatory human adjudication. Separately, we mine regions with no +wearer response that nevertheless constitute false-positive opportunities: +same-speaker pauses and other-to-other transitions for EOT, and backchannels or +non-takeover overlaps for INT. Only human-confirmed positives and hard negatives +enter evaluation. Appendix~\ref{app:event_annotation} gives the full protocol. -\textbf{Hard-negative region definition.} -Committed positive events and hard-negative regions are derived separately -from the participant-resolved speech timeline. Positive EOT-response and -INT-yield events are anchored by observed wearer behavior. In contrast, -hard-negative regions do not require any wearer transition. For example, a -mid-turn pause followed by the same non-wearer resuming speech, or an -other-to-other floor transfer with the wearer remaining silent throughout, -defines an EOT hard-negative region: a take-turn commitment within this region -is counted as a false positive. Likewise, brief backchannels or non-takeover -overlaps during wearer speech define INT hard-negative regions. Ambiguous or -annotation-corrupted regions are excluded from scoring. +\begin{table}[t] +\caption{Behavioral-event annotation statistics after human adjudication. +Counts must be filled only after the candidate protocol and annotation set are +frozen. Scoring-eligible totals include confirmed positives and hard negatives +only.} +\label{tab:behavior_annotation_stats} +\centering +\small +\begin{tabular}{lcc} +\hline +Annotation outcome & Train/dev & Test \\ +\hline +\multicolumn{3}{l}{\emph{EOT-response}}\\ +Positive: EOT-response & \todo{--} & \todo{--} \\ +HN: same-speaker pause & \todo{--} & \todo{--} \\ +HN: other-to-other transition & \todo{--} & \todo{--} \\ +HN: wearer backchannel & \todo{--} & \todo{--} \\ +\hline +\multicolumn{3}{l}{\emph{INT-yield}}\\ +Positive: INT-yield & \todo{--} & \todo{--} \\ +HN: other backchannel & \todo{--} & \todo{--} \\ +HN: non-takeover overlap & \todo{--} & \todo{--} \\ +HN: wearer holds floor & \todo{--} & \todo{--} \\ +\hline +Not applicable & \todo{--} & \todo{--} \\ +Excluded / unscorable & \todo{--} & \todo{--} \\ +Unresolved & \todo{--} & \todo{--} \\ +\textbf{Scoring-eligible total} & \todo{--} & \todo{--} \\ +\hline +\end{tabular} +\end{table} Participant identity is used only to construct these evaluation annotations and is never provided to the prediction model. Provided person boxes are used @@ -518,118 +549,77 @@ do not make the method claim.} \section{LLM-Assisted Behavioral Event Annotation} \label{app:event_annotation} -EasyCom provides participant-resolved speech intervals, transcripts, and -target-of-speech metadata, but raw speech transitions do not uniquely specify -conversational floor function. A wearer onset can be a genuine floor entry, a -backchannel, or a within-turn resumption; a wearer offset can reflect yielding, -natural completion, or a temporary pause. We therefore construct a separate -behavioral-event layer while preserving the original speech intervals used for -dense activity targets. The pipeline has three stages: deterministic candidate -construction, language-model-assisted semantic pre-labeling, and mandatory -human adjudication. The language model never supplies final ground truth. +EasyCom's raw speech boundaries are acoustic annotations rather than floor +labels. We preserve them unchanged for dense forecasting and create a separate +behavioral-event layer. Its design principle is to anchor an observed wearer +behavior first and then retrieve the conversational opportunity that could make +that behavior meaningful. False-positive opportunities are mined independently +because, by definition, many contain no wearer transition. -\subsection{Deterministic candidate construction} +\subsection{Positive-event mining} -\textbf{Floor-level wearer entries.} -Raw wearer onsets are conditionally grouped into floor-level episodes. Let -$g$ be the silence from the preceding wearer offset to a new raw onset. In the -current audit configuration, an onset is stored as a -\textsc{Wearer-Resume} control rather than proposed as an EOT entry when -$g\leq0.8$~s and the gap contains no plausible competing non-wearer floor -activity. A non-feedback contribution that occupies at least $0.4$~s of the -gap, or continues across the resumed wearer onset, conservatively preserves a -new entry candidate. A short ($\leq1.0$~s) feedback-like lexical or nonverbal -contribution is retained as evidence but does not by itself split the wearer -episode. These rules group events only; they do not modify the source -annotations. +\textbf{Floor-level transitions.} +Short wearer-silence gaps are conditionally merged before positive mining. In +the current audit configuration, a raw onset after at most 0.8~s of silence is +treated as a within-turn resumption unless the gap contains sustained competing +non-wearer activity. Short feedback-like activity does not by itself split the +wearer episode. This grouping changes only the behavioral-event layer, not the +source intervals or dense targets. -Following each candidate non-wearer release, the generator opens an EOT -opportunity and scans forward until a floor-level wearer entry or a competing -non-wearer onset terminates it. A retained \textsc{Wearer-Onset} candidate -therefore contains two immutable anchors: the non-wearer release -(\emph{opportunity anchor}) and the subsequent floor-level wearer onset -(\emph{behavior anchor}). The response delay is stored explicitly. Silence -duration alone does not cancel the opportunity; an unpaired wearer onset is -kept only as an audit control and does not enter the primary EOT queue. +\textbf{EOT-response.} +Starting from a floor-level wearer onset at $t_b$, we search backward for a +plausible non-wearer floor release at $t_r$. A candidate is retained only when +the interval from $t_r$ to $t_b$ contains neither intervening wearer speech nor +a competing non-wearer onset. The record stores both $t_r$ as the +\emph{opportunity anchor} and $t_b$ as the \emph{behavior anchor}, together with +the response delay $t_b-t_r$. Silence duration alone does not reject a +candidate. The LLM and human annotator then decide whether the release is +conversationally valid and whether the wearer speech is a genuine floor take. +Unpaired wearer onsets remain audit controls. -\textbf{EOT hard negatives.} -Two participant-resolved timing rules independently propose regions in which -the wearer should remain silent. A \textsc{Same-Speaker-Pause} region is -created when a non-wearer resumes within 2~s and both the wearer and all other -participants remain silent in the intervening gap. An -\textsc{Other-to-Other} region is created when a different non-wearer begins -within 1.5~s, the original speaker does not first resume, and the wearer stays -silent through the handover. The semantic pre-labeler can additionally assign -\textsc{EOT-Wearer-Backchannel} to a floor-level onset that is interpretable -as feedback rather than a response. - -\textbf{INT candidates and hard negatives.} -Let $t_w$ be the start of the current floor-level wearer episode, $t_o$ a new -non-wearer onset, and $t_y$ the episode-ending wearer offset. A potential -\textsc{INT-Yield} is proposed only when +\textbf{INT-yield.} +Starting from an episode-ending wearer offset $t_y$, we search backward inside +the current wearer episode for a new competing non-wearer onset $t_o$. With +$t_w$ denoting the wearer-episode start, an INT candidate must satisfy \begin{equation} - t_w < t_o < t_y < t_o^{\mathrm{off}}, + t_w < t_o < t_y < t_o^{\mathrm{off}}. \label{eq:int_candidate} \end{equation} -with a one-frame (50-ms) tolerance only at the offset boundary in the current -audit. Thus the competing activity must begin after the wearer acquired the -floor and must continue across the wearer yield. Pre-existing non-wearer -activity, post-completion onsets, offsets internal to merged wearer episodes, -and offsets without crossing activity are retained separately as -\textsc{Wearer-Offset-Control} records and are not sent for semantic -pre-labeling. Thus ordinary completion-to-silence cases do not enter the -primary INT review queue. +The current audit allows a one-frame (50-ms) tolerance only at the offset +boundary. Pre-existing other activity, post-completion onsets, offsets inside a +merged wearer episode, and completion-to-silence remain controls. Timing only +retrieves a possible interruption; the LLM and human annotator decide whether +the competitor genuinely takes over and the wearer yields. -INT hard-negative regions are also proposed without requiring a wearer -offset. If a non-wearer contribution ends while the wearer remains active -throughout it and for at least another 0.4~s, the region is labeled by rule as -\textsc{INT-Other-Backchannel} when it is short and feedback-like, and -\textsc{INT-Non-Takeover-Overlap} otherwise. These timing labels are candidate -priors, not final semantic annotations. +\subsection{False-positive opportunity mining} -\subsection{Semantic pre-labeling} +Hard-negative regions are retrieved independently of positive wearer +transitions. For EOT, a same-speaker pause followed by resumption and an +other-to-other handover while the wearer stays silent are false-entry +opportunities. For INT, a short non-wearer backchannel or a non-takeover overlap +during which the wearer retains and continues the floor is a false-yield +opportunity. The timing rules provide candidate types, not final semantic +labels. -For every primary candidate, the annotation assistant receives the candidate -source and rule type, immutable timestamps, participant-resolved activity, -neighboring onsets and offsets, gap and overlap measurements, transcript -context, and target-of-speech metadata when available. Participant identities -are used only inside this annotation package. The assistant is instructed to -treat timing as factual, not to invent prosody, gaze, gesture, intention, or -other unavailable cues, and not to equate onset with floor entry, offset with -yield, silence with EOT, or overlap with interruption. EasyCom's \texttt{[L]} -marker is treated as evidence of laughter only: it is not by itself a sustained -floor take or takeover. +\subsection{Semantic pre-labeling and human adjudication} -The allowed decisions are \textsc{Positive}, \textsc{Hard-Negative}, -\textsc{Not-Applicable}, \textsc{Exclude}, and \textsc{Unsure}. Positive -events are typed as \textsc{EOT-Response} or \textsc{INT-Yield}; hard -negatives use a fixed subtype vocabulary. The current pre-labeling run uses -\texttt{deepseek-v4-flash}, temperature zero, and two passes: a primary pass -and a skeptical pass that seeks alternative explanations. If they disagree on -the verdict, event type, or hard-negative subtype, the combined suggestion is -\textsc{Unsure}. Model identifier, prompt version, per-request prompt hashes, -both raw decisions, confidence values, and explanations are stored with every -candidate. +The annotation assistant jointly receives immutable timestamps, +participant-resolved activity, neighboring boundaries, gap and overlap +measurements, transcripts, and target-of-speech metadata when available. It is +instructed not to equate onset with entry, offset with yield, silence with EOT, +or overlap with interruption, and not to invent unavailable prosody, gaze, or +intent. EasyCom's \texttt{[L]} marker supplies laughter evidence only and is not +by itself a sustained floor take or takeover. -\subsection{Human adjudication and scoring eligibility} - -Every AI-pre-labeled candidate enters an event-centered interface containing -the original video, local participant timeline, transcript, deterministic -timing evidence, and the AI suggestion. The adjudicator performs one primary -action: accept the complete AI label, relabel it, or exclude an unscorable -sample. Relabeling exposes the same constrained verdict, event-type, and -hard-negative subtype vocabulary. A confidence level is required; notes are -optional, and an optional $\pm0.2$~s boundary correction is available only for -an evident source-annotation error. The original event time is always retained. - -Only the completed human decision is eligible to become semantic ground truth. -\textsc{Not-Applicable} denotes a valid observation outside the committed -EOT/INT task, whereas \textsc{Exclude} denotes a region that cannot be scored -reliably because of annotation corruption, a recording boundary, missing -context, timestamp corruption, or another unresolved defect. Neither category -enters the primary event denominator. Transcripts, participant identities, -target-of-speech metadata, annotation-model outputs, and future observations -are never inputs to the evaluated forecasting model. +We use \texttt{deepseek-v4-flash} at temperature zero with a primary and a +skeptical pass. Disagreement on verdict, event type, or hard-negative subtype +produces an \textsc{Unsure} suggestion. Every candidate then receives mandatory +human adjudication: accept the complete pre-label, relabel it, or exclude an +unscorable region. Only human-confirmed \textsc{EOT-Response}, +\textsc{INT-Yield}, and hard-negative opportunities enter evaluation; +\textsc{Not-Applicable}, \textsc{Exclude}, and unresolved candidates do not. +Annotation-only transcripts, identities, target-of-speech metadata, LLM output, +and future observations are never supplied to the evaluated forecasting model. \todo{Before freezing this annotation protocol, audit randomly sampled continuous timeline segments without candidate hints to estimate candidate diff --git a/prepare_easycom_eot_int_review.py b/prepare_easycom_eot_int_review.py index 9ed314b..ceaedc1 100644 --- a/prepare_easycom_eot_int_review.py +++ b/prepare_easycom_eot_int_review.py @@ -422,18 +422,21 @@ def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[ reason="short raw wearer pause without sustained competing other takeover; merged into the existing floor episode", ), ("WEARER_RESUME", frame, ())) - # A1: open an opportunity at a non-wearer release and pair it with the - # subsequent floor-level wearer entry only if no competing other onset - # or wearer activity intervenes. Long silence alone does not invalidate - # the opportunity. Unpaired floor onsets are retained as controls. + # A1: anchor the observed floor-level wearer entry, then retrieve the + # most recent preceding non-wearer release. Keep the pair only if no + # competing other onset or wearer activity intervenes. Long silence + # alone does not invalidate the opportunity. Unpaired floor onsets are + # retained as controls. paired_floor_onsets: set[int] = set() - for release in other_rows: - opportunity_frame = release.dense_end_frame_exclusive - onset_frame = next( - (value for value in floor_onsets if value >= opportunity_frame), None - ) - if onset_frame is None: + for onset_frame in floor_onsets: + releases = [ + row for row in other_rows + if row.dense_end_frame_exclusive <= onset_frame + ] + release = max(releases, key=lambda row: row.dense_end_frame_exclusive, default=None) + if release is None: continue + opportunity_frame = release.dense_end_frame_exclusive wearer_silent = ( opportunity_frame == onset_frame or _all_state(