#!/usr/bin/env python3 """Generate wearer-centric EasyCom entry/INT candidates for AI and human review. This stage never changes EasyCom speech boundaries and never claims semantic ground truth. Every observed wearer onset and offset is retained as a semantic candidate. Deterministic code only records participant-resolved timing, transcripts, and target-of-speech facts; it never decides floor entry, within-turn resumption, or takeover semantics. All timing windows are CLI parameters, and raw evidence is retained for DeepSeek pre-labeling and human correction. """ from __future__ import annotations import argparse from collections import Counter, defaultdict from dataclasses import asdict, dataclass import json from pathlib import Path import random from typing import Any, Iterable, Sequence from derive_easycom_interaction_events import ( CAMERA_WEARER_ID, SOURCE_FPS, Clip, Rules, Utterance, build_clips, is_lexical_feedback, is_pure_nonverbal, load_utterances, ) from easycom_manifest import ALL_SESSIONS, TEST_SESSIONS SCHEMA_VERSION = 6 PROTOCOL_VERSION = "easycom-wearer-entry-int-candidates-v8" @dataclass(frozen=True) class CandidateConfig: context_seconds: float = 3.0 relation_window_seconds: float = 3.0 same_speaker_pause_max_seconds: float = 2.0 other_handover_max_seconds: float = 1.5 short_vocalization_max_seconds: float = 1.0 continuation_min_seconds: float = 0.4 int_offset_tolerance_seconds: float = 0.05 @dataclass(frozen=True) class ProtocolCandidate: schema_version: int protocol_version: str uid: str event_id: str session: str split: str task: str event_type: str candidate_source: str rule_candidate_type: str automatic_protocol_label: str automatic_hard_negative_type: str | None automatic_confidence: str anchor_type: str anchor_dense_frame: int anchor_dense_seconds: float event_time: float region_start_seconds: float region_end_seconds: float clip_stem: str source_start_frame: int previous_other_participant_ids: tuple[int, ...] incoming_other_participant_ids: tuple[int, ...] temporal_evidence: dict[str, Any] speaker_timeline: tuple[dict[str, Any], ...] context: tuple[dict[str, Any], ...] automatic_reason: str ai_verdict: None = None ai_event_type: None = None ai_subtype: None = None ai_confidence: None = None ai_reason: None = None human_status: str = "UNREVIEWED" human_verdict: None = None human_event_type: None = None human_subtype: None = None exclude_reason: None = None def requires_semantic_adjudication(row: dict[str, Any]) -> bool: """Return whether a stored row belongs in the primary AI/human queue. Explicit legacy control sources are authoritative. Current WEARER_OFFSET rows are always semantic candidates: crossing/order fields are evidence, never admission gates. """ source = row.get("candidate_source") if source in {"WEARER_ONSET_CONTROL", "WEARER_OFFSET_CONTROL", "WEARER_RESUME"}: return False return True def _fill_states(rows: Sequence[Utterance], *, segment_start: int, segment_end: int, wearer_id: int) -> tuple[bytearray, bytearray]: length = segment_end - segment_start wearer, others = bytearray(length), bytearray(length) for row in rows: left = max(segment_start, row.dense_start_frame) - segment_start right = min(segment_end, row.dense_end_frame_exclusive) - segment_start if right <= left: continue target = wearer if row.participant_id == wearer_id else others target[left:right] = b"\x01" * (right - left) return wearer, others def _run_after(state: bytearray, index: int, value: bool) -> int: end = index while end < len(state) and bool(state[end]) == value: end += 1 return end - index def _run_before(state: bytearray, index: int, value: bool) -> int: start = index - 1 while start >= 0 and bool(state[start]) == value: start -= 1 return index - start - 1 def _all_state(state: bytearray, start: int, stop: int, value: bool) -> bool: left, right = max(0, start), min(len(state), stop) return right > left and all(bool(state[i]) == value for i in range(left, right)) def _active_other_ids(rows: Sequence[Utterance], frame: int, *, wearer_id: int) -> tuple[int, ...]: return tuple(sorted({ row.participant_id for row in rows if row.participant_id != wearer_id and row.dense_start_frame <= frame < row.dense_end_frame_exclusive })) def _other_ids_crossing_offset( rows: Sequence[Utterance], *, frame: int, wearer_id: int, tolerance_frames: int ) -> tuple[int, ...]: """Return other speakers whose annotated interval crosses a wearer offset. With zero tolerance this records the strict temporal relation ``other.start < wearer_offset < other.end``. A one-frame tolerance accepts an annotation boundary at the same 20 Hz frame, but not an other onset one or more frames after the wearer offset. """ return tuple(sorted({ row.participant_id for row in rows if row.participant_id != wearer_id and row.dense_start_frame < frame + tolerance_frames and row.dense_end_frame_exclusive > frame - tolerance_frames })) def _wearer_gap_evidence( rows: Sequence[Utterance], *, gap_start_frame: int, onset_frame: int, wearer_id: int, ) -> dict[str, Any]: """Return descriptive other-speaker facts within a wearer-silence gap.""" gap_rows = [ row for row in rows if row.participant_id != wearer_id and row.dense_start_frame < onset_frame and row.dense_end_frame_exclusive > gap_start_frame ] max_gap_coverage = 0 intervals: list[dict[str, Any]] = [] for row in gap_rows: coverage = max( 0, min(row.dense_end_frame_exclusive, onset_frame) - max(row.dense_start_frame, gap_start_frame), ) max_gap_coverage = max(max_gap_coverage, coverage) intervals.append({ "uid": row.uid, "participant_id": row.participant_id, "onset_seconds": row.dense_start_seconds, "offset_seconds": row.dense_end_seconds, "gap_overlap_seconds": coverage / SOURCE_FPS, "continues_across_wearer_onset": row.dense_end_frame_exclusive > onset_frame, "transcription": row.transcription, "target_of_speech": list(row.target_of_speech), }) return { "gap_other_participant_ids": sorted({row.participant_id for row in gap_rows}), "gap_other_intervals": intervals, "max_other_gap_coverage_seconds": max_gap_coverage / SOURCE_FPS, } def _clip_for_frame(clips: Sequence[Clip], frame: int) -> Clip | None: return next((clip for clip in clips if clip.dense_start_frame <= frame < clip.dense_start_frame + clip.num_frames), None) def _context_rows(rows: Sequence[Utterance], *, frame: int, context_frames: int, wearer_id: int) -> tuple[dict[str, Any], ...]: left, right = frame - context_frames, frame + context_frames return tuple({ "uid": row.uid, "participant_id": row.participant_id, "participant_role": "wearer" if row.participant_id == wearer_id else "other", "start_seconds": row.dense_start_seconds, "end_seconds": row.dense_end_seconds, "duration_seconds": row.duration_seconds, "transcription": row.transcription, "target_of_speech": list(row.target_of_speech), } for row in rows if row.dense_end_frame_exclusive > left and row.dense_start_frame < right) def _relation_evidence(rows: Sequence[Utterance], *, frame: int, wearer_id: int, window_frames: int) -> dict[str, Any]: previous = [row for row in rows if row.participant_id != wearer_id and row.dense_end_frame_exclusive <= frame and frame - row.dense_end_frame_exclusive <= window_frames] following = [row for row in rows if row.participant_id != wearer_id and row.dense_start_frame >= frame and row.dense_start_frame - frame <= window_frames] prev = max(previous, key=lambda row: row.dense_end_frame_exclusive, default=None) nxt = min(following, key=lambda row: row.dense_start_frame, default=None) active_rows = [row for row in rows if row.participant_id != wearer_id and row.dense_start_frame < frame < row.dense_end_frame_exclusive] overlapping = sorted({row.participant_id for row in active_rows}) resumes = [] if prev is None else [row for row in following if row.participant_id == prev.participant_id] resumed = min(resumes, key=lambda row: row.dense_start_frame, default=None) nearby_onsets = [row for row in rows if row.participant_id != wearer_id and abs(row.dense_start_frame - frame) <= window_frames] closest_onset = min(nearby_onsets, key=lambda row: abs(row.dense_start_frame - frame), default=None) return { "previous_speaker_id": None if prev is None else prev.participant_id, "previous_speaker_offset_seconds": None if prev is None else prev.dense_end_seconds, "gap_from_previous_speaker_offset_seconds": ( None if prev is None else (frame - prev.dense_end_frame_exclusive) / SOURCE_FPS), "overlapping_other_ids": overlapping, "other_overlap_at_event_seconds": max( [0.0] + [(row.dense_end_frame_exclusive - frame) / SOURCE_FPS for row in active_rows] ), "next_speaker_id": None if nxt is None else nxt.participant_id, "next_other_onset_seconds": None if nxt is None else nxt.dense_start_seconds, "previous_speaker_resumes": resumed is not None, "previous_speaker_resume_delay_seconds": ( None if resumed is None or prev is None else (resumed.dense_start_frame - prev.dense_end_frame_exclusive) / SOURCE_FPS ), "closest_other_onset_relative_seconds": ( None if closest_onset is None else (closest_onset.dense_start_frame - frame) / SOURCE_FPS ), } def _candidate(*, session: str, task: str, source: str, rule_type: str, anchor_type: str, frame: int, region_start: int, region_end: int, label: str, subtype: str | None, previous_ids: tuple[int, ...], incoming_ids: tuple[int, ...], evidence: dict[str, Any], clips: Sequence[Clip], rows: Sequence[Utterance], wearer_id: int, context_frames: int, reason: str) -> ProtocolCandidate | None: clip = _clip_for_frame(clips, frame) if clip is None: return None participant_key = "prev-" + ("-".join(f"p{x}" for x in previous_ids) or "none") participant_key += "__in-" + ("-".join(f"p{x}" for x in incoming_ids) or "none") uid = f"{session}/{source.lower()}/{frame:08d}/{participant_key}" context = _context_rows(rows, frame=frame, context_frames=context_frames, wearer_id=wearer_id) return ProtocolCandidate( schema_version=SCHEMA_VERSION, protocol_version=PROTOCOL_VERSION, uid=uid, event_id=uid, session=session, split="test" if session in TEST_SESSIONS else "train", task=task, event_type=source.lower(), candidate_source=source, rule_candidate_type=rule_type, automatic_protocol_label=label, automatic_hard_negative_type=subtype, automatic_confidence="rule_candidate", anchor_type=anchor_type, anchor_dense_frame=frame, anchor_dense_seconds=frame / SOURCE_FPS, event_time=frame / SOURCE_FPS, region_start_seconds=region_start / SOURCE_FPS, region_end_seconds=region_end / SOURCE_FPS, clip_stem=clip.stem, source_start_frame=frame - clip.dense_start_frame + 1, previous_other_participant_ids=previous_ids, incoming_other_participant_ids=incoming_ids, temporal_evidence=evidence, speaker_timeline=context, context=context, automatic_reason=reason, ) def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[Clip], *, session: str, wearer_id: int = CAMERA_WEARER_ID, config: CandidateConfig | None = None, window_seconds: float | None = None, context_seconds: float | None = None) -> list[ProtocolCandidate]: """Return primary semantic candidates plus separate offset controls.""" cfg = config or CandidateConfig() values = asdict(cfg) if window_seconds is not None: values["relation_window_seconds"] = window_seconds if context_seconds is not None: values["context_seconds"] = context_seconds cfg = CandidateConfig(**values) if any(value < 0 for value in asdict(cfg).values()) or cfg.relation_window_seconds == 0: raise ValueError("candidate timing parameters must be non-negative and relation window positive") context_frames = round(cfg.context_seconds * SOURCE_FPS) relation_frames = round(cfg.relation_window_seconds * SOURCE_FPS) pause_frames = round(cfg.same_speaker_pause_max_seconds * SOURCE_FPS) handover_frames = round(cfg.other_handover_max_seconds * SOURCE_FPS) short_frames = round(cfg.short_vocalization_max_seconds * SOURCE_FPS) continuation_frames = round(cfg.continuation_min_seconds * SOURCE_FPS) int_offset_tolerance_frames = round(cfg.int_offset_tolerance_seconds * SOURCE_FPS) rows_by_segment: dict[int, list[Utterance]] = defaultdict(list) clips_by_segment: dict[int, list[Clip]] = defaultdict(list) for row in utterances: rows_by_segment[row.segment_id].append(row) for clip in clips: clips_by_segment[clip.segment_id].append(clip) candidates: list[ProtocolCandidate] = [] seen: set[tuple[str, int, tuple[int, ...]]] = set() def add(item: ProtocolCandidate | None, key: tuple[str, int, tuple[int, ...]]) -> None: if item is not None and key not in seen: seen.add(key) candidates.append(item) for segment_id, segment_clips in sorted(clips_by_segment.items()): segment_clips.sort(key=lambda clip: clip.dense_start_frame) rows = sorted(rows_by_segment.get(segment_id, []), key=lambda row: (row.dense_start_frame, row.dense_end_frame_exclusive, row.uid)) segment_start = segment_clips[0].dense_start_frame segment_end = segment_clips[-1].dense_start_frame + segment_clips[-1].num_frames wearer, others = _fill_states(rows, segment_start=segment_start, segment_end=segment_end, wearer_id=wearer_id) other_rows = [row for row in rows if row.participant_id != wearer_id] # A: retain every raw wearer onset and offset. Gap and nearby-speaker # properties are evidence only; they never suppress a semantic item. onset_records: dict[int, dict[str, Any]] = {} raw_offsets: list[int] = [] for index in range(1, len(wearer)): frame = segment_start + index if not wearer[index - 1] and wearer[index]: gap_frames = _run_before(wearer, index, False) previous_wearer_index = index - gap_frames - 1 has_previous_wearer = previous_wearer_index >= 0 and bool(wearer[previous_wearer_index]) previous_wearer_offset = segment_start + index - gap_frames gap_evidence = _wearer_gap_evidence( rows, gap_start_frame=previous_wearer_offset, onset_frame=frame, wearer_id=wearer_id, ) if has_previous_wearer else { "gap_other_participant_ids": [], "gap_other_intervals": [], "max_other_gap_coverage_seconds": 0.0, } evidence = _relation_evidence(rows, frame=frame, wearer_id=wearer_id, window_frames=relation_frames) evidence.update( wearer_speech_duration_after_onset_seconds=_run_after(wearer, index, True) / SOURCE_FPS, previous_wearer_silence_duration_seconds=gap_frames / SOURCE_FPS, previous_wearer_offset_seconds=( previous_wearer_offset / SOURCE_FPS if has_previous_wearer else None ), original_timestamp_seconds=frame / SOURCE_FPS, **gap_evidence, ) onset_records[frame] = {"evidence": evidence} elif wearer[index - 1] and not wearer[index]: raw_offsets.append(frame) # A1: every observed raw wearer onset is an unresolved entry candidate. # Every nearby preceding other offset is supplied with factual relation # flags; no release is declared admissible or causal by this stage. for onset_frame in sorted(onset_records): release_options: list[tuple[Utterance, int]] = [] for release in other_rows: opportunity_frame = release.dense_end_frame_exclusive if opportunity_frame > onset_frame or onset_frame - opportunity_frame > relation_frames: continue release_options.append((release, opportunity_frame)) release_options.sort( key=lambda item: (item[1], item[0].participant_id, item[0].uid) ) earliest_opportunity_frame = ( release_options[0][1] if release_options else max(segment_start, onset_frame - relation_frames) ) option_rows = [] for release, release_frame in release_options: interval_left = release_frame - segment_start interval_right = onset_frame - segment_start option_rows.append({ "uid": release.uid, "anchor_type": "other_offset", "anchor_frame": release_frame, "anchor_seconds": release_frame / SOURCE_FPS, "participant_id": release.participant_id, "transcription": release.transcription, "target_of_speech": list(release.target_of_speech), "response_delay_seconds": (onset_frame - release_frame) / SOURCE_FPS, "wearer_active_between_release_and_onset": any( bool(wearer[i]) for i in range(max(0, interval_left), min(len(wearer), interval_right)) ), "other_onset_between_release_and_onset": any( release_frame <= row.dense_start_frame < onset_frame for row in other_rows ), "other_active_at_wearer_onset": bool( _active_other_ids(rows, onset_frame, wearer_id=wearer_id) ), }) evidence = dict(onset_records[onset_frame]["evidence"]) evidence.update( opportunity_options=option_rows, opportunity_anchor_type=None, opportunity_anchor_frame=None, opportunity_anchor_seconds=None, behavior_anchor_type="wearer_raw_onset", behavior_anchor_frame=onset_frame, behavior_anchor_seconds=onset_frame / SOURCE_FPS, preceding_release_options_present=bool(option_rows), minimum_response_delay_seconds=( None if not option_rows else min(option["response_delay_seconds"] for option in option_rows) ), maximum_response_delay_seconds=( None if not option_rows else max(option["response_delay_seconds"] for option in option_rows) ), entry_context_timing=( "OTHER_ACTIVE" if _active_other_ids(rows, onset_frame, wearer_id=wearer_id) else "NO_OTHER_ACTIVE_AT_ONSET" ), ) previous_ids = tuple(sorted({row.participant_id for row, _ in release_options})) add(_candidate( session=session, task="wearer_entry", source="WEARER_ONSET", rule_type="WEARER_ENTRY_CANDIDATE", anchor_type="wearer_onset", frame=onset_frame, region_start=earliest_opportunity_frame, region_end=min(segment_end, onset_frame + relation_frames), label="candidate", subtype=None, previous_ids=previous_ids, incoming_ids=_active_other_ids(rows, onset_frame, wearer_id=wearer_id), evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id, context_frames=context_frames, reason=( "raw wearer onset retained without floor-entry or within-turn-resumption gating; " "nearby release relations are descriptive evidence" ), ), ("WEARER_ONSET", onset_frame, previous_ids)) # A2: every raw wearer offset is an unresolved INT candidate. The code # records temporal relationships but does not infer takeover semantics. # In particular, new/pre-existing/post-offset other activity and # boundary crossing are evidence for DeepSeek and the human reviewer, # never deterministic admission gates. current_activity_run_start = segment_start if wearer and wearer[0] else None activity_run_start_by_offset: dict[int, int | None] = {} for index in range(1, len(wearer)): frame = segment_start + index if not wearer[index - 1] and wearer[index]: current_activity_run_start = frame elif wearer[index - 1] and not wearer[index]: activity_run_start_by_offset[frame] = current_activity_run_start current_activity_run_start = None for frame in raw_offsets: index = frame - segment_start activity_run_start = activity_run_start_by_offset[frame] strict_crossing_ids = _other_ids_crossing_offset( other_rows, frame=frame, wearer_id=wearer_id, tolerance_frames=0 ) tolerance_crossing_ids = _other_ids_crossing_offset( other_rows, frame=frame, wearer_id=wearer_id, tolerance_frames=int_offset_tolerance_frames, ) new_other_rows = [ row for row in other_rows if activity_run_start is not None and activity_run_start < row.dense_start_frame < frame ] new_crossing_rows = [ row for row in new_other_rows if row.dense_end_frame_exclusive > frame - int_offset_tolerance_frames ] new_other_ids = tuple(sorted({row.participant_id for row in new_other_rows})) new_crossing_ids = tuple(sorted({row.participant_id for row in new_crossing_rows})) preexisting_rows = [ row for row in other_rows if activity_run_start is not None and row.dense_start_frame <= activity_run_start and row.dense_end_frame_exclusive > activity_run_start ] preexisting_ids = tuple(sorted({row.participant_id for row in preexisting_rows})) nearby_other_rows = [ row for row in other_rows if row.dense_end_frame_exclusive > frame - relation_frames and row.dense_start_frame < frame + relation_frames ] nearby_other_intervals = [{ "uid": row.uid, "participant_id": row.participant_id, "onset_seconds": row.dense_start_seconds, "offset_seconds": row.dense_end_seconds, "onset_relative_to_wearer_offset_seconds": ( row.dense_start_frame - frame ) / SOURCE_FPS, "offset_relative_to_wearer_offset_seconds": ( row.dense_end_frame_exclusive - frame ) / SOURCE_FPS, "starts_during_wearer_activity_run": bool( activity_run_start is not None and activity_run_start < row.dense_start_frame < frame ), "preexisting_at_wearer_activity_run_start": bool( activity_run_start is not None and row.dense_start_frame <= activity_run_start and row.dense_end_frame_exclusive > activity_run_start ), "active_immediately_before_offset": bool( row.dense_start_frame <= frame - 1 < row.dense_end_frame_exclusive ), "active_immediately_after_offset": bool( row.dense_start_frame <= frame < row.dense_end_frame_exclusive ), "strictly_crosses_offset": bool( row.dense_start_frame < frame < row.dense_end_frame_exclusive ), "crosses_offset_with_tolerance": bool( row.dense_start_frame < frame + int_offset_tolerance_frames and row.dense_end_frame_exclusive > frame - int_offset_tolerance_frames ), "overlap_with_wearer_before_offset_seconds": ( 0.0 if activity_run_start is None else max( 0, min(row.dense_end_frame_exclusive, frame) - max(row.dense_start_frame, activity_run_start) ) / SOURCE_FPS ), "transcription": row.transcription, "target_of_speech": list(row.target_of_speech), } for row in nearby_other_rows] next_rows = sorted( [ row for row in rows if frame <= row.dense_start_frame <= frame + relation_frames ], key=lambda row: (row.dense_start_frame, row.participant_id, row.uid), ) first_next_frame = next_rows[0].dense_start_frame if next_rows else None first_next_rows = ( [] if first_next_frame is None else [row for row in next_rows if row.dense_start_frame == first_next_frame] ) active_other_ids_after_offset = _active_other_ids( rows, frame, wearer_id=wearer_id ) next_speaker_ids = ( list(active_other_ids_after_offset) if active_other_ids_after_offset else [row.participant_id for row in first_next_rows] ) next_speaker_roles = ( ["other"] * len(active_other_ids_after_offset) if active_other_ids_after_offset else [ "wearer" if row.participant_id == wearer_id else "other" for row in first_next_rows ] ) evidence = _relation_evidence( rows, frame=frame, wearer_id=wearer_id, window_frames=relation_frames ) evidence.update( wearer_activity_run_start_frame=activity_run_start, wearer_activity_run_start_seconds=( None if activity_run_start is None else activity_run_start / SOURCE_FPS ), wearer_activity_run_duration_to_offset_seconds=( None if activity_run_start is None else (frame - activity_run_start) / SOURCE_FPS ), wearer_offset_frame=frame, wearer_offset_seconds=frame / SOURCE_FPS, wearer_speech_duration_before_offset_seconds=_run_before(wearer, index, True) / SOURCE_FPS, wearer_silence_duration_after_offset_seconds=_run_after(wearer, index, False) / SOURCE_FPS, active_other_ids_before_offset=list(_active_other_ids(rows, frame - 1, wearer_id=wearer_id)), active_other_ids_after_offset=list(active_other_ids_after_offset), strict_crossing_other_ids=list(strict_crossing_ids), tolerance_crossing_other_ids=list(tolerance_crossing_ids), new_other_onset_ids_during_wearer=list(new_other_ids), new_other_onsets_during_wearer=[{ "participant_id": row.participant_id, "onset_seconds": row.dense_start_seconds, "offset_seconds": row.dense_end_seconds, "uid": row.uid, } for row in new_other_rows], new_other_onset_ids_crossing_offset=list(new_crossing_ids), preexisting_other_ids_at_wearer_activity_run_start=list(preexisting_ids), preexisting_other_intervals=[{ "participant_id": row.participant_id, "onset_seconds": row.dense_start_seconds, "offset_seconds": row.dense_end_seconds, "uid": row.uid, } for row in preexisting_rows], nearby_other_intervals=nearby_other_intervals, next_speech_onsets=[{ "participant_id": row.participant_id, "participant_role": "wearer" if row.participant_id == wearer_id else "other", "onset_seconds": row.dense_start_seconds, "offset_seconds": row.dense_end_seconds, "uid": row.uid, "transcription": row.transcription, "target_of_speech": list(row.target_of_speech), } for row in next_rows], next_speaker_ids=next_speaker_ids, next_speaker_roles=next_speaker_roles, next_speaker_basis=( "active_across_offset" if active_other_ids_after_offset else ("first_new_onset_after_offset" if first_next_rows else None) ), next_speech_onset_seconds=( frame / SOURCE_FPS if active_other_ids_after_offset else (None if first_next_frame is None else first_next_frame / SOURCE_FPS) ), other_crosses_wearer_offset=bool(tolerance_crossing_ids), new_other_onset_during_wearer=bool(new_other_rows), new_other_onset_crosses_offset=bool(new_crossing_rows), int_offset_tolerance_seconds=cfg.int_offset_tolerance_seconds, original_timestamp_seconds=frame / SOURCE_FPS, ) add(_candidate( session=session, task="int_yield", source="WEARER_OFFSET", rule_type="YIELD_CANDIDATE", anchor_type="wearer_offset", frame=frame, region_start=max(segment_start, frame - relation_frames), region_end=min(segment_end, frame + relation_frames), label="candidate", subtype=None, previous_ids=_active_other_ids(rows, frame - 1, wearer_id=wearer_id), incoming_ids=new_other_ids, evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id, context_frames=context_frames, reason=( "raw wearer offset retained without timing-based takeover gating; " "all nearby activity relations are descriptive evidence" ), ), ("WEARER_OFFSET", frame, new_other_ids)) # B1: same speaker pauses and resumes while the wearer remains silent. by_speaker: dict[int, list[Utterance]] = defaultdict(list) for row in other_rows: by_speaker[row.participant_id].append(row) for participant_id, speaker_rows in by_speaker.items(): for first, resumed in zip(speaker_rows, speaker_rows[1:]): gap = resumed.dense_start_frame - first.dense_end_frame_exclusive left = first.dense_end_frame_exclusive - segment_start right = resumed.dense_start_frame - segment_start if (gap <= 0 or gap > pause_frames or not _all_state(wearer, left, right, False) or not _all_state(others, left, right, False)): continue frame = first.dense_end_frame_exclusive evidence = { "speaker_id": participant_id, "speaker_offset_seconds": first.dense_end_seconds, "speaker_resume_seconds": resumed.dense_start_seconds, "resume_delay_seconds": gap / SOURCE_FPS, "wearer_active_during_pause": False, "original_timestamp_seconds": frame / SOURCE_FPS, } add(_candidate( session=session, task="wearer_entry", source="SAME_SPEAKER_PAUSE", rule_type="ENTRY_HARD_NEGATIVE_REGION", anchor_type="other_offset", frame=frame, region_start=frame, region_end=resumed.dense_start_frame, label="hard_negative", subtype="ENTRY_OTHER_MIDTURN_PAUSE", previous_ids=(participant_id,), incoming_ids=(participant_id,), evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id, context_frames=context_frames, reason="same participant resumes after a configurable pause while wearer stays silent", ), ("SAME_SPEAKER_PAUSE", frame, (participant_id,))) # B2: participant-resolved non-wearer to non-wearer handover. for first in other_rows: options = [row for row in other_rows if row.participant_id != first.participant_id and row.dense_start_frame >= first.dense_end_frame_exclusive and row.dense_start_frame - first.dense_end_frame_exclusive <= handover_frames] if not options: continue incoming = min(options, key=lambda row: row.dense_start_frame) same_resumes = any(row.participant_id == first.participant_id and first.dense_end_frame_exclusive < row.dense_start_frame <= incoming.dense_start_frame for row in other_rows) left = first.dense_end_frame_exclusive - segment_start right = max(left + 1, incoming.dense_start_frame - segment_start + 1) if same_resumes or not _all_state(wearer, left, right, False): continue frame = first.dense_end_frame_exclusive evidence = { "previous_speaker_id": first.participant_id, "incoming_speaker_id": incoming.participant_id, "previous_offset_seconds": first.dense_end_seconds, "incoming_onset_seconds": incoming.dense_start_seconds, "handover_gap_seconds": (incoming.dense_start_frame - frame) / SOURCE_FPS, "wearer_active_during_handover": False, "original_timestamp_seconds": frame / SOURCE_FPS, } add(_candidate( session=session, task="wearer_entry", source="OTHER_TO_OTHER", rule_type="ENTRY_HARD_NEGATIVE_REGION", anchor_type="other_offset", frame=frame, region_start=frame, region_end=incoming.dense_start_frame, label="hard_negative", subtype="ENTRY_OTHER_TO_OTHER", previous_ids=(first.participant_id,), incoming_ids=(incoming.participant_id,), evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id, context_frames=context_frames, reason="participant-level IDs show a non-wearer to non-wearer handover", ), ("OTHER_TO_OTHER", frame, (first.participant_id, incoming.participant_id))) # B3/B4: other activity ends while the wearer continuously holds and continues the floor. for other in other_rows: start = other.dense_start_frame - segment_start end = other.dense_end_frame_exclusive - segment_start if (start < 0 or end + continuation_frames > len(wearer) or not _all_state(wearer, start, end + continuation_frames, True)): continue feedback = (other.dense_end_frame_exclusive - other.dense_start_frame <= short_frames and (is_lexical_feedback(other.transcription) or is_pure_nonverbal(other.transcription))) source = "OTHER_BACKCHANNEL" if feedback else "NON_TAKEOVER_OVERLAP" subtype = "INT_OTHER_BACKCHANNEL" if feedback else "INT_NON_TAKEOVER_OVERLAP" evidence = { "other_speaker_id": other.participant_id, "other_onset_seconds": other.dense_start_seconds, "other_offset_seconds": other.dense_end_seconds, "other_duration_seconds": other.duration_seconds, "wearer_active_through_region": True, "wearer_continuation_after_other_seconds": continuation_frames / SOURCE_FPS, "lexical_or_nonverbal_feedback_hint": feedback, "original_timestamp_seconds": other.dense_start_seconds, } add(_candidate( session=session, task="int_yield", source=source, rule_type="INT_HARD_NEGATIVE_REGION", anchor_type="other_onset", frame=other.dense_start_frame, region_start=other.dense_start_frame, region_end=other.dense_end_frame_exclusive, label="hard_negative", subtype=subtype, previous_ids=_active_other_ids(rows, other.dense_start_frame - 1, wearer_id=wearer_id), incoming_ids=(other.participant_id,), evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id, context_frames=context_frames, reason=("short feedback-like other activity ends while wearer holds the floor" if feedback else "other overlaps but wearer retains and continues the floor"), ), (source, other.dense_start_frame, (other.participant_id,))) return sorted(candidates, key=lambda row: (row.anchor_dense_frame, row.candidate_source, row.uid)) def stratified_limit(rows: Sequence[ProtocolCandidate], limit: int | None, *, seed: int) -> list[ProtocolCandidate]: if limit is None or limit >= len(rows): return list(rows) if limit < 0: raise ValueError("--max-candidates must be non-negative") rng = random.Random(seed) buckets: dict[tuple[str, str], list[ProtocolCandidate]] = defaultdict(list) for row in rows: buckets[(row.session, row.candidate_source)].append(row) for bucket in buckets.values(): rng.shuffle(bucket) selected: list[ProtocolCandidate] = [] while len(selected) < limit and any(buckets.values()): for key in sorted(buckets): if buckets[key] and len(selected) < limit: selected.append(buckets[key].pop()) return sorted(selected, key=lambda row: (row.session, row.anchor_dense_frame, row.candidate_source)) def _write_jsonl(path: Path, rows: Iterable[ProtocolCandidate]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8") as handle: for row in rows: handle.write(json.dumps(asdict(row), ensure_ascii=False) + "\n") def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--dataset-root", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--sessions", nargs="+", default=list(ALL_SESSIONS)) parser.add_argument("--camera-wearer-id", type=int, default=CAMERA_WEARER_ID) parser.add_argument("--context-seconds", type=float, default=3.0) parser.add_argument("--relation-window-seconds", type=float, default=3.0) parser.add_argument("--same-speaker-pause-max-seconds", type=float, default=2.0) parser.add_argument("--other-handover-max-seconds", type=float, default=1.5) parser.add_argument("--short-vocalization-max-seconds", type=float, default=1.0) parser.add_argument("--continuation-min-seconds", type=float, default=0.4) parser.add_argument("--int-offset-tolerance-seconds", type=float, default=0.05) parser.add_argument( "--wearer-resume-merge-max-seconds", type=float, help=argparse.SUPPRESS, ) # Deprecated compatibility option; raw wearer boundaries are never merged. parser.add_argument("--candidate-sources", nargs="+") parser.add_argument("--max-candidates", type=int) parser.add_argument("--selection-seed", type=int, default=2027) return parser.parse_args() def main() -> None: args = parse_args() unknown = sorted(set(args.sessions) - set(ALL_SESSIONS)) if unknown: raise ValueError(f"Unknown EasyCom sessions: {unknown}") config = CandidateConfig( context_seconds=args.context_seconds, relation_window_seconds=args.relation_window_seconds, same_speaker_pause_max_seconds=args.same_speaker_pause_max_seconds, other_handover_max_seconds=args.other_handover_max_seconds, short_vocalization_max_seconds=args.short_vocalization_max_seconds, continuation_min_seconds=args.continuation_min_seconds, int_offset_tolerance_seconds=args.int_offset_tolerance_seconds, ) dataset_root, output_dir = args.dataset_root.expanduser().resolve(), args.output_dir.expanduser().resolve() all_rows: list[ProtocolCandidate] = [] session_timelines: dict[str, list[dict[str, Any]]] = {} rules = Rules() for session in args.sessions: clips = build_clips(dataset_root, session, rules) utterances = load_utterances(session, clips) session_timelines[session] = [{ "uid": row.uid, "participant_id": row.participant_id, "participant_role": "wearer" if row.participant_id == args.camera_wearer_id else "other", "start_seconds": row.dense_start_seconds, "end_seconds": row.dense_end_seconds, } for row in utterances] rows = derive_protocol_candidates(utterances, clips, session=session, wearer_id=args.camera_wearer_id, config=config) all_rows.extend(rows) print(f"{session}: candidates={len(rows)} sources={dict(sorted(Counter(row.candidate_source for row in rows).items()))}") sources = None if args.candidate_sources is None else set(args.candidate_sources) controls = [row for row in all_rows if not requires_semantic_adjudication(asdict(row))] primary = [row for row in all_rows if requires_semantic_adjudication(asdict(row))] eligible = [row for row in primary if sources is None or row.candidate_source in sources] selected = stratified_limit(eligible, args.max_candidates, seed=args.selection_seed) output_path = output_dir / "easycom_eot_int_candidates.jsonl" _write_jsonl(output_path, selected) controls_path = output_dir / "easycom_eot_int_controls.jsonl" _write_jsonl(controls_path, controls) counts = Counter((row.split, row.candidate_source, row.rule_candidate_type) for row in selected) summary = { "schema_version": SCHEMA_VERSION, "protocol_version": PROTOCOL_VERSION, "dataset": "EasyCom", "semantic_warning": "Rules create candidates only; AI labels need human review and never modify dense activity ground truth.", "sessions": list(args.sessions), "camera_wearer_id": args.camera_wearer_id, "candidate_config": asdict(config), "selection_seed": args.selection_seed, "max_candidates": args.max_candidates, "candidate_sources": None if sources is None else sorted(sources), "num_candidates_before_filter": len(primary), "num_candidates_before_limit": len(eligible), "num_candidates_written": len(selected), "num_controls_written": len(controls), "controls_file": controls_path.name, "counts": [{"split": key[0], "candidate_source": key[1], "rule_candidate_type": key[2], "count": value} for key, value in sorted(counts.items())], } output_dir.mkdir(parents=True, exist_ok=True) (output_dir / "summary.json").write_text( json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") (output_dir / "easycom_session_timelines.json").write_text( json.dumps({"schema_version": 1, "sessions": session_timelines}, ensure_ascii=False) + "\n", encoding="utf-8", ) print(f"Saved {len(selected)} wearer-centric review candidates to {output_path}") print(f"Saved {len(controls)} deterministic boundary controls to {controls_path}") if __name__ == "__main__": main()