843 lines
44 KiB
Python
843 lines
44 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate wearer-centric EasyCom entry/INT candidates for AI and human review.
|
|
|
|
This stage never changes EasyCom speech boundaries and never claims semantic
|
|
ground truth. It groups short wearer resumptions into floor-level episodes,
|
|
starts entry candidates from every observed floor-level wearer onset and
|
|
retains preceding non-wearer release options as context, requires a new
|
|
non-wearer onset inside the wearer floor episode for INT candidates, and keeps
|
|
ineligible raw boundaries as deterministic controls. All timing
|
|
thresholds are CLI parameters, and raw evidence is retained for DeepSeek
|
|
pre-labeling and human correction.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import Counter, defaultdict
|
|
from dataclasses import asdict, dataclass
|
|
import json
|
|
from pathlib import Path
|
|
import random
|
|
from typing import Any, Iterable, Sequence
|
|
|
|
from derive_easycom_interaction_events import (
|
|
CAMERA_WEARER_ID, SOURCE_FPS, Clip, Rules, Utterance, build_clips,
|
|
is_lexical_feedback, is_pure_nonverbal, load_utterances,
|
|
)
|
|
from easycom_manifest import ALL_SESSIONS, TEST_SESSIONS
|
|
|
|
|
|
SCHEMA_VERSION = 5
|
|
PROTOCOL_VERSION = "easycom-wearer-entry-int-candidates-v7"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CandidateConfig:
|
|
context_seconds: float = 3.0
|
|
relation_window_seconds: float = 3.0
|
|
same_speaker_pause_max_seconds: float = 2.0
|
|
other_handover_max_seconds: float = 1.5
|
|
short_vocalization_max_seconds: float = 1.0
|
|
continuation_min_seconds: float = 0.4
|
|
int_offset_tolerance_seconds: float = 0.05
|
|
wearer_resume_merge_max_seconds: float = 0.8
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ProtocolCandidate:
|
|
schema_version: int
|
|
protocol_version: str
|
|
uid: str
|
|
event_id: str
|
|
session: str
|
|
split: str
|
|
task: str
|
|
event_type: str
|
|
candidate_source: str
|
|
rule_candidate_type: str
|
|
automatic_protocol_label: str
|
|
automatic_hard_negative_type: str | None
|
|
automatic_confidence: str
|
|
anchor_type: str
|
|
anchor_dense_frame: int
|
|
anchor_dense_seconds: float
|
|
event_time: float
|
|
region_start_seconds: float
|
|
region_end_seconds: float
|
|
clip_stem: str
|
|
source_start_frame: int
|
|
previous_other_participant_ids: tuple[int, ...]
|
|
incoming_other_participant_ids: tuple[int, ...]
|
|
temporal_evidence: dict[str, Any]
|
|
speaker_timeline: tuple[dict[str, Any], ...]
|
|
context: tuple[dict[str, Any], ...]
|
|
automatic_reason: str
|
|
ai_verdict: None = None
|
|
ai_event_type: None = None
|
|
ai_subtype: None = None
|
|
ai_confidence: None = None
|
|
ai_reason: None = None
|
|
human_status: str = "UNREVIEWED"
|
|
human_verdict: None = None
|
|
human_event_type: None = None
|
|
human_subtype: None = None
|
|
exclude_reason: None = None
|
|
|
|
|
|
def requires_semantic_adjudication(row: dict[str, Any]) -> bool:
|
|
"""Return whether a stored row belongs in the primary AI/human queue.
|
|
|
|
Explicit legacy control sources are authoritative. For old v2 candidate
|
|
files, reconstruct the same-speaker crossing requirement from the stored
|
|
activity IDs. Missing legacy evidence remains reviewable rather than being
|
|
silently discarded.
|
|
"""
|
|
source = row.get("candidate_source")
|
|
if source in {"WEARER_ONSET_CONTROL", "WEARER_OFFSET_CONTROL", "WEARER_RESUME"}:
|
|
return False
|
|
if source != "WEARER_OFFSET":
|
|
return True
|
|
evidence = row.get("temporal_evidence")
|
|
if not isinstance(evidence, dict):
|
|
return True
|
|
explicit = evidence.get("other_crosses_wearer_offset")
|
|
if isinstance(explicit, bool):
|
|
return explicit
|
|
before = evidence.get("active_other_ids_before_offset")
|
|
after = evidence.get("active_other_ids_after_offset")
|
|
if isinstance(before, list) and isinstance(after, list):
|
|
return bool(set(before) & set(after))
|
|
return True
|
|
|
|
|
|
def _fill_states(rows: Sequence[Utterance], *, segment_start: int, segment_end: int,
|
|
wearer_id: int) -> tuple[bytearray, bytearray]:
|
|
length = segment_end - segment_start
|
|
wearer, others = bytearray(length), bytearray(length)
|
|
for row in rows:
|
|
left = max(segment_start, row.dense_start_frame) - segment_start
|
|
right = min(segment_end, row.dense_end_frame_exclusive) - segment_start
|
|
if right <= left:
|
|
continue
|
|
target = wearer if row.participant_id == wearer_id else others
|
|
target[left:right] = b"\x01" * (right - left)
|
|
return wearer, others
|
|
|
|
|
|
def _run_after(state: bytearray, index: int, value: bool) -> int:
|
|
end = index
|
|
while end < len(state) and bool(state[end]) == value:
|
|
end += 1
|
|
return end - index
|
|
|
|
|
|
def _run_before(state: bytearray, index: int, value: bool) -> int:
|
|
start = index - 1
|
|
while start >= 0 and bool(state[start]) == value:
|
|
start -= 1
|
|
return index - start - 1
|
|
|
|
|
|
def _all_state(state: bytearray, start: int, stop: int, value: bool) -> bool:
|
|
left, right = max(0, start), min(len(state), stop)
|
|
return right > left and all(bool(state[i]) == value for i in range(left, right))
|
|
|
|
|
|
def _active_other_ids(rows: Sequence[Utterance], frame: int, *, wearer_id: int) -> tuple[int, ...]:
|
|
return tuple(sorted({
|
|
row.participant_id for row in rows
|
|
if row.participant_id != wearer_id
|
|
and row.dense_start_frame <= frame < row.dense_end_frame_exclusive
|
|
}))
|
|
|
|
|
|
def _other_ids_crossing_offset(
|
|
rows: Sequence[Utterance], *, frame: int, wearer_id: int, tolerance_frames: int
|
|
) -> tuple[int, ...]:
|
|
"""Return other speakers whose annotated interval crosses a wearer offset.
|
|
|
|
With zero tolerance this encodes the strict prerequisite
|
|
``other.start < wearer_offset < other.end``. A one-frame tolerance accepts
|
|
an annotation boundary at the same 20 Hz frame, but not an other onset one
|
|
or more frames after the wearer offset.
|
|
"""
|
|
return tuple(sorted({
|
|
row.participant_id for row in rows
|
|
if row.participant_id != wearer_id
|
|
and row.dense_start_frame < frame + tolerance_frames
|
|
and row.dense_end_frame_exclusive > frame - tolerance_frames
|
|
}))
|
|
|
|
|
|
def _wearer_gap_competition(
|
|
rows: Sequence[Utterance], *, gap_start_frame: int, onset_frame: int,
|
|
wearer_id: int, short_frames: int, continuation_frames: int,
|
|
) -> dict[str, Any]:
|
|
"""Summarize plausible competing floor activity inside a short wearer gap.
|
|
|
|
Feedback-like short lexical/nonverbal activity is retained as evidence but
|
|
does not by itself split the wearer floor episode. Any other contribution
|
|
that occupies the gap for ``continuation_frames`` or continues across the
|
|
resumed wearer onset conservatively preserves a new ENTRY candidate.
|
|
"""
|
|
gap_rows = [
|
|
row for row in rows
|
|
if row.participant_id != wearer_id
|
|
and row.dense_start_frame < onset_frame
|
|
and row.dense_end_frame_exclusive > gap_start_frame
|
|
]
|
|
feedback_ids: set[int] = set()
|
|
competing_ids: set[int] = set()
|
|
max_gap_coverage = 0
|
|
for row in gap_rows:
|
|
coverage = max(
|
|
0,
|
|
min(row.dense_end_frame_exclusive, onset_frame)
|
|
- max(row.dense_start_frame, gap_start_frame),
|
|
)
|
|
max_gap_coverage = max(max_gap_coverage, coverage)
|
|
feedback_like = (
|
|
row.dense_end_frame_exclusive - row.dense_start_frame <= short_frames
|
|
and (is_lexical_feedback(row.transcription) or is_pure_nonverbal(row.transcription))
|
|
)
|
|
if feedback_like:
|
|
feedback_ids.add(row.participant_id)
|
|
elif coverage >= continuation_frames or row.dense_end_frame_exclusive > onset_frame:
|
|
competing_ids.add(row.participant_id)
|
|
return {
|
|
"gap_other_participant_ids": sorted({row.participant_id for row in gap_rows}),
|
|
"feedback_like_other_ids_during_gap": sorted(feedback_ids),
|
|
"sustained_competing_other_ids_during_gap": sorted(competing_ids),
|
|
"max_other_gap_coverage_seconds": max_gap_coverage / SOURCE_FPS,
|
|
"sustained_other_takeover_during_gap": bool(competing_ids),
|
|
}
|
|
|
|
|
|
def _clip_for_frame(clips: Sequence[Clip], frame: int) -> Clip | None:
|
|
return next((clip for clip in clips
|
|
if clip.dense_start_frame <= frame < clip.dense_start_frame + clip.num_frames), None)
|
|
|
|
|
|
def _context_rows(rows: Sequence[Utterance], *, frame: int, context_frames: int,
|
|
wearer_id: int) -> tuple[dict[str, Any], ...]:
|
|
left, right = frame - context_frames, frame + context_frames
|
|
return tuple({
|
|
"uid": row.uid,
|
|
"participant_id": row.participant_id,
|
|
"participant_role": "wearer" if row.participant_id == wearer_id else "other",
|
|
"start_seconds": row.dense_start_seconds,
|
|
"end_seconds": row.dense_end_seconds,
|
|
"duration_seconds": row.duration_seconds,
|
|
"transcription": row.transcription,
|
|
"target_of_speech": list(row.target_of_speech),
|
|
} for row in rows
|
|
if row.dense_end_frame_exclusive > left and row.dense_start_frame < right)
|
|
|
|
|
|
def _relation_evidence(rows: Sequence[Utterance], *, frame: int, wearer_id: int,
|
|
window_frames: int) -> dict[str, Any]:
|
|
previous = [row for row in rows if row.participant_id != wearer_id
|
|
and row.dense_end_frame_exclusive <= frame
|
|
and frame - row.dense_end_frame_exclusive <= window_frames]
|
|
following = [row for row in rows if row.participant_id != wearer_id
|
|
and row.dense_start_frame >= frame
|
|
and row.dense_start_frame - frame <= window_frames]
|
|
prev = max(previous, key=lambda row: row.dense_end_frame_exclusive, default=None)
|
|
nxt = min(following, key=lambda row: row.dense_start_frame, default=None)
|
|
active_rows = [row for row in rows if row.participant_id != wearer_id
|
|
and row.dense_start_frame < frame < row.dense_end_frame_exclusive]
|
|
overlapping = sorted({row.participant_id for row in active_rows})
|
|
resumes = [] if prev is None else [row for row in following if row.participant_id == prev.participant_id]
|
|
resumed = min(resumes, key=lambda row: row.dense_start_frame, default=None)
|
|
nearby_onsets = [row for row in rows if row.participant_id != wearer_id
|
|
and abs(row.dense_start_frame - frame) <= window_frames]
|
|
closest_onset = min(nearby_onsets, key=lambda row: abs(row.dense_start_frame - frame), default=None)
|
|
return {
|
|
"previous_speaker_id": None if prev is None else prev.participant_id,
|
|
"previous_speaker_offset_seconds": None if prev is None else prev.dense_end_seconds,
|
|
"gap_from_previous_speaker_offset_seconds": (
|
|
None if prev is None else (frame - prev.dense_end_frame_exclusive) / SOURCE_FPS),
|
|
"overlapping_other_ids": overlapping,
|
|
"other_overlap_at_event_seconds": max(
|
|
[0.0] + [(row.dense_end_frame_exclusive - frame) / SOURCE_FPS for row in active_rows]
|
|
),
|
|
"next_speaker_id": None if nxt is None else nxt.participant_id,
|
|
"next_other_onset_seconds": None if nxt is None else nxt.dense_start_seconds,
|
|
"previous_speaker_resumes": resumed is not None,
|
|
"previous_speaker_resume_delay_seconds": (
|
|
None if resumed is None or prev is None
|
|
else (resumed.dense_start_frame - prev.dense_end_frame_exclusive) / SOURCE_FPS
|
|
),
|
|
"closest_other_onset_relative_seconds": (
|
|
None if closest_onset is None else (closest_onset.dense_start_frame - frame) / SOURCE_FPS
|
|
),
|
|
}
|
|
|
|
|
|
def _candidate(*, session: str, task: str, source: str, rule_type: str,
|
|
anchor_type: str, frame: int, region_start: int, region_end: int,
|
|
label: str, subtype: str | None, previous_ids: tuple[int, ...],
|
|
incoming_ids: tuple[int, ...], evidence: dict[str, Any], clips: Sequence[Clip],
|
|
rows: Sequence[Utterance], wearer_id: int, context_frames: int,
|
|
reason: str) -> ProtocolCandidate | None:
|
|
clip = _clip_for_frame(clips, frame)
|
|
if clip is None:
|
|
return None
|
|
participant_key = "prev-" + ("-".join(f"p{x}" for x in previous_ids) or "none")
|
|
participant_key += "__in-" + ("-".join(f"p{x}" for x in incoming_ids) or "none")
|
|
uid = f"{session}/{source.lower()}/{frame:08d}/{participant_key}"
|
|
context = _context_rows(rows, frame=frame, context_frames=context_frames, wearer_id=wearer_id)
|
|
return ProtocolCandidate(
|
|
schema_version=SCHEMA_VERSION, protocol_version=PROTOCOL_VERSION,
|
|
uid=uid, event_id=uid, session=session,
|
|
split="test" if session in TEST_SESSIONS else "train", task=task,
|
|
event_type=source.lower(), candidate_source=source, rule_candidate_type=rule_type,
|
|
automatic_protocol_label=label, automatic_hard_negative_type=subtype,
|
|
automatic_confidence="rule_candidate", anchor_type=anchor_type,
|
|
anchor_dense_frame=frame, anchor_dense_seconds=frame / SOURCE_FPS,
|
|
event_time=frame / SOURCE_FPS, region_start_seconds=region_start / SOURCE_FPS,
|
|
region_end_seconds=region_end / SOURCE_FPS, clip_stem=clip.stem,
|
|
source_start_frame=frame - clip.dense_start_frame + 1,
|
|
previous_other_participant_ids=previous_ids,
|
|
incoming_other_participant_ids=incoming_ids, temporal_evidence=evidence,
|
|
speaker_timeline=context, context=context, automatic_reason=reason,
|
|
)
|
|
|
|
|
|
def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[Clip], *,
|
|
session: str, wearer_id: int = CAMERA_WEARER_ID,
|
|
config: CandidateConfig | None = None,
|
|
window_seconds: float | None = None,
|
|
context_seconds: float | None = None) -> list[ProtocolCandidate]:
|
|
"""Return primary semantic candidates plus separate offset controls."""
|
|
cfg = config or CandidateConfig()
|
|
values = asdict(cfg)
|
|
if window_seconds is not None:
|
|
values["relation_window_seconds"] = window_seconds
|
|
if context_seconds is not None:
|
|
values["context_seconds"] = context_seconds
|
|
cfg = CandidateConfig(**values)
|
|
if any(value < 0 for value in asdict(cfg).values()) or cfg.relation_window_seconds == 0:
|
|
raise ValueError("candidate timing parameters must be non-negative and relation window positive")
|
|
context_frames = round(cfg.context_seconds * SOURCE_FPS)
|
|
relation_frames = round(cfg.relation_window_seconds * SOURCE_FPS)
|
|
pause_frames = round(cfg.same_speaker_pause_max_seconds * SOURCE_FPS)
|
|
handover_frames = round(cfg.other_handover_max_seconds * SOURCE_FPS)
|
|
short_frames = round(cfg.short_vocalization_max_seconds * SOURCE_FPS)
|
|
continuation_frames = round(cfg.continuation_min_seconds * SOURCE_FPS)
|
|
int_offset_tolerance_frames = round(cfg.int_offset_tolerance_seconds * SOURCE_FPS)
|
|
wearer_resume_merge_frames = round(cfg.wearer_resume_merge_max_seconds * SOURCE_FPS)
|
|
rows_by_segment: dict[int, list[Utterance]] = defaultdict(list)
|
|
clips_by_segment: dict[int, list[Clip]] = defaultdict(list)
|
|
for row in utterances:
|
|
rows_by_segment[row.segment_id].append(row)
|
|
for clip in clips:
|
|
clips_by_segment[clip.segment_id].append(clip)
|
|
candidates: list[ProtocolCandidate] = []
|
|
seen: set[tuple[str, int, tuple[int, ...]]] = set()
|
|
|
|
def add(item: ProtocolCandidate | None, key: tuple[str, int, tuple[int, ...]]) -> None:
|
|
if item is not None and key not in seen:
|
|
seen.add(key)
|
|
candidates.append(item)
|
|
|
|
for segment_id, segment_clips in sorted(clips_by_segment.items()):
|
|
segment_clips.sort(key=lambda clip: clip.dense_start_frame)
|
|
rows = sorted(rows_by_segment.get(segment_id, []),
|
|
key=lambda row: (row.dense_start_frame, row.dense_end_frame_exclusive, row.uid))
|
|
segment_start = segment_clips[0].dense_start_frame
|
|
segment_end = segment_clips[-1].dense_start_frame + segment_clips[-1].num_frames
|
|
wearer, others = _fill_states(rows, segment_start=segment_start,
|
|
segment_end=segment_end, wearer_id=wearer_id)
|
|
other_rows = [row for row in rows if row.participant_id != wearer_id]
|
|
|
|
# A: classify raw wearer onsets before proposing any behavioral event.
|
|
# Short uncompeted gaps are internal to a floor episode. All other raw
|
|
# onsets begin a new floor episode and are all entry candidates. A
|
|
# preceding non-wearer release is optional context, not a gate.
|
|
onset_records: dict[int, dict[str, Any]] = {}
|
|
raw_offsets: list[int] = []
|
|
for index in range(1, len(wearer)):
|
|
frame = segment_start + index
|
|
if not wearer[index - 1] and wearer[index]:
|
|
gap_frames = _run_before(wearer, index, False)
|
|
previous_wearer_index = index - gap_frames - 1
|
|
has_previous_wearer = previous_wearer_index >= 0 and bool(wearer[previous_wearer_index])
|
|
previous_wearer_offset = segment_start + index - gap_frames
|
|
competition = _wearer_gap_competition(
|
|
rows,
|
|
gap_start_frame=previous_wearer_offset,
|
|
onset_frame=frame,
|
|
wearer_id=wearer_id,
|
|
short_frames=short_frames,
|
|
continuation_frames=continuation_frames,
|
|
) if has_previous_wearer else {
|
|
"gap_other_participant_ids": [],
|
|
"feedback_like_other_ids_during_gap": [],
|
|
"sustained_competing_other_ids_during_gap": [],
|
|
"max_other_gap_coverage_seconds": 0.0,
|
|
"sustained_other_takeover_during_gap": False,
|
|
}
|
|
within_turn_resume = (
|
|
has_previous_wearer
|
|
and gap_frames <= wearer_resume_merge_frames
|
|
and not competition["sustained_other_takeover_during_gap"]
|
|
)
|
|
evidence = _relation_evidence(rows, frame=frame, wearer_id=wearer_id,
|
|
window_frames=relation_frames)
|
|
evidence.update(
|
|
wearer_speech_duration_after_onset_seconds=_run_after(wearer, index, True) / SOURCE_FPS,
|
|
previous_wearer_silence_duration_seconds=gap_frames / SOURCE_FPS,
|
|
previous_wearer_offset_seconds=(
|
|
previous_wearer_offset / SOURCE_FPS if has_previous_wearer else None
|
|
),
|
|
wearer_resume_merge_max_seconds=cfg.wearer_resume_merge_max_seconds,
|
|
floor_episode_merged=within_turn_resume,
|
|
original_timestamp_seconds=frame / SOURCE_FPS,
|
|
**competition,
|
|
)
|
|
onset_records[frame] = {
|
|
"within_turn_resume": within_turn_resume,
|
|
"evidence": evidence,
|
|
}
|
|
elif wearer[index - 1] and not wearer[index]:
|
|
raw_offsets.append(frame)
|
|
|
|
resume_onsets = {
|
|
frame for frame, record in onset_records.items() if record["within_turn_resume"]
|
|
}
|
|
floor_onsets = sorted(set(onset_records) - resume_onsets)
|
|
|
|
for frame in sorted(resume_onsets):
|
|
record = onset_records[frame]
|
|
add(_candidate(
|
|
session=session, task="wearer_entry_control", source="WEARER_RESUME",
|
|
rule_type="WITHIN_TURN_RESUME_CONTROL", anchor_type="wearer_onset", frame=frame,
|
|
region_start=frame, region_end=min(segment_end, frame + relation_frames),
|
|
label="control", subtype=None,
|
|
previous_ids=_active_other_ids(rows, frame - 1, wearer_id=wearer_id),
|
|
incoming_ids=_active_other_ids(rows, frame, wearer_id=wearer_id),
|
|
evidence=record["evidence"], clips=segment_clips, rows=rows,
|
|
wearer_id=wearer_id, context_frames=context_frames,
|
|
reason="short raw wearer pause without sustained competing other takeover; merged into the existing floor episode",
|
|
), ("WEARER_RESUME", frame, ()))
|
|
|
|
# A1: every observed floor-level wearer onset is an entry candidate.
|
|
# Search backward for release anchors only to preserve response context;
|
|
# no preceding release is required for a positive WEARER_ENTRY.
|
|
for onset_frame in floor_onsets:
|
|
release_options: list[tuple[Utterance, int]] = []
|
|
for release in other_rows:
|
|
opportunity_frame = release.dense_end_frame_exclusive
|
|
if opportunity_frame > onset_frame:
|
|
continue
|
|
wearer_silent = (
|
|
opportunity_frame == onset_frame
|
|
or _all_state(
|
|
wearer,
|
|
opportunity_frame - segment_start,
|
|
onset_frame - segment_start,
|
|
False,
|
|
)
|
|
)
|
|
competing_onset = any(
|
|
opportunity_frame <= row.dense_start_frame < onset_frame
|
|
for row in other_rows
|
|
)
|
|
other_already_holds = bool(
|
|
_active_other_ids(rows, opportunity_frame, wearer_id=wearer_id)
|
|
)
|
|
if wearer_silent and not competing_onset and not other_already_holds:
|
|
release_options.append((release, opportunity_frame))
|
|
|
|
release_options.sort(
|
|
key=lambda item: (item[1], item[0].participant_id, item[0].uid)
|
|
)
|
|
if release_options:
|
|
_, latest_opportunity_frame = release_options[-1]
|
|
earliest_opportunity_frame = release_options[0][1]
|
|
option_rows = [{
|
|
"uid": release.uid,
|
|
"anchor_type": "other_offset",
|
|
"anchor_frame": frame,
|
|
"anchor_seconds": frame / SOURCE_FPS,
|
|
"participant_id": release.participant_id,
|
|
"transcription": release.transcription,
|
|
"target_of_speech": list(release.target_of_speech),
|
|
"response_delay_seconds": (onset_frame - frame) / SOURCE_FPS,
|
|
} for release, frame in release_options]
|
|
evidence = dict(onset_records[onset_frame]["evidence"])
|
|
evidence.update(
|
|
opportunity_options=option_rows,
|
|
opportunity_anchor_type=None,
|
|
opportunity_anchor_frame=None,
|
|
opportunity_anchor_seconds=None,
|
|
behavior_anchor_type="wearer_floor_onset",
|
|
behavior_anchor_frame=onset_frame,
|
|
behavior_anchor_seconds=onset_frame / SOURCE_FPS,
|
|
minimum_response_delay_seconds=(onset_frame - latest_opportunity_frame) / SOURCE_FPS,
|
|
maximum_response_delay_seconds=(onset_frame - earliest_opportunity_frame) / SOURCE_FPS,
|
|
competing_other_onsets_before_behavior=[],
|
|
preceding_release_paired=True,
|
|
entry_context_timing=(
|
|
"OTHER_ACTIVE"
|
|
if _active_other_ids(rows, onset_frame, wearer_id=wearer_id)
|
|
else "AFTER_OTHER_RELEASE"
|
|
),
|
|
)
|
|
previous_ids = tuple(sorted({row.participant_id for row, _ in release_options}))
|
|
add(_candidate(
|
|
session=session, task="wearer_entry", source="WEARER_ONSET",
|
|
rule_type="WEARER_ENTRY_CANDIDATE", anchor_type="wearer_onset",
|
|
frame=onset_frame, region_start=earliest_opportunity_frame,
|
|
region_end=min(segment_end, onset_frame + relation_frames),
|
|
label="candidate", subtype=None, previous_ids=previous_ids,
|
|
incoming_ids=_active_other_ids(rows, onset_frame, wearer_id=wearer_id),
|
|
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
|
|
context_frames=context_frames,
|
|
reason="floor-level wearer onset; preceding release options retained as context",
|
|
), ("WEARER_ONSET", onset_frame, previous_ids))
|
|
continue
|
|
|
|
frame = onset_frame
|
|
evidence = dict(onset_records[frame]["evidence"])
|
|
evidence.update(
|
|
opportunity_options=[],
|
|
opportunity_anchor_type=None,
|
|
opportunity_anchor_frame=None,
|
|
opportunity_anchor_seconds=None,
|
|
behavior_anchor_type="wearer_floor_onset",
|
|
behavior_anchor_frame=frame,
|
|
behavior_anchor_seconds=frame / SOURCE_FPS,
|
|
preceding_release_paired=False,
|
|
entry_context_timing=(
|
|
"OTHER_ACTIVE"
|
|
if _active_other_ids(rows, frame, wearer_id=wearer_id)
|
|
else "FROM_SILENCE_OR_UNRESOLVED"
|
|
),
|
|
)
|
|
previous_ids = _active_other_ids(rows, frame - 1, wearer_id=wearer_id)
|
|
add(_candidate(
|
|
session=session, task="wearer_entry", source="WEARER_ONSET",
|
|
rule_type="WEARER_ENTRY_CANDIDATE", anchor_type="wearer_onset", frame=frame,
|
|
region_start=max(segment_start, frame - relation_frames),
|
|
region_end=min(segment_end, frame + relation_frames),
|
|
label="candidate", subtype=None,
|
|
previous_ids=previous_ids,
|
|
incoming_ids=_active_other_ids(rows, frame, wearer_id=wearer_id),
|
|
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
|
|
context_frames=context_frames,
|
|
reason="floor-level wearer onset without a surviving preceding release option",
|
|
), ("WEARER_ONSET", frame, previous_ids))
|
|
|
|
# A2: a raw wearer offset is an INT candidate only at the end of a
|
|
# floor-level wearer episode and only if a new non-wearer onset obeys
|
|
# t_w < t_o < t_y and that same interval crosses the yield boundary.
|
|
# All other offsets remain auditable controls.
|
|
ordered_raw_onsets = sorted(onset_records)
|
|
current_floor_start = segment_start if wearer and wearer[0] else None
|
|
floor_start_by_offset: dict[int, tuple[int | None, bool]] = {}
|
|
for index in range(1, len(wearer)):
|
|
frame = segment_start + index
|
|
if not wearer[index - 1] and wearer[index] and frame not in resume_onsets:
|
|
current_floor_start = frame
|
|
elif wearer[index - 1] and not wearer[index]:
|
|
next_onset = next((value for value in ordered_raw_onsets if value > frame), None)
|
|
internal_pause = next_onset in resume_onsets
|
|
floor_start_by_offset[frame] = (current_floor_start, internal_pause)
|
|
if not internal_pause:
|
|
current_floor_start = None
|
|
|
|
for frame in raw_offsets:
|
|
index = frame - segment_start
|
|
floor_start, internal_pause = floor_start_by_offset[frame]
|
|
strict_crossing_ids = _other_ids_crossing_offset(
|
|
other_rows, frame=frame, wearer_id=wearer_id, tolerance_frames=0
|
|
)
|
|
tolerance_crossing_ids = _other_ids_crossing_offset(
|
|
other_rows, frame=frame, wearer_id=wearer_id,
|
|
tolerance_frames=int_offset_tolerance_frames,
|
|
)
|
|
eligible_rows = [
|
|
row for row in other_rows
|
|
if floor_start is not None
|
|
and floor_start < row.dense_start_frame < frame
|
|
and row.dense_end_frame_exclusive > frame - int_offset_tolerance_frames
|
|
]
|
|
eligible_ids = tuple(sorted({row.participant_id for row in eligible_rows}))
|
|
preexisting_ids = tuple(sorted({
|
|
row.participant_id for row in other_rows
|
|
if floor_start is not None
|
|
and row.dense_start_frame <= floor_start
|
|
and row.dense_end_frame_exclusive > frame - int_offset_tolerance_frames
|
|
}))
|
|
is_int_candidate = bool(eligible_ids) and not internal_pause
|
|
evidence = _relation_evidence(
|
|
rows, frame=frame, wearer_id=wearer_id, window_frames=relation_frames
|
|
)
|
|
evidence.update(
|
|
wearer_floor_episode_start_frame=floor_start,
|
|
wearer_floor_episode_start_seconds=(
|
|
None if floor_start is None else floor_start / SOURCE_FPS
|
|
),
|
|
wearer_speech_duration_before_offset_seconds=_run_before(wearer, index, True) / SOURCE_FPS,
|
|
wearer_silence_duration_after_offset_seconds=_run_after(wearer, index, False) / SOURCE_FPS,
|
|
raw_offset_is_within_merged_floor_episode=internal_pause,
|
|
active_other_ids_before_offset=list(_active_other_ids(rows, frame - 1, wearer_id=wearer_id)),
|
|
active_other_ids_after_offset=list(_active_other_ids(rows, frame, wearer_id=wearer_id)),
|
|
strict_crossing_other_ids=list(strict_crossing_ids),
|
|
tolerance_crossing_other_ids=list(tolerance_crossing_ids),
|
|
eligible_new_other_onset_ids=list(eligible_ids),
|
|
eligible_new_other_onsets=[{
|
|
"participant_id": row.participant_id,
|
|
"onset_seconds": row.dense_start_seconds,
|
|
"offset_seconds": row.dense_end_seconds,
|
|
"uid": row.uid,
|
|
} for row in eligible_rows],
|
|
preexisting_crossing_other_ids=list(preexisting_ids),
|
|
other_crosses_wearer_offset=bool(tolerance_crossing_ids),
|
|
int_ordering_satisfied=bool(eligible_ids),
|
|
int_offset_tolerance_seconds=cfg.int_offset_tolerance_seconds,
|
|
original_timestamp_seconds=frame / SOURCE_FPS,
|
|
)
|
|
control_reason = (
|
|
"raw wearer offset is internal to a merged floor episode"
|
|
if internal_pause
|
|
else "no new non-wearer onset satisfies t_w < t_o < t_y and crosses the wearer offset"
|
|
)
|
|
add(_candidate(
|
|
session=session,
|
|
task="int_yield" if is_int_candidate else "int_yield_control",
|
|
source="WEARER_OFFSET" if is_int_candidate else "WEARER_OFFSET_CONTROL",
|
|
rule_type="YIELD_CANDIDATE" if is_int_candidate else "WEARER_OFFSET_CONTROL",
|
|
anchor_type="wearer_offset", frame=frame,
|
|
region_start=max(segment_start, frame - relation_frames),
|
|
region_end=min(segment_end, frame + relation_frames),
|
|
label="candidate" if is_int_candidate else "control", subtype=None,
|
|
previous_ids=_active_other_ids(rows, frame - 1, wearer_id=wearer_id),
|
|
incoming_ids=eligible_ids,
|
|
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
|
|
context_frames=context_frames,
|
|
reason=(
|
|
"new non-wearer onset occurs inside the wearer floor episode and crosses the yield boundary; semantic takeover status is unresolved"
|
|
if is_int_candidate else control_reason
|
|
),
|
|
), ("WEARER_OFFSET" if is_int_candidate else "WEARER_OFFSET_CONTROL", frame, eligible_ids))
|
|
|
|
# B1: same speaker pauses and resumes while the wearer remains silent.
|
|
by_speaker: dict[int, list[Utterance]] = defaultdict(list)
|
|
for row in other_rows:
|
|
by_speaker[row.participant_id].append(row)
|
|
for participant_id, speaker_rows in by_speaker.items():
|
|
for first, resumed in zip(speaker_rows, speaker_rows[1:]):
|
|
gap = resumed.dense_start_frame - first.dense_end_frame_exclusive
|
|
left = first.dense_end_frame_exclusive - segment_start
|
|
right = resumed.dense_start_frame - segment_start
|
|
if (gap <= 0 or gap > pause_frames
|
|
or not _all_state(wearer, left, right, False)
|
|
or not _all_state(others, left, right, False)):
|
|
continue
|
|
frame = first.dense_end_frame_exclusive
|
|
evidence = {
|
|
"speaker_id": participant_id, "speaker_offset_seconds": first.dense_end_seconds,
|
|
"speaker_resume_seconds": resumed.dense_start_seconds,
|
|
"resume_delay_seconds": gap / SOURCE_FPS, "wearer_active_during_pause": False,
|
|
"original_timestamp_seconds": frame / SOURCE_FPS,
|
|
}
|
|
add(_candidate(
|
|
session=session, task="wearer_entry", source="SAME_SPEAKER_PAUSE",
|
|
rule_type="ENTRY_HARD_NEGATIVE_REGION", anchor_type="other_offset", frame=frame,
|
|
region_start=frame, region_end=resumed.dense_start_frame,
|
|
label="hard_negative", subtype="ENTRY_OTHER_MIDTURN_PAUSE",
|
|
previous_ids=(participant_id,), incoming_ids=(participant_id,), evidence=evidence,
|
|
clips=segment_clips, rows=rows, wearer_id=wearer_id, context_frames=context_frames,
|
|
reason="same participant resumes after a configurable pause while wearer stays silent",
|
|
), ("SAME_SPEAKER_PAUSE", frame, (participant_id,)))
|
|
|
|
# B2: participant-resolved non-wearer to non-wearer handover.
|
|
for first in other_rows:
|
|
options = [row for row in other_rows if row.participant_id != first.participant_id
|
|
and row.dense_start_frame >= first.dense_end_frame_exclusive
|
|
and row.dense_start_frame - first.dense_end_frame_exclusive <= handover_frames]
|
|
if not options:
|
|
continue
|
|
incoming = min(options, key=lambda row: row.dense_start_frame)
|
|
same_resumes = any(row.participant_id == first.participant_id
|
|
and first.dense_end_frame_exclusive < row.dense_start_frame <= incoming.dense_start_frame
|
|
for row in other_rows)
|
|
left = first.dense_end_frame_exclusive - segment_start
|
|
right = max(left + 1, incoming.dense_start_frame - segment_start + 1)
|
|
if same_resumes or not _all_state(wearer, left, right, False):
|
|
continue
|
|
frame = first.dense_end_frame_exclusive
|
|
evidence = {
|
|
"previous_speaker_id": first.participant_id, "incoming_speaker_id": incoming.participant_id,
|
|
"previous_offset_seconds": first.dense_end_seconds,
|
|
"incoming_onset_seconds": incoming.dense_start_seconds,
|
|
"handover_gap_seconds": (incoming.dense_start_frame - frame) / SOURCE_FPS,
|
|
"wearer_active_during_handover": False, "original_timestamp_seconds": frame / SOURCE_FPS,
|
|
}
|
|
add(_candidate(
|
|
session=session, task="wearer_entry", source="OTHER_TO_OTHER",
|
|
rule_type="ENTRY_HARD_NEGATIVE_REGION", anchor_type="other_offset", frame=frame,
|
|
region_start=frame, region_end=incoming.dense_start_frame,
|
|
label="hard_negative", subtype="ENTRY_OTHER_TO_OTHER",
|
|
previous_ids=(first.participant_id,), incoming_ids=(incoming.participant_id,),
|
|
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
|
|
context_frames=context_frames,
|
|
reason="participant-level IDs show a non-wearer to non-wearer handover",
|
|
), ("OTHER_TO_OTHER", frame, (first.participant_id, incoming.participant_id)))
|
|
|
|
# B3/B4: other activity ends while the wearer continuously holds and continues the floor.
|
|
for other in other_rows:
|
|
start = other.dense_start_frame - segment_start
|
|
end = other.dense_end_frame_exclusive - segment_start
|
|
if (start < 0 or end + continuation_frames > len(wearer)
|
|
or not _all_state(wearer, start, end + continuation_frames, True)):
|
|
continue
|
|
feedback = (other.dense_end_frame_exclusive - other.dense_start_frame <= short_frames
|
|
and (is_lexical_feedback(other.transcription) or is_pure_nonverbal(other.transcription)))
|
|
source = "OTHER_BACKCHANNEL" if feedback else "NON_TAKEOVER_OVERLAP"
|
|
subtype = "INT_OTHER_BACKCHANNEL" if feedback else "INT_NON_TAKEOVER_OVERLAP"
|
|
evidence = {
|
|
"other_speaker_id": other.participant_id, "other_onset_seconds": other.dense_start_seconds,
|
|
"other_offset_seconds": other.dense_end_seconds, "other_duration_seconds": other.duration_seconds,
|
|
"wearer_active_through_region": True,
|
|
"wearer_continuation_after_other_seconds": continuation_frames / SOURCE_FPS,
|
|
"lexical_or_nonverbal_feedback_hint": feedback,
|
|
"original_timestamp_seconds": other.dense_start_seconds,
|
|
}
|
|
add(_candidate(
|
|
session=session, task="int_yield", source=source,
|
|
rule_type="INT_HARD_NEGATIVE_REGION", anchor_type="other_onset",
|
|
frame=other.dense_start_frame, region_start=other.dense_start_frame,
|
|
region_end=other.dense_end_frame_exclusive, label="hard_negative", subtype=subtype,
|
|
previous_ids=_active_other_ids(rows, other.dense_start_frame - 1, wearer_id=wearer_id),
|
|
incoming_ids=(other.participant_id,), evidence=evidence, clips=segment_clips,
|
|
rows=rows, wearer_id=wearer_id, context_frames=context_frames,
|
|
reason=("short feedback-like other activity ends while wearer holds the floor"
|
|
if feedback else "other overlaps but wearer retains and continues the floor"),
|
|
), (source, other.dense_start_frame, (other.participant_id,)))
|
|
|
|
return sorted(candidates, key=lambda row: (row.anchor_dense_frame, row.candidate_source, row.uid))
|
|
|
|
|
|
def stratified_limit(rows: Sequence[ProtocolCandidate], limit: int | None, *, seed: int) -> list[ProtocolCandidate]:
|
|
if limit is None or limit >= len(rows):
|
|
return list(rows)
|
|
if limit < 0:
|
|
raise ValueError("--max-candidates must be non-negative")
|
|
rng = random.Random(seed)
|
|
buckets: dict[tuple[str, str], list[ProtocolCandidate]] = defaultdict(list)
|
|
for row in rows:
|
|
buckets[(row.session, row.candidate_source)].append(row)
|
|
for bucket in buckets.values():
|
|
rng.shuffle(bucket)
|
|
selected: list[ProtocolCandidate] = []
|
|
while len(selected) < limit and any(buckets.values()):
|
|
for key in sorted(buckets):
|
|
if buckets[key] and len(selected) < limit:
|
|
selected.append(buckets[key].pop())
|
|
return sorted(selected, key=lambda row: (row.session, row.anchor_dense_frame, row.candidate_source))
|
|
|
|
|
|
def _write_jsonl(path: Path, rows: Iterable[ProtocolCandidate]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("w", encoding="utf-8") as handle:
|
|
for row in rows:
|
|
handle.write(json.dumps(asdict(row), ensure_ascii=False) + "\n")
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--dataset-root", type=Path, required=True)
|
|
parser.add_argument("--output-dir", type=Path, required=True)
|
|
parser.add_argument("--sessions", nargs="+", default=list(ALL_SESSIONS))
|
|
parser.add_argument("--camera-wearer-id", type=int, default=CAMERA_WEARER_ID)
|
|
parser.add_argument("--context-seconds", type=float, default=3.0)
|
|
parser.add_argument("--relation-window-seconds", type=float, default=3.0)
|
|
parser.add_argument("--same-speaker-pause-max-seconds", type=float, default=2.0)
|
|
parser.add_argument("--other-handover-max-seconds", type=float, default=1.5)
|
|
parser.add_argument("--short-vocalization-max-seconds", type=float, default=1.0)
|
|
parser.add_argument("--continuation-min-seconds", type=float, default=0.4)
|
|
parser.add_argument("--int-offset-tolerance-seconds", type=float, default=0.05)
|
|
parser.add_argument("--wearer-resume-merge-max-seconds", type=float, default=0.8)
|
|
parser.add_argument("--candidate-sources", nargs="+")
|
|
parser.add_argument("--max-candidates", type=int)
|
|
parser.add_argument("--selection-seed", type=int, default=2027)
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
unknown = sorted(set(args.sessions) - set(ALL_SESSIONS))
|
|
if unknown:
|
|
raise ValueError(f"Unknown EasyCom sessions: {unknown}")
|
|
config = CandidateConfig(
|
|
context_seconds=args.context_seconds,
|
|
relation_window_seconds=args.relation_window_seconds,
|
|
same_speaker_pause_max_seconds=args.same_speaker_pause_max_seconds,
|
|
other_handover_max_seconds=args.other_handover_max_seconds,
|
|
short_vocalization_max_seconds=args.short_vocalization_max_seconds,
|
|
continuation_min_seconds=args.continuation_min_seconds,
|
|
int_offset_tolerance_seconds=args.int_offset_tolerance_seconds,
|
|
wearer_resume_merge_max_seconds=args.wearer_resume_merge_max_seconds,
|
|
)
|
|
dataset_root, output_dir = args.dataset_root.expanduser().resolve(), args.output_dir.expanduser().resolve()
|
|
all_rows: list[ProtocolCandidate] = []
|
|
session_timelines: dict[str, list[dict[str, Any]]] = {}
|
|
rules = Rules()
|
|
for session in args.sessions:
|
|
clips = build_clips(dataset_root, session, rules)
|
|
utterances = load_utterances(session, clips)
|
|
session_timelines[session] = [{
|
|
"uid": row.uid,
|
|
"participant_id": row.participant_id,
|
|
"participant_role": "wearer" if row.participant_id == args.camera_wearer_id else "other",
|
|
"start_seconds": row.dense_start_seconds,
|
|
"end_seconds": row.dense_end_seconds,
|
|
} for row in utterances]
|
|
rows = derive_protocol_candidates(utterances, clips, session=session,
|
|
wearer_id=args.camera_wearer_id, config=config)
|
|
all_rows.extend(rows)
|
|
print(f"{session}: candidates={len(rows)} sources={dict(sorted(Counter(row.candidate_source for row in rows).items()))}")
|
|
sources = None if args.candidate_sources is None else set(args.candidate_sources)
|
|
controls = [row for row in all_rows if not requires_semantic_adjudication(asdict(row))]
|
|
primary = [row for row in all_rows if requires_semantic_adjudication(asdict(row))]
|
|
eligible = [row for row in primary if sources is None or row.candidate_source in sources]
|
|
selected = stratified_limit(eligible, args.max_candidates, seed=args.selection_seed)
|
|
output_path = output_dir / "easycom_eot_int_candidates.jsonl"
|
|
_write_jsonl(output_path, selected)
|
|
controls_path = output_dir / "easycom_eot_int_controls.jsonl"
|
|
_write_jsonl(controls_path, controls)
|
|
counts = Counter((row.split, row.candidate_source, row.rule_candidate_type) for row in selected)
|
|
summary = {
|
|
"schema_version": SCHEMA_VERSION, "protocol_version": PROTOCOL_VERSION, "dataset": "EasyCom",
|
|
"semantic_warning": "Rules create candidates only; AI labels need human review and never modify dense activity ground truth.",
|
|
"sessions": list(args.sessions), "camera_wearer_id": args.camera_wearer_id,
|
|
"candidate_config": asdict(config), "selection_seed": args.selection_seed,
|
|
"max_candidates": args.max_candidates,
|
|
"candidate_sources": None if sources is None else sorted(sources),
|
|
"num_candidates_before_filter": len(primary), "num_candidates_before_limit": len(eligible),
|
|
"num_candidates_written": len(selected),
|
|
"num_controls_written": len(controls),
|
|
"controls_file": controls_path.name,
|
|
"counts": [{"split": key[0], "candidate_source": key[1],
|
|
"rule_candidate_type": key[2], "count": value}
|
|
for key, value in sorted(counts.items())],
|
|
}
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
(output_dir / "summary.json").write_text(
|
|
json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
(output_dir / "easycom_session_timelines.json").write_text(
|
|
json.dumps({"schema_version": 1, "sessions": session_timelines}, ensure_ascii=False) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
print(f"Saved {len(selected)} wearer-centric review candidates to {output_path}")
|
|
print(f"Saved {len(controls)} deterministic boundary controls to {controls_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|