864 lines
44 KiB
Python
864 lines
44 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate wearer-centric EasyCom entry/INT candidates for AI and human review.
|
|
|
|
This stage never changes EasyCom speech boundaries and never claims semantic
|
|
ground truth. Every observed wearer onset and offset is retained as a semantic
|
|
candidate. Deterministic code only records participant-resolved timing,
|
|
transcripts, and target-of-speech facts; it never decides floor entry,
|
|
within-turn resumption, or takeover semantics. All timing windows are CLI
|
|
parameters, and raw evidence is retained for DeepSeek pre-labeling and human
|
|
correction.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import Counter, defaultdict
|
|
from dataclasses import asdict, dataclass
|
|
import json
|
|
from pathlib import Path
|
|
import random
|
|
from typing import Any, Iterable, Sequence
|
|
|
|
from derive_easycom_interaction_events import (
|
|
CAMERA_WEARER_ID, SOURCE_FPS, Clip, Rules, Utterance, build_clips,
|
|
is_lexical_feedback, is_pure_nonverbal, load_utterances,
|
|
)
|
|
from easycom_manifest import ALL_SESSIONS, TEST_SESSIONS
|
|
|
|
|
|
SCHEMA_VERSION = 6
|
|
PROTOCOL_VERSION = "easycom-wearer-entry-int-candidates-v8"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CandidateConfig:
|
|
context_seconds: float = 3.0
|
|
relation_window_seconds: float = 3.0
|
|
same_speaker_pause_max_seconds: float = 2.0
|
|
other_handover_max_seconds: float = 1.5
|
|
short_vocalization_max_seconds: float = 1.0
|
|
continuation_min_seconds: float = 0.4
|
|
int_offset_tolerance_seconds: float = 0.05
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ProtocolCandidate:
|
|
schema_version: int
|
|
protocol_version: str
|
|
uid: str
|
|
event_id: str
|
|
session: str
|
|
split: str
|
|
task: str
|
|
event_type: str
|
|
candidate_source: str
|
|
rule_candidate_type: str
|
|
automatic_protocol_label: str
|
|
automatic_hard_negative_type: str | None
|
|
automatic_confidence: str
|
|
anchor_type: str
|
|
anchor_dense_frame: int
|
|
anchor_dense_seconds: float
|
|
event_time: float
|
|
region_start_seconds: float
|
|
region_end_seconds: float
|
|
clip_stem: str
|
|
source_start_frame: int
|
|
previous_other_participant_ids: tuple[int, ...]
|
|
incoming_other_participant_ids: tuple[int, ...]
|
|
temporal_evidence: dict[str, Any]
|
|
speaker_timeline: tuple[dict[str, Any], ...]
|
|
context: tuple[dict[str, Any], ...]
|
|
automatic_reason: str
|
|
ai_verdict: None = None
|
|
ai_event_type: None = None
|
|
ai_subtype: None = None
|
|
ai_confidence: None = None
|
|
ai_reason: None = None
|
|
human_status: str = "UNREVIEWED"
|
|
human_verdict: None = None
|
|
human_event_type: None = None
|
|
human_subtype: None = None
|
|
exclude_reason: None = None
|
|
|
|
|
|
def requires_semantic_adjudication(row: dict[str, Any]) -> bool:
|
|
"""Return whether a stored row belongs in the primary AI/human queue.
|
|
|
|
Explicit legacy control sources are authoritative. Current WEARER_OFFSET
|
|
rows are always semantic candidates: crossing/order fields are evidence,
|
|
never admission gates.
|
|
"""
|
|
source = row.get("candidate_source")
|
|
if source in {"WEARER_ONSET_CONTROL", "WEARER_OFFSET_CONTROL", "WEARER_RESUME"}:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _fill_states(rows: Sequence[Utterance], *, segment_start: int, segment_end: int,
|
|
wearer_id: int) -> tuple[bytearray, bytearray]:
|
|
length = segment_end - segment_start
|
|
wearer, others = bytearray(length), bytearray(length)
|
|
for row in rows:
|
|
left = max(segment_start, row.dense_start_frame) - segment_start
|
|
right = min(segment_end, row.dense_end_frame_exclusive) - segment_start
|
|
if right <= left:
|
|
continue
|
|
target = wearer if row.participant_id == wearer_id else others
|
|
target[left:right] = b"\x01" * (right - left)
|
|
return wearer, others
|
|
|
|
|
|
def _run_after(state: bytearray, index: int, value: bool) -> int:
|
|
end = index
|
|
while end < len(state) and bool(state[end]) == value:
|
|
end += 1
|
|
return end - index
|
|
|
|
|
|
def _run_before(state: bytearray, index: int, value: bool) -> int:
|
|
start = index - 1
|
|
while start >= 0 and bool(state[start]) == value:
|
|
start -= 1
|
|
return index - start - 1
|
|
|
|
|
|
def _all_state(state: bytearray, start: int, stop: int, value: bool) -> bool:
|
|
left, right = max(0, start), min(len(state), stop)
|
|
return right > left and all(bool(state[i]) == value for i in range(left, right))
|
|
|
|
|
|
def _active_other_ids(rows: Sequence[Utterance], frame: int, *, wearer_id: int) -> tuple[int, ...]:
|
|
return tuple(sorted({
|
|
row.participant_id for row in rows
|
|
if row.participant_id != wearer_id
|
|
and row.dense_start_frame <= frame < row.dense_end_frame_exclusive
|
|
}))
|
|
|
|
|
|
def _other_ids_crossing_offset(
|
|
rows: Sequence[Utterance], *, frame: int, wearer_id: int, tolerance_frames: int
|
|
) -> tuple[int, ...]:
|
|
"""Return other speakers whose annotated interval crosses a wearer offset.
|
|
|
|
With zero tolerance this records the strict temporal relation
|
|
``other.start < wearer_offset < other.end``. A one-frame tolerance accepts
|
|
an annotation boundary at the same 20 Hz frame, but not an other onset one
|
|
or more frames after the wearer offset.
|
|
"""
|
|
return tuple(sorted({
|
|
row.participant_id for row in rows
|
|
if row.participant_id != wearer_id
|
|
and row.dense_start_frame < frame + tolerance_frames
|
|
and row.dense_end_frame_exclusive > frame - tolerance_frames
|
|
}))
|
|
|
|
|
|
def _wearer_gap_evidence(
|
|
rows: Sequence[Utterance], *, gap_start_frame: int, onset_frame: int,
|
|
wearer_id: int,
|
|
) -> dict[str, Any]:
|
|
"""Return descriptive other-speaker facts within a wearer-silence gap."""
|
|
gap_rows = [
|
|
row for row in rows
|
|
if row.participant_id != wearer_id
|
|
and row.dense_start_frame < onset_frame
|
|
and row.dense_end_frame_exclusive > gap_start_frame
|
|
]
|
|
max_gap_coverage = 0
|
|
intervals: list[dict[str, Any]] = []
|
|
for row in gap_rows:
|
|
coverage = max(
|
|
0,
|
|
min(row.dense_end_frame_exclusive, onset_frame)
|
|
- max(row.dense_start_frame, gap_start_frame),
|
|
)
|
|
max_gap_coverage = max(max_gap_coverage, coverage)
|
|
intervals.append({
|
|
"uid": row.uid,
|
|
"participant_id": row.participant_id,
|
|
"onset_seconds": row.dense_start_seconds,
|
|
"offset_seconds": row.dense_end_seconds,
|
|
"gap_overlap_seconds": coverage / SOURCE_FPS,
|
|
"continues_across_wearer_onset": row.dense_end_frame_exclusive > onset_frame,
|
|
"transcription": row.transcription,
|
|
"target_of_speech": list(row.target_of_speech),
|
|
})
|
|
return {
|
|
"gap_other_participant_ids": sorted({row.participant_id for row in gap_rows}),
|
|
"gap_other_intervals": intervals,
|
|
"max_other_gap_coverage_seconds": max_gap_coverage / SOURCE_FPS,
|
|
}
|
|
|
|
|
|
def _clip_for_frame(clips: Sequence[Clip], frame: int) -> Clip | None:
|
|
return next((clip for clip in clips
|
|
if clip.dense_start_frame <= frame < clip.dense_start_frame + clip.num_frames), None)
|
|
|
|
|
|
def _context_rows(rows: Sequence[Utterance], *, frame: int, context_frames: int,
|
|
wearer_id: int) -> tuple[dict[str, Any], ...]:
|
|
left, right = frame - context_frames, frame + context_frames
|
|
return tuple({
|
|
"uid": row.uid,
|
|
"participant_id": row.participant_id,
|
|
"participant_role": "wearer" if row.participant_id == wearer_id else "other",
|
|
"start_seconds": row.dense_start_seconds,
|
|
"end_seconds": row.dense_end_seconds,
|
|
"duration_seconds": row.duration_seconds,
|
|
"transcription": row.transcription,
|
|
"target_of_speech": list(row.target_of_speech),
|
|
} for row in rows
|
|
if row.dense_end_frame_exclusive > left and row.dense_start_frame < right)
|
|
|
|
|
|
def _relation_evidence(rows: Sequence[Utterance], *, frame: int, wearer_id: int,
|
|
window_frames: int) -> dict[str, Any]:
|
|
previous = [row for row in rows if row.participant_id != wearer_id
|
|
and row.dense_end_frame_exclusive <= frame
|
|
and frame - row.dense_end_frame_exclusive <= window_frames]
|
|
following = [row for row in rows if row.participant_id != wearer_id
|
|
and row.dense_start_frame >= frame
|
|
and row.dense_start_frame - frame <= window_frames]
|
|
prev = max(previous, key=lambda row: row.dense_end_frame_exclusive, default=None)
|
|
nxt = min(following, key=lambda row: row.dense_start_frame, default=None)
|
|
active_rows = [row for row in rows if row.participant_id != wearer_id
|
|
and row.dense_start_frame < frame < row.dense_end_frame_exclusive]
|
|
overlapping = sorted({row.participant_id for row in active_rows})
|
|
resumes = [] if prev is None else [row for row in following if row.participant_id == prev.participant_id]
|
|
resumed = min(resumes, key=lambda row: row.dense_start_frame, default=None)
|
|
nearby_onsets = [row for row in rows if row.participant_id != wearer_id
|
|
and abs(row.dense_start_frame - frame) <= window_frames]
|
|
closest_onset = min(nearby_onsets, key=lambda row: abs(row.dense_start_frame - frame), default=None)
|
|
return {
|
|
"previous_speaker_id": None if prev is None else prev.participant_id,
|
|
"previous_speaker_offset_seconds": None if prev is None else prev.dense_end_seconds,
|
|
"gap_from_previous_speaker_offset_seconds": (
|
|
None if prev is None else (frame - prev.dense_end_frame_exclusive) / SOURCE_FPS),
|
|
"overlapping_other_ids": overlapping,
|
|
"other_overlap_at_event_seconds": max(
|
|
[0.0] + [(row.dense_end_frame_exclusive - frame) / SOURCE_FPS for row in active_rows]
|
|
),
|
|
"next_speaker_id": None if nxt is None else nxt.participant_id,
|
|
"next_other_onset_seconds": None if nxt is None else nxt.dense_start_seconds,
|
|
"previous_speaker_resumes": resumed is not None,
|
|
"previous_speaker_resume_delay_seconds": (
|
|
None if resumed is None or prev is None
|
|
else (resumed.dense_start_frame - prev.dense_end_frame_exclusive) / SOURCE_FPS
|
|
),
|
|
"closest_other_onset_relative_seconds": (
|
|
None if closest_onset is None else (closest_onset.dense_start_frame - frame) / SOURCE_FPS
|
|
),
|
|
}
|
|
|
|
|
|
def _candidate(*, session: str, task: str, source: str, rule_type: str,
|
|
anchor_type: str, frame: int, region_start: int, region_end: int,
|
|
label: str, subtype: str | None, previous_ids: tuple[int, ...],
|
|
incoming_ids: tuple[int, ...], evidence: dict[str, Any], clips: Sequence[Clip],
|
|
rows: Sequence[Utterance], wearer_id: int, context_frames: int,
|
|
reason: str) -> ProtocolCandidate | None:
|
|
clip = _clip_for_frame(clips, frame)
|
|
if clip is None:
|
|
return None
|
|
participant_key = "prev-" + ("-".join(f"p{x}" for x in previous_ids) or "none")
|
|
participant_key += "__in-" + ("-".join(f"p{x}" for x in incoming_ids) or "none")
|
|
uid = f"{session}/{source.lower()}/{frame:08d}/{participant_key}"
|
|
context = _context_rows(rows, frame=frame, context_frames=context_frames, wearer_id=wearer_id)
|
|
return ProtocolCandidate(
|
|
schema_version=SCHEMA_VERSION, protocol_version=PROTOCOL_VERSION,
|
|
uid=uid, event_id=uid, session=session,
|
|
split="test" if session in TEST_SESSIONS else "train", task=task,
|
|
event_type=source.lower(), candidate_source=source, rule_candidate_type=rule_type,
|
|
automatic_protocol_label=label, automatic_hard_negative_type=subtype,
|
|
automatic_confidence="rule_candidate", anchor_type=anchor_type,
|
|
anchor_dense_frame=frame, anchor_dense_seconds=frame / SOURCE_FPS,
|
|
event_time=frame / SOURCE_FPS, region_start_seconds=region_start / SOURCE_FPS,
|
|
region_end_seconds=region_end / SOURCE_FPS, clip_stem=clip.stem,
|
|
source_start_frame=frame - clip.dense_start_frame + 1,
|
|
previous_other_participant_ids=previous_ids,
|
|
incoming_other_participant_ids=incoming_ids, temporal_evidence=evidence,
|
|
speaker_timeline=context, context=context, automatic_reason=reason,
|
|
)
|
|
|
|
|
|
def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[Clip], *,
|
|
session: str, wearer_id: int = CAMERA_WEARER_ID,
|
|
config: CandidateConfig | None = None,
|
|
window_seconds: float | None = None,
|
|
context_seconds: float | None = None) -> list[ProtocolCandidate]:
|
|
"""Return primary semantic candidates plus separate offset controls."""
|
|
cfg = config or CandidateConfig()
|
|
values = asdict(cfg)
|
|
if window_seconds is not None:
|
|
values["relation_window_seconds"] = window_seconds
|
|
if context_seconds is not None:
|
|
values["context_seconds"] = context_seconds
|
|
cfg = CandidateConfig(**values)
|
|
if any(value < 0 for value in asdict(cfg).values()) or cfg.relation_window_seconds == 0:
|
|
raise ValueError("candidate timing parameters must be non-negative and relation window positive")
|
|
context_frames = round(cfg.context_seconds * SOURCE_FPS)
|
|
relation_frames = round(cfg.relation_window_seconds * SOURCE_FPS)
|
|
pause_frames = round(cfg.same_speaker_pause_max_seconds * SOURCE_FPS)
|
|
handover_frames = round(cfg.other_handover_max_seconds * SOURCE_FPS)
|
|
short_frames = round(cfg.short_vocalization_max_seconds * SOURCE_FPS)
|
|
continuation_frames = round(cfg.continuation_min_seconds * SOURCE_FPS)
|
|
int_offset_tolerance_frames = round(cfg.int_offset_tolerance_seconds * SOURCE_FPS)
|
|
rows_by_segment: dict[int, list[Utterance]] = defaultdict(list)
|
|
clips_by_segment: dict[int, list[Clip]] = defaultdict(list)
|
|
for row in utterances:
|
|
rows_by_segment[row.segment_id].append(row)
|
|
for clip in clips:
|
|
clips_by_segment[clip.segment_id].append(clip)
|
|
candidates: list[ProtocolCandidate] = []
|
|
seen: set[tuple[str, int, tuple[int, ...]]] = set()
|
|
|
|
def add(item: ProtocolCandidate | None, key: tuple[str, int, tuple[int, ...]]) -> None:
|
|
if item is not None and key not in seen:
|
|
seen.add(key)
|
|
candidates.append(item)
|
|
|
|
for segment_id, segment_clips in sorted(clips_by_segment.items()):
|
|
segment_clips.sort(key=lambda clip: clip.dense_start_frame)
|
|
rows = sorted(rows_by_segment.get(segment_id, []),
|
|
key=lambda row: (row.dense_start_frame, row.dense_end_frame_exclusive, row.uid))
|
|
segment_start = segment_clips[0].dense_start_frame
|
|
segment_end = segment_clips[-1].dense_start_frame + segment_clips[-1].num_frames
|
|
wearer, others = _fill_states(rows, segment_start=segment_start,
|
|
segment_end=segment_end, wearer_id=wearer_id)
|
|
other_rows = [row for row in rows if row.participant_id != wearer_id]
|
|
|
|
# A: retain every raw wearer onset and offset. Gap and nearby-speaker
|
|
# properties are evidence only; they never suppress a semantic item.
|
|
onset_records: dict[int, dict[str, Any]] = {}
|
|
raw_offsets: list[int] = []
|
|
for index in range(1, len(wearer)):
|
|
frame = segment_start + index
|
|
if not wearer[index - 1] and wearer[index]:
|
|
gap_frames = _run_before(wearer, index, False)
|
|
previous_wearer_index = index - gap_frames - 1
|
|
has_previous_wearer = previous_wearer_index >= 0 and bool(wearer[previous_wearer_index])
|
|
previous_wearer_offset = segment_start + index - gap_frames
|
|
gap_evidence = _wearer_gap_evidence(
|
|
rows,
|
|
gap_start_frame=previous_wearer_offset,
|
|
onset_frame=frame,
|
|
wearer_id=wearer_id,
|
|
) if has_previous_wearer else {
|
|
"gap_other_participant_ids": [],
|
|
"gap_other_intervals": [],
|
|
"max_other_gap_coverage_seconds": 0.0,
|
|
}
|
|
evidence = _relation_evidence(rows, frame=frame, wearer_id=wearer_id,
|
|
window_frames=relation_frames)
|
|
evidence.update(
|
|
wearer_speech_duration_after_onset_seconds=_run_after(wearer, index, True) / SOURCE_FPS,
|
|
previous_wearer_silence_duration_seconds=gap_frames / SOURCE_FPS,
|
|
previous_wearer_offset_seconds=(
|
|
previous_wearer_offset / SOURCE_FPS if has_previous_wearer else None
|
|
),
|
|
original_timestamp_seconds=frame / SOURCE_FPS,
|
|
**gap_evidence,
|
|
)
|
|
onset_records[frame] = {"evidence": evidence}
|
|
elif wearer[index - 1] and not wearer[index]:
|
|
raw_offsets.append(frame)
|
|
|
|
# A1: every observed raw wearer onset is an unresolved entry candidate.
|
|
# Every nearby preceding other offset is supplied with factual relation
|
|
# flags; no release is declared admissible or causal by this stage.
|
|
for onset_frame in sorted(onset_records):
|
|
release_options: list[tuple[Utterance, int]] = []
|
|
for release in other_rows:
|
|
opportunity_frame = release.dense_end_frame_exclusive
|
|
if opportunity_frame > onset_frame or onset_frame - opportunity_frame > relation_frames:
|
|
continue
|
|
release_options.append((release, opportunity_frame))
|
|
|
|
release_options.sort(
|
|
key=lambda item: (item[1], item[0].participant_id, item[0].uid)
|
|
)
|
|
earliest_opportunity_frame = (
|
|
release_options[0][1] if release_options else max(segment_start, onset_frame - relation_frames)
|
|
)
|
|
option_rows = []
|
|
for release, release_frame in release_options:
|
|
interval_left = release_frame - segment_start
|
|
interval_right = onset_frame - segment_start
|
|
option_rows.append({
|
|
"uid": release.uid,
|
|
"anchor_type": "other_offset",
|
|
"anchor_frame": release_frame,
|
|
"anchor_seconds": release_frame / SOURCE_FPS,
|
|
"participant_id": release.participant_id,
|
|
"transcription": release.transcription,
|
|
"target_of_speech": list(release.target_of_speech),
|
|
"response_delay_seconds": (onset_frame - release_frame) / SOURCE_FPS,
|
|
"wearer_active_between_release_and_onset": any(
|
|
bool(wearer[i]) for i in range(max(0, interval_left), min(len(wearer), interval_right))
|
|
),
|
|
"other_onset_between_release_and_onset": any(
|
|
release_frame <= row.dense_start_frame < onset_frame for row in other_rows
|
|
),
|
|
"other_active_at_wearer_onset": bool(
|
|
_active_other_ids(rows, onset_frame, wearer_id=wearer_id)
|
|
),
|
|
})
|
|
evidence = dict(onset_records[onset_frame]["evidence"])
|
|
evidence.update(
|
|
opportunity_options=option_rows,
|
|
opportunity_anchor_type=None,
|
|
opportunity_anchor_frame=None,
|
|
opportunity_anchor_seconds=None,
|
|
behavior_anchor_type="wearer_raw_onset",
|
|
behavior_anchor_frame=onset_frame,
|
|
behavior_anchor_seconds=onset_frame / SOURCE_FPS,
|
|
preceding_release_options_present=bool(option_rows),
|
|
minimum_response_delay_seconds=(
|
|
None if not option_rows
|
|
else min(option["response_delay_seconds"] for option in option_rows)
|
|
),
|
|
maximum_response_delay_seconds=(
|
|
None if not option_rows
|
|
else max(option["response_delay_seconds"] for option in option_rows)
|
|
),
|
|
entry_context_timing=(
|
|
"OTHER_ACTIVE"
|
|
if _active_other_ids(rows, onset_frame, wearer_id=wearer_id)
|
|
else "NO_OTHER_ACTIVE_AT_ONSET"
|
|
),
|
|
)
|
|
previous_ids = tuple(sorted({row.participant_id for row, _ in release_options}))
|
|
add(_candidate(
|
|
session=session, task="wearer_entry", source="WEARER_ONSET",
|
|
rule_type="WEARER_ENTRY_CANDIDATE", anchor_type="wearer_onset", frame=onset_frame,
|
|
region_start=earliest_opportunity_frame,
|
|
region_end=min(segment_end, onset_frame + relation_frames),
|
|
label="candidate", subtype=None,
|
|
previous_ids=previous_ids,
|
|
incoming_ids=_active_other_ids(rows, onset_frame, wearer_id=wearer_id),
|
|
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
|
|
context_frames=context_frames,
|
|
reason=(
|
|
"raw wearer onset retained without floor-entry or within-turn-resumption gating; "
|
|
"nearby release relations are descriptive evidence"
|
|
),
|
|
), ("WEARER_ONSET", onset_frame, previous_ids))
|
|
|
|
# A2: every raw wearer offset is an unresolved INT candidate. The code
|
|
# records temporal relationships but does not infer takeover semantics.
|
|
# In particular, new/pre-existing/post-offset other activity and
|
|
# boundary crossing are evidence for DeepSeek and the human reviewer,
|
|
# never deterministic admission gates.
|
|
current_activity_run_start = segment_start if wearer and wearer[0] else None
|
|
activity_run_start_by_offset: dict[int, int | None] = {}
|
|
for index in range(1, len(wearer)):
|
|
frame = segment_start + index
|
|
if not wearer[index - 1] and wearer[index]:
|
|
current_activity_run_start = frame
|
|
elif wearer[index - 1] and not wearer[index]:
|
|
activity_run_start_by_offset[frame] = current_activity_run_start
|
|
current_activity_run_start = None
|
|
|
|
for frame in raw_offsets:
|
|
index = frame - segment_start
|
|
activity_run_start = activity_run_start_by_offset[frame]
|
|
strict_crossing_ids = _other_ids_crossing_offset(
|
|
other_rows, frame=frame, wearer_id=wearer_id, tolerance_frames=0
|
|
)
|
|
tolerance_crossing_ids = _other_ids_crossing_offset(
|
|
other_rows, frame=frame, wearer_id=wearer_id,
|
|
tolerance_frames=int_offset_tolerance_frames,
|
|
)
|
|
new_other_rows = [
|
|
row for row in other_rows
|
|
if activity_run_start is not None
|
|
and activity_run_start < row.dense_start_frame < frame
|
|
]
|
|
new_crossing_rows = [
|
|
row for row in new_other_rows
|
|
if row.dense_end_frame_exclusive > frame - int_offset_tolerance_frames
|
|
]
|
|
new_other_ids = tuple(sorted({row.participant_id for row in new_other_rows}))
|
|
new_crossing_ids = tuple(sorted({row.participant_id for row in new_crossing_rows}))
|
|
preexisting_rows = [
|
|
row for row in other_rows
|
|
if activity_run_start is not None
|
|
and row.dense_start_frame <= activity_run_start
|
|
and row.dense_end_frame_exclusive > activity_run_start
|
|
]
|
|
preexisting_ids = tuple(sorted({row.participant_id for row in preexisting_rows}))
|
|
nearby_other_rows = [
|
|
row for row in other_rows
|
|
if row.dense_end_frame_exclusive > frame - relation_frames
|
|
and row.dense_start_frame < frame + relation_frames
|
|
]
|
|
nearby_other_intervals = [{
|
|
"uid": row.uid,
|
|
"participant_id": row.participant_id,
|
|
"onset_seconds": row.dense_start_seconds,
|
|
"offset_seconds": row.dense_end_seconds,
|
|
"onset_relative_to_wearer_offset_seconds": (
|
|
row.dense_start_frame - frame
|
|
) / SOURCE_FPS,
|
|
"offset_relative_to_wearer_offset_seconds": (
|
|
row.dense_end_frame_exclusive - frame
|
|
) / SOURCE_FPS,
|
|
"starts_during_wearer_activity_run": bool(
|
|
activity_run_start is not None
|
|
and activity_run_start < row.dense_start_frame < frame
|
|
),
|
|
"preexisting_at_wearer_activity_run_start": bool(
|
|
activity_run_start is not None
|
|
and row.dense_start_frame <= activity_run_start
|
|
and row.dense_end_frame_exclusive > activity_run_start
|
|
),
|
|
"active_immediately_before_offset": bool(
|
|
row.dense_start_frame <= frame - 1 < row.dense_end_frame_exclusive
|
|
),
|
|
"active_immediately_after_offset": bool(
|
|
row.dense_start_frame <= frame < row.dense_end_frame_exclusive
|
|
),
|
|
"strictly_crosses_offset": bool(
|
|
row.dense_start_frame < frame < row.dense_end_frame_exclusive
|
|
),
|
|
"crosses_offset_with_tolerance": bool(
|
|
row.dense_start_frame < frame + int_offset_tolerance_frames
|
|
and row.dense_end_frame_exclusive > frame - int_offset_tolerance_frames
|
|
),
|
|
"overlap_with_wearer_before_offset_seconds": (
|
|
0.0 if activity_run_start is None else max(
|
|
0, min(row.dense_end_frame_exclusive, frame)
|
|
- max(row.dense_start_frame, activity_run_start)
|
|
) / SOURCE_FPS
|
|
),
|
|
"transcription": row.transcription,
|
|
"target_of_speech": list(row.target_of_speech),
|
|
} for row in nearby_other_rows]
|
|
next_rows = sorted(
|
|
[
|
|
row for row in rows
|
|
if frame <= row.dense_start_frame <= frame + relation_frames
|
|
],
|
|
key=lambda row: (row.dense_start_frame, row.participant_id, row.uid),
|
|
)
|
|
first_next_frame = next_rows[0].dense_start_frame if next_rows else None
|
|
first_next_rows = (
|
|
[] if first_next_frame is None
|
|
else [row for row in next_rows if row.dense_start_frame == first_next_frame]
|
|
)
|
|
active_other_ids_after_offset = _active_other_ids(
|
|
rows, frame, wearer_id=wearer_id
|
|
)
|
|
next_speaker_ids = (
|
|
list(active_other_ids_after_offset)
|
|
if active_other_ids_after_offset
|
|
else [row.participant_id for row in first_next_rows]
|
|
)
|
|
next_speaker_roles = (
|
|
["other"] * len(active_other_ids_after_offset)
|
|
if active_other_ids_after_offset
|
|
else [
|
|
"wearer" if row.participant_id == wearer_id else "other"
|
|
for row in first_next_rows
|
|
]
|
|
)
|
|
evidence = _relation_evidence(
|
|
rows, frame=frame, wearer_id=wearer_id, window_frames=relation_frames
|
|
)
|
|
evidence.update(
|
|
wearer_activity_run_start_frame=activity_run_start,
|
|
wearer_activity_run_start_seconds=(
|
|
None if activity_run_start is None else activity_run_start / SOURCE_FPS
|
|
),
|
|
wearer_activity_run_duration_to_offset_seconds=(
|
|
None if activity_run_start is None
|
|
else (frame - activity_run_start) / SOURCE_FPS
|
|
),
|
|
wearer_offset_frame=frame,
|
|
wearer_offset_seconds=frame / SOURCE_FPS,
|
|
wearer_speech_duration_before_offset_seconds=_run_before(wearer, index, True) / SOURCE_FPS,
|
|
wearer_silence_duration_after_offset_seconds=_run_after(wearer, index, False) / SOURCE_FPS,
|
|
active_other_ids_before_offset=list(_active_other_ids(rows, frame - 1, wearer_id=wearer_id)),
|
|
active_other_ids_after_offset=list(active_other_ids_after_offset),
|
|
strict_crossing_other_ids=list(strict_crossing_ids),
|
|
tolerance_crossing_other_ids=list(tolerance_crossing_ids),
|
|
new_other_onset_ids_during_wearer=list(new_other_ids),
|
|
new_other_onsets_during_wearer=[{
|
|
"participant_id": row.participant_id,
|
|
"onset_seconds": row.dense_start_seconds,
|
|
"offset_seconds": row.dense_end_seconds,
|
|
"uid": row.uid,
|
|
} for row in new_other_rows],
|
|
new_other_onset_ids_crossing_offset=list(new_crossing_ids),
|
|
preexisting_other_ids_at_wearer_activity_run_start=list(preexisting_ids),
|
|
preexisting_other_intervals=[{
|
|
"participant_id": row.participant_id,
|
|
"onset_seconds": row.dense_start_seconds,
|
|
"offset_seconds": row.dense_end_seconds,
|
|
"uid": row.uid,
|
|
} for row in preexisting_rows],
|
|
nearby_other_intervals=nearby_other_intervals,
|
|
next_speech_onsets=[{
|
|
"participant_id": row.participant_id,
|
|
"participant_role": "wearer" if row.participant_id == wearer_id else "other",
|
|
"onset_seconds": row.dense_start_seconds,
|
|
"offset_seconds": row.dense_end_seconds,
|
|
"uid": row.uid,
|
|
"transcription": row.transcription,
|
|
"target_of_speech": list(row.target_of_speech),
|
|
} for row in next_rows],
|
|
next_speaker_ids=next_speaker_ids,
|
|
next_speaker_roles=next_speaker_roles,
|
|
next_speaker_basis=(
|
|
"active_across_offset"
|
|
if active_other_ids_after_offset
|
|
else ("first_new_onset_after_offset" if first_next_rows else None)
|
|
),
|
|
next_speech_onset_seconds=(
|
|
frame / SOURCE_FPS
|
|
if active_other_ids_after_offset
|
|
else (None if first_next_frame is None else first_next_frame / SOURCE_FPS)
|
|
),
|
|
other_crosses_wearer_offset=bool(tolerance_crossing_ids),
|
|
new_other_onset_during_wearer=bool(new_other_rows),
|
|
new_other_onset_crosses_offset=bool(new_crossing_rows),
|
|
int_offset_tolerance_seconds=cfg.int_offset_tolerance_seconds,
|
|
original_timestamp_seconds=frame / SOURCE_FPS,
|
|
)
|
|
add(_candidate(
|
|
session=session, task="int_yield", source="WEARER_OFFSET",
|
|
rule_type="YIELD_CANDIDATE",
|
|
anchor_type="wearer_offset", frame=frame,
|
|
region_start=max(segment_start, frame - relation_frames),
|
|
region_end=min(segment_end, frame + relation_frames),
|
|
label="candidate", subtype=None,
|
|
previous_ids=_active_other_ids(rows, frame - 1, wearer_id=wearer_id),
|
|
incoming_ids=new_other_ids,
|
|
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
|
|
context_frames=context_frames,
|
|
reason=(
|
|
"raw wearer offset retained without timing-based takeover gating; "
|
|
"all nearby activity relations are descriptive evidence"
|
|
),
|
|
), ("WEARER_OFFSET", frame, new_other_ids))
|
|
|
|
# B1: same speaker pauses and resumes while the wearer remains silent.
|
|
by_speaker: dict[int, list[Utterance]] = defaultdict(list)
|
|
for row in other_rows:
|
|
by_speaker[row.participant_id].append(row)
|
|
for participant_id, speaker_rows in by_speaker.items():
|
|
for first, resumed in zip(speaker_rows, speaker_rows[1:]):
|
|
gap = resumed.dense_start_frame - first.dense_end_frame_exclusive
|
|
left = first.dense_end_frame_exclusive - segment_start
|
|
right = resumed.dense_start_frame - segment_start
|
|
if (gap <= 0 or gap > pause_frames
|
|
or not _all_state(wearer, left, right, False)
|
|
or not _all_state(others, left, right, False)):
|
|
continue
|
|
frame = first.dense_end_frame_exclusive
|
|
evidence = {
|
|
"speaker_id": participant_id, "speaker_offset_seconds": first.dense_end_seconds,
|
|
"speaker_resume_seconds": resumed.dense_start_seconds,
|
|
"resume_delay_seconds": gap / SOURCE_FPS, "wearer_active_during_pause": False,
|
|
"original_timestamp_seconds": frame / SOURCE_FPS,
|
|
}
|
|
add(_candidate(
|
|
session=session, task="wearer_entry", source="SAME_SPEAKER_PAUSE",
|
|
rule_type="ENTRY_HARD_NEGATIVE_REGION", anchor_type="other_offset", frame=frame,
|
|
region_start=frame, region_end=resumed.dense_start_frame,
|
|
label="hard_negative", subtype="ENTRY_OTHER_MIDTURN_PAUSE",
|
|
previous_ids=(participant_id,), incoming_ids=(participant_id,), evidence=evidence,
|
|
clips=segment_clips, rows=rows, wearer_id=wearer_id, context_frames=context_frames,
|
|
reason="same participant resumes after a configurable pause while wearer stays silent",
|
|
), ("SAME_SPEAKER_PAUSE", frame, (participant_id,)))
|
|
|
|
# B2: participant-resolved non-wearer to non-wearer handover.
|
|
for first in other_rows:
|
|
options = [row for row in other_rows if row.participant_id != first.participant_id
|
|
and row.dense_start_frame >= first.dense_end_frame_exclusive
|
|
and row.dense_start_frame - first.dense_end_frame_exclusive <= handover_frames]
|
|
if not options:
|
|
continue
|
|
incoming = min(options, key=lambda row: row.dense_start_frame)
|
|
same_resumes = any(row.participant_id == first.participant_id
|
|
and first.dense_end_frame_exclusive < row.dense_start_frame <= incoming.dense_start_frame
|
|
for row in other_rows)
|
|
left = first.dense_end_frame_exclusive - segment_start
|
|
right = max(left + 1, incoming.dense_start_frame - segment_start + 1)
|
|
if same_resumes or not _all_state(wearer, left, right, False):
|
|
continue
|
|
frame = first.dense_end_frame_exclusive
|
|
evidence = {
|
|
"previous_speaker_id": first.participant_id, "incoming_speaker_id": incoming.participant_id,
|
|
"previous_offset_seconds": first.dense_end_seconds,
|
|
"incoming_onset_seconds": incoming.dense_start_seconds,
|
|
"handover_gap_seconds": (incoming.dense_start_frame - frame) / SOURCE_FPS,
|
|
"wearer_active_during_handover": False, "original_timestamp_seconds": frame / SOURCE_FPS,
|
|
}
|
|
add(_candidate(
|
|
session=session, task="wearer_entry", source="OTHER_TO_OTHER",
|
|
rule_type="ENTRY_HARD_NEGATIVE_REGION", anchor_type="other_offset", frame=frame,
|
|
region_start=frame, region_end=incoming.dense_start_frame,
|
|
label="hard_negative", subtype="ENTRY_OTHER_TO_OTHER",
|
|
previous_ids=(first.participant_id,), incoming_ids=(incoming.participant_id,),
|
|
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
|
|
context_frames=context_frames,
|
|
reason="participant-level IDs show a non-wearer to non-wearer handover",
|
|
), ("OTHER_TO_OTHER", frame, (first.participant_id, incoming.participant_id)))
|
|
|
|
# B3/B4: other activity ends while the wearer continuously holds and continues the floor.
|
|
for other in other_rows:
|
|
start = other.dense_start_frame - segment_start
|
|
end = other.dense_end_frame_exclusive - segment_start
|
|
if (start < 0 or end + continuation_frames > len(wearer)
|
|
or not _all_state(wearer, start, end + continuation_frames, True)):
|
|
continue
|
|
feedback = (other.dense_end_frame_exclusive - other.dense_start_frame <= short_frames
|
|
and (is_lexical_feedback(other.transcription) or is_pure_nonverbal(other.transcription)))
|
|
source = "OTHER_BACKCHANNEL" if feedback else "NON_TAKEOVER_OVERLAP"
|
|
subtype = "INT_OTHER_BACKCHANNEL" if feedback else "INT_NON_TAKEOVER_OVERLAP"
|
|
evidence = {
|
|
"other_speaker_id": other.participant_id, "other_onset_seconds": other.dense_start_seconds,
|
|
"other_offset_seconds": other.dense_end_seconds, "other_duration_seconds": other.duration_seconds,
|
|
"wearer_active_through_region": True,
|
|
"wearer_continuation_after_other_seconds": continuation_frames / SOURCE_FPS,
|
|
"lexical_or_nonverbal_feedback_hint": feedback,
|
|
"original_timestamp_seconds": other.dense_start_seconds,
|
|
}
|
|
add(_candidate(
|
|
session=session, task="int_yield", source=source,
|
|
rule_type="INT_HARD_NEGATIVE_REGION", anchor_type="other_onset",
|
|
frame=other.dense_start_frame, region_start=other.dense_start_frame,
|
|
region_end=other.dense_end_frame_exclusive, label="hard_negative", subtype=subtype,
|
|
previous_ids=_active_other_ids(rows, other.dense_start_frame - 1, wearer_id=wearer_id),
|
|
incoming_ids=(other.participant_id,), evidence=evidence, clips=segment_clips,
|
|
rows=rows, wearer_id=wearer_id, context_frames=context_frames,
|
|
reason=("short feedback-like other activity ends while wearer holds the floor"
|
|
if feedback else "other overlaps but wearer retains and continues the floor"),
|
|
), (source, other.dense_start_frame, (other.participant_id,)))
|
|
|
|
return sorted(candidates, key=lambda row: (row.anchor_dense_frame, row.candidate_source, row.uid))
|
|
|
|
|
|
def stratified_limit(rows: Sequence[ProtocolCandidate], limit: int | None, *, seed: int) -> list[ProtocolCandidate]:
|
|
if limit is None or limit >= len(rows):
|
|
return list(rows)
|
|
if limit < 0:
|
|
raise ValueError("--max-candidates must be non-negative")
|
|
rng = random.Random(seed)
|
|
buckets: dict[tuple[str, str], list[ProtocolCandidate]] = defaultdict(list)
|
|
for row in rows:
|
|
buckets[(row.session, row.candidate_source)].append(row)
|
|
for bucket in buckets.values():
|
|
rng.shuffle(bucket)
|
|
selected: list[ProtocolCandidate] = []
|
|
while len(selected) < limit and any(buckets.values()):
|
|
for key in sorted(buckets):
|
|
if buckets[key] and len(selected) < limit:
|
|
selected.append(buckets[key].pop())
|
|
return sorted(selected, key=lambda row: (row.session, row.anchor_dense_frame, row.candidate_source))
|
|
|
|
|
|
def _write_jsonl(path: Path, rows: Iterable[ProtocolCandidate]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("w", encoding="utf-8") as handle:
|
|
for row in rows:
|
|
handle.write(json.dumps(asdict(row), ensure_ascii=False) + "\n")
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--dataset-root", type=Path, required=True)
|
|
parser.add_argument("--output-dir", type=Path, required=True)
|
|
parser.add_argument("--sessions", nargs="+", default=list(ALL_SESSIONS))
|
|
parser.add_argument("--camera-wearer-id", type=int, default=CAMERA_WEARER_ID)
|
|
parser.add_argument("--context-seconds", type=float, default=3.0)
|
|
parser.add_argument("--relation-window-seconds", type=float, default=3.0)
|
|
parser.add_argument("--same-speaker-pause-max-seconds", type=float, default=2.0)
|
|
parser.add_argument("--other-handover-max-seconds", type=float, default=1.5)
|
|
parser.add_argument("--short-vocalization-max-seconds", type=float, default=1.0)
|
|
parser.add_argument("--continuation-min-seconds", type=float, default=0.4)
|
|
parser.add_argument("--int-offset-tolerance-seconds", type=float, default=0.05)
|
|
parser.add_argument(
|
|
"--wearer-resume-merge-max-seconds", type=float,
|
|
help=argparse.SUPPRESS,
|
|
) # Deprecated compatibility option; raw wearer boundaries are never merged.
|
|
parser.add_argument("--candidate-sources", nargs="+")
|
|
parser.add_argument("--max-candidates", type=int)
|
|
parser.add_argument("--selection-seed", type=int, default=2027)
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
unknown = sorted(set(args.sessions) - set(ALL_SESSIONS))
|
|
if unknown:
|
|
raise ValueError(f"Unknown EasyCom sessions: {unknown}")
|
|
config = CandidateConfig(
|
|
context_seconds=args.context_seconds,
|
|
relation_window_seconds=args.relation_window_seconds,
|
|
same_speaker_pause_max_seconds=args.same_speaker_pause_max_seconds,
|
|
other_handover_max_seconds=args.other_handover_max_seconds,
|
|
short_vocalization_max_seconds=args.short_vocalization_max_seconds,
|
|
continuation_min_seconds=args.continuation_min_seconds,
|
|
int_offset_tolerance_seconds=args.int_offset_tolerance_seconds,
|
|
)
|
|
dataset_root, output_dir = args.dataset_root.expanduser().resolve(), args.output_dir.expanduser().resolve()
|
|
all_rows: list[ProtocolCandidate] = []
|
|
session_timelines: dict[str, list[dict[str, Any]]] = {}
|
|
rules = Rules()
|
|
for session in args.sessions:
|
|
clips = build_clips(dataset_root, session, rules)
|
|
utterances = load_utterances(session, clips)
|
|
session_timelines[session] = [{
|
|
"uid": row.uid,
|
|
"participant_id": row.participant_id,
|
|
"participant_role": "wearer" if row.participant_id == args.camera_wearer_id else "other",
|
|
"start_seconds": row.dense_start_seconds,
|
|
"end_seconds": row.dense_end_seconds,
|
|
} for row in utterances]
|
|
rows = derive_protocol_candidates(utterances, clips, session=session,
|
|
wearer_id=args.camera_wearer_id, config=config)
|
|
all_rows.extend(rows)
|
|
print(f"{session}: candidates={len(rows)} sources={dict(sorted(Counter(row.candidate_source for row in rows).items()))}")
|
|
sources = None if args.candidate_sources is None else set(args.candidate_sources)
|
|
controls = [row for row in all_rows if not requires_semantic_adjudication(asdict(row))]
|
|
primary = [row for row in all_rows if requires_semantic_adjudication(asdict(row))]
|
|
eligible = [row for row in primary if sources is None or row.candidate_source in sources]
|
|
selected = stratified_limit(eligible, args.max_candidates, seed=args.selection_seed)
|
|
output_path = output_dir / "easycom_eot_int_candidates.jsonl"
|
|
_write_jsonl(output_path, selected)
|
|
controls_path = output_dir / "easycom_eot_int_controls.jsonl"
|
|
_write_jsonl(controls_path, controls)
|
|
counts = Counter((row.split, row.candidate_source, row.rule_candidate_type) for row in selected)
|
|
summary = {
|
|
"schema_version": SCHEMA_VERSION, "protocol_version": PROTOCOL_VERSION, "dataset": "EasyCom",
|
|
"semantic_warning": "Rules create candidates only; AI labels need human review and never modify dense activity ground truth.",
|
|
"sessions": list(args.sessions), "camera_wearer_id": args.camera_wearer_id,
|
|
"candidate_config": asdict(config), "selection_seed": args.selection_seed,
|
|
"max_candidates": args.max_candidates,
|
|
"candidate_sources": None if sources is None else sorted(sources),
|
|
"num_candidates_before_filter": len(primary), "num_candidates_before_limit": len(eligible),
|
|
"num_candidates_written": len(selected),
|
|
"num_controls_written": len(controls),
|
|
"controls_file": controls_path.name,
|
|
"counts": [{"split": key[0], "candidate_source": key[1],
|
|
"rule_candidate_type": key[2], "count": value}
|
|
for key, value in sorted(counts.items())],
|
|
}
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
(output_dir / "summary.json").write_text(
|
|
json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
(output_dir / "easycom_session_timelines.json").write_text(
|
|
json.dumps({"schema_version": 1, "sessions": session_timelines}, ensure_ascii=False) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
print(f"Saved {len(selected)} wearer-centric review candidates to {output_path}")
|
|
print(f"Saved {len(controls)} deterministic boundary controls to {controls_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|