572 lines
21 KiB
Python
572 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
"""Prepare manuscript-aligned EasyCom EOT/INT regions for human review.
|
|
|
|
The released EasyCom annotations provide participant-resolved speech intervals,
|
|
not semantic EOT/INT ground truth. This script therefore creates auditable
|
|
candidate regions from the original 20 Hz timeline. It keeps the two wearer-
|
|
relative directions distinct:
|
|
|
|
* EOT-response: an aggregate non-wearer offset while the wearer is silent;
|
|
* INT-yield: an aggregate non-wearer onset while the wearer is speaking.
|
|
|
|
Observed wearer transitions form timing-positive candidates. Competing other
|
|
transitions and fully observed no-transition windows form hard-negative
|
|
candidates. Source-boundary cases are retained as exclusions. Every output
|
|
row remains an automatic candidate until it is independently human reviewed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import Counter, defaultdict
|
|
from dataclasses import asdict, dataclass
|
|
import json
|
|
from pathlib import Path
|
|
import random
|
|
from typing import Any, Iterable, Sequence
|
|
|
|
from derive_easycom_interaction_events import (
|
|
CAMERA_WEARER_ID,
|
|
SOURCE_FPS,
|
|
Clip,
|
|
Rules,
|
|
Utterance,
|
|
build_clips,
|
|
is_lexical_feedback,
|
|
is_pure_nonverbal,
|
|
load_utterances,
|
|
)
|
|
from easycom_manifest import ALL_SESSIONS, TEST_SESSIONS
|
|
|
|
|
|
SCHEMA_VERSION = 1
|
|
PROTOCOL_VERSION = "easycom-manuscript-eot-int-v1"
|
|
DEFAULT_WINDOW_SECONDS = 2.0
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ProtocolCandidate:
|
|
schema_version: int
|
|
protocol_version: str
|
|
uid: str
|
|
session: str
|
|
split: str
|
|
task: str
|
|
event_type: str
|
|
automatic_protocol_label: str
|
|
automatic_hard_negative_type: str
|
|
automatic_confidence: str
|
|
anchor_type: str
|
|
anchor_dense_frame: int
|
|
anchor_dense_seconds: float
|
|
behavior_anchor_type: str | None
|
|
behavior_dense_frame: int | None
|
|
behavior_seconds: float | None
|
|
region_start_seconds: float
|
|
region_end_seconds: float
|
|
region_end_reason: str
|
|
previous_other_participant_ids: tuple[int, ...]
|
|
incoming_other_participant_ids: tuple[int, ...]
|
|
clip_stem: str
|
|
source_start_frame: int
|
|
context: tuple[dict[str, Any], ...]
|
|
automatic_reason: str
|
|
|
|
|
|
def _fill_state(
|
|
rows: Sequence[Utterance],
|
|
*,
|
|
segment_start: int,
|
|
segment_end: int,
|
|
wearer_id: int,
|
|
) -> tuple[bytearray, bytearray]:
|
|
length = segment_end - segment_start
|
|
wearer = bytearray(length)
|
|
others = bytearray(length)
|
|
for row in rows:
|
|
left = max(segment_start, row.dense_start_frame) - segment_start
|
|
right = min(segment_end, row.dense_end_frame_exclusive) - segment_start
|
|
if right <= left:
|
|
continue
|
|
target = wearer if row.participant_id == wearer_id else others
|
|
target[left:right] = b"\x01" * (right - left)
|
|
return wearer, others
|
|
|
|
|
|
def _first_transition(
|
|
state: bytearray,
|
|
*,
|
|
start: int,
|
|
stop: int,
|
|
rising: bool,
|
|
) -> int | None:
|
|
for index in range(max(1, start), min(stop, len(state))):
|
|
before = bool(state[index - 1])
|
|
after = bool(state[index])
|
|
if (not before and after) if rising else (before and not after):
|
|
return index
|
|
return None
|
|
|
|
|
|
def _active_other_ids(
|
|
rows: Sequence[Utterance],
|
|
dense_frame: int,
|
|
*,
|
|
wearer_id: int,
|
|
) -> tuple[int, ...]:
|
|
return tuple(sorted({
|
|
row.participant_id
|
|
for row in rows
|
|
if row.participant_id != wearer_id
|
|
and row.dense_start_frame <= dense_frame < row.dense_end_frame_exclusive
|
|
}))
|
|
|
|
|
|
def _clip_for_frame(clips: Sequence[Clip], dense_frame: int) -> Clip | None:
|
|
return next(
|
|
(
|
|
clip
|
|
for clip in clips
|
|
if clip.dense_start_frame <= dense_frame < clip.dense_start_frame + clip.num_frames
|
|
),
|
|
None,
|
|
)
|
|
|
|
|
|
def _context_rows(
|
|
rows: Sequence[Utterance],
|
|
*,
|
|
anchor_frame: int,
|
|
context_frames: int,
|
|
wearer_id: int,
|
|
) -> tuple[dict[str, Any], ...]:
|
|
left = anchor_frame - context_frames
|
|
right = anchor_frame + context_frames
|
|
selected = []
|
|
for row in rows:
|
|
if row.dense_end_frame_exclusive <= left or row.dense_start_frame >= right:
|
|
continue
|
|
selected.append(
|
|
{
|
|
"uid": row.uid,
|
|
"participant_id": row.participant_id,
|
|
"participant_role": "wearer" if row.participant_id == wearer_id else "other",
|
|
"start_seconds": row.dense_start_seconds,
|
|
"end_seconds": row.dense_end_seconds,
|
|
"transcription": row.transcription,
|
|
"target_of_speech": list(row.target_of_speech),
|
|
}
|
|
)
|
|
return tuple(selected)
|
|
|
|
|
|
def _incoming_is_backchannel(
|
|
rows: Sequence[Utterance],
|
|
*,
|
|
dense_frame: int,
|
|
wearer_id: int,
|
|
max_seconds: float,
|
|
) -> bool:
|
|
incoming = [
|
|
row
|
|
for row in rows
|
|
if row.participant_id != wearer_id
|
|
and row.dense_start_frame <= dense_frame < row.dense_end_frame_exclusive
|
|
and row.dense_start_frame >= dense_frame - 1
|
|
]
|
|
return bool(incoming) and all(row.duration_seconds <= max_seconds for row in incoming) and any(
|
|
is_lexical_feedback(row.transcription) or is_pure_nonverbal(row.transcription)
|
|
for row in incoming
|
|
)
|
|
|
|
|
|
def _candidate(
|
|
*,
|
|
session: str,
|
|
task: str,
|
|
automatic_protocol_label: str,
|
|
automatic_hard_negative_type: str,
|
|
anchor_frame: int,
|
|
behavior_frame: int | None,
|
|
region_end_frame: int,
|
|
region_end_reason: str,
|
|
previous_ids: tuple[int, ...],
|
|
incoming_ids: tuple[int, ...],
|
|
clips: Sequence[Clip],
|
|
rows: Sequence[Utterance],
|
|
wearer_id: int,
|
|
context_frames: int,
|
|
reason: str,
|
|
) -> ProtocolCandidate:
|
|
clip = _clip_for_frame(clips, anchor_frame)
|
|
if clip is None:
|
|
raise ValueError(f"No source clip for {session} dense frame {anchor_frame}")
|
|
event_type = "eot_response_candidate" if task == "eot_response" else "int_yield_candidate"
|
|
anchor_type = "other_offset" if task == "eot_response" else "other_onset"
|
|
behavior_type = None
|
|
if behavior_frame is not None:
|
|
behavior_type = "wearer_onset" if task == "eot_response" else "wearer_offset"
|
|
return ProtocolCandidate(
|
|
schema_version=SCHEMA_VERSION,
|
|
protocol_version=PROTOCOL_VERSION,
|
|
uid=f"{session}/{task}/{anchor_frame:08d}",
|
|
session=session,
|
|
split="test" if session in TEST_SESSIONS else "train",
|
|
task=task,
|
|
event_type=event_type,
|
|
automatic_protocol_label=automatic_protocol_label,
|
|
automatic_hard_negative_type=automatic_hard_negative_type,
|
|
automatic_confidence="timing_candidate",
|
|
anchor_type=anchor_type,
|
|
anchor_dense_frame=anchor_frame,
|
|
anchor_dense_seconds=anchor_frame / SOURCE_FPS,
|
|
behavior_anchor_type=behavior_type,
|
|
behavior_dense_frame=behavior_frame,
|
|
behavior_seconds=None if behavior_frame is None else behavior_frame / SOURCE_FPS,
|
|
region_start_seconds=anchor_frame / SOURCE_FPS,
|
|
region_end_seconds=region_end_frame / SOURCE_FPS,
|
|
region_end_reason=region_end_reason,
|
|
previous_other_participant_ids=previous_ids,
|
|
incoming_other_participant_ids=incoming_ids,
|
|
clip_stem=clip.stem,
|
|
source_start_frame=anchor_frame - clip.dense_start_frame + 1,
|
|
context=_context_rows(
|
|
rows,
|
|
anchor_frame=anchor_frame,
|
|
context_frames=context_frames,
|
|
wearer_id=wearer_id,
|
|
),
|
|
automatic_reason=reason,
|
|
)
|
|
|
|
|
|
def derive_protocol_candidates(
|
|
utterances: Sequence[Utterance],
|
|
clips: Sequence[Clip],
|
|
*,
|
|
session: str,
|
|
wearer_id: int = CAMERA_WEARER_ID,
|
|
window_seconds: float = DEFAULT_WINDOW_SECONDS,
|
|
context_seconds: float = 3.0,
|
|
) -> list[ProtocolCandidate]:
|
|
"""Construct timing candidates with manuscript-aligned directionality."""
|
|
|
|
if window_seconds <= 0 or context_seconds < 0:
|
|
raise ValueError("window_seconds must be positive and context_seconds non-negative")
|
|
window_frames = max(1, round(window_seconds * SOURCE_FPS))
|
|
context_frames = max(0, round(context_seconds * SOURCE_FPS))
|
|
rows_by_segment: dict[int, list[Utterance]] = defaultdict(list)
|
|
clips_by_segment: dict[int, list[Clip]] = defaultdict(list)
|
|
for row in utterances:
|
|
rows_by_segment[row.segment_id].append(row)
|
|
for clip in clips:
|
|
clips_by_segment[clip.segment_id].append(clip)
|
|
|
|
candidates: list[ProtocolCandidate] = []
|
|
for segment_id, segment_clips in sorted(clips_by_segment.items()):
|
|
segment_clips.sort(key=lambda clip: clip.dense_start_frame)
|
|
segment_rows = sorted(
|
|
rows_by_segment.get(segment_id, []),
|
|
key=lambda row: (row.dense_start_frame, row.dense_end_frame_exclusive, row.uid),
|
|
)
|
|
segment_start = segment_clips[0].dense_start_frame
|
|
segment_end = segment_clips[-1].dense_start_frame + segment_clips[-1].num_frames
|
|
wearer, others = _fill_state(
|
|
segment_rows,
|
|
segment_start=segment_start,
|
|
segment_end=segment_end,
|
|
wearer_id=wearer_id,
|
|
)
|
|
|
|
for index in range(1, len(others)):
|
|
dense_frame = segment_start + index
|
|
if others[index - 1] and not others[index] and not wearer[index - 1]:
|
|
competing = _first_transition(
|
|
others,
|
|
start=index + 1,
|
|
stop=min(len(others), index + window_frames + 1),
|
|
rising=True,
|
|
)
|
|
nominal_end = index + window_frames
|
|
observable_end = min(nominal_end, len(others))
|
|
effective_end = min(observable_end, competing if competing is not None else observable_end)
|
|
outcome = _first_transition(
|
|
wearer,
|
|
start=index,
|
|
stop=effective_end,
|
|
rising=True,
|
|
)
|
|
previous_ids = _active_other_ids(
|
|
segment_rows, dense_frame - 1, wearer_id=wearer_id
|
|
)
|
|
incoming_ids = (
|
|
()
|
|
if competing is None
|
|
else _active_other_ids(
|
|
segment_rows, segment_start + competing, wearer_id=wearer_id
|
|
)
|
|
)
|
|
if outcome is not None:
|
|
label = "positive"
|
|
subtype = "not_applicable"
|
|
end_index = outcome
|
|
end_reason = "wearer_onset"
|
|
reason = "wearer onset observed after other offset and before competing other onset"
|
|
elif competing is not None:
|
|
label = "hard_negative"
|
|
subtype = (
|
|
"eot_midturn_pause"
|
|
if set(previous_ids) & set(incoming_ids)
|
|
else "eot_other_to_other"
|
|
)
|
|
end_index = competing
|
|
end_reason = "competing_other_onset"
|
|
reason = "wearer remains silent until another non-wearer onset"
|
|
elif nominal_end <= len(others):
|
|
label = "hard_negative"
|
|
subtype = "eot_no_response"
|
|
end_index = nominal_end
|
|
end_reason = "full_window"
|
|
reason = "wearer remains silent for the full observable response window"
|
|
else:
|
|
label = "exclude"
|
|
subtype = "other"
|
|
end_index = len(others)
|
|
end_reason = "right_censored"
|
|
reason = "recording segment ends before a wearer outcome or full window"
|
|
candidates.append(_candidate(
|
|
session=session,
|
|
task="eot_response",
|
|
automatic_protocol_label=label,
|
|
automatic_hard_negative_type=subtype,
|
|
anchor_frame=dense_frame,
|
|
behavior_frame=None if outcome is None else segment_start + outcome,
|
|
region_end_frame=segment_start + end_index,
|
|
region_end_reason=end_reason,
|
|
previous_ids=previous_ids,
|
|
incoming_ids=incoming_ids,
|
|
clips=segment_clips,
|
|
rows=segment_rows,
|
|
wearer_id=wearer_id,
|
|
context_frames=context_frames,
|
|
reason=reason,
|
|
))
|
|
|
|
if not others[index - 1] and others[index] and wearer[index - 1]:
|
|
competing = _first_transition(
|
|
others,
|
|
start=index + 1,
|
|
stop=min(len(others), index + window_frames + 1),
|
|
rising=False,
|
|
)
|
|
nominal_end = index + window_frames
|
|
observable_end = min(nominal_end, len(others))
|
|
effective_end = min(observable_end, competing if competing is not None else observable_end)
|
|
outcome = _first_transition(
|
|
wearer,
|
|
start=index,
|
|
stop=effective_end,
|
|
rising=False,
|
|
)
|
|
previous_ids = _active_other_ids(
|
|
segment_rows, dense_frame - 1, wearer_id=wearer_id
|
|
)
|
|
incoming_ids = _active_other_ids(
|
|
segment_rows, dense_frame, wearer_id=wearer_id
|
|
)
|
|
if outcome is not None:
|
|
label = "positive"
|
|
subtype = "not_applicable"
|
|
end_index = outcome
|
|
end_reason = "wearer_offset"
|
|
reason = "wearer offset observed after other onset and before competing other offset"
|
|
elif competing is not None:
|
|
label = "hard_negative"
|
|
subtype = (
|
|
"int_backchannel"
|
|
if _incoming_is_backchannel(
|
|
segment_rows,
|
|
dense_frame=dense_frame,
|
|
wearer_id=wearer_id,
|
|
max_seconds=1.0,
|
|
)
|
|
else "int_non_takeover_overlap"
|
|
)
|
|
end_index = competing
|
|
end_reason = "competing_other_offset"
|
|
reason = "wearer retains the floor until the incoming other activity ends"
|
|
elif nominal_end <= len(others):
|
|
label = "hard_negative"
|
|
subtype = "int_retain_floor"
|
|
end_index = nominal_end
|
|
end_reason = "full_window"
|
|
reason = "wearer remains active through the full observable yield window"
|
|
else:
|
|
label = "exclude"
|
|
subtype = "other"
|
|
end_index = len(others)
|
|
end_reason = "right_censored"
|
|
reason = "recording segment ends before a wearer outcome or full window"
|
|
candidates.append(_candidate(
|
|
session=session,
|
|
task="int_yield",
|
|
automatic_protocol_label=label,
|
|
automatic_hard_negative_type=subtype,
|
|
anchor_frame=dense_frame,
|
|
behavior_frame=None if outcome is None else segment_start + outcome,
|
|
region_end_frame=segment_start + end_index,
|
|
region_end_reason=end_reason,
|
|
previous_ids=previous_ids,
|
|
incoming_ids=incoming_ids,
|
|
clips=segment_clips,
|
|
rows=segment_rows,
|
|
wearer_id=wearer_id,
|
|
context_frames=context_frames,
|
|
reason=reason,
|
|
))
|
|
|
|
return sorted(candidates, key=lambda row: (row.anchor_dense_frame, row.task, row.uid))
|
|
|
|
|
|
def stratified_limit(
|
|
rows: Sequence[ProtocolCandidate],
|
|
limit: int | None,
|
|
*,
|
|
seed: int,
|
|
) -> list[ProtocolCandidate]:
|
|
if limit is None or limit >= len(rows):
|
|
return list(rows)
|
|
if limit < 0:
|
|
raise ValueError("--max-candidates must be non-negative")
|
|
rng = random.Random(seed)
|
|
buckets: dict[tuple[str, str, str], list[ProtocolCandidate]] = defaultdict(list)
|
|
for row in rows:
|
|
buckets[(row.task, row.automatic_protocol_label, row.automatic_hard_negative_type)].append(row)
|
|
keys = sorted(buckets)
|
|
for key in keys:
|
|
rng.shuffle(buckets[key])
|
|
rng.shuffle(keys)
|
|
selected: list[ProtocolCandidate] = []
|
|
while len(selected) < limit:
|
|
progressed = False
|
|
for key in keys:
|
|
if buckets[key]:
|
|
selected.append(buckets[key].pop())
|
|
progressed = True
|
|
if len(selected) == limit:
|
|
break
|
|
if not progressed:
|
|
break
|
|
return sorted(selected, key=lambda row: (row.session, row.anchor_dense_frame, row.task))
|
|
|
|
|
|
def _write_jsonl(path: Path, rows: Iterable[ProtocolCandidate]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("w", encoding="utf-8") as handle:
|
|
for row in rows:
|
|
handle.write(json.dumps(asdict(row), ensure_ascii=False) + "\n")
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--dataset-root", type=Path, required=True)
|
|
parser.add_argument("--output-dir", type=Path, required=True)
|
|
parser.add_argument("--sessions", nargs="+", default=list(ALL_SESSIONS))
|
|
parser.add_argument("--camera-wearer-id", type=int, default=CAMERA_WEARER_ID)
|
|
parser.add_argument("--window-seconds", type=float, default=DEFAULT_WINDOW_SECONDS)
|
|
parser.add_argument("--context-seconds", type=float, default=3.0)
|
|
parser.add_argument(
|
|
"--tasks",
|
|
nargs="+",
|
|
choices=("eot_response", "int_yield"),
|
|
default=("eot_response", "int_yield"),
|
|
)
|
|
parser.add_argument(
|
|
"--automatic-labels",
|
|
nargs="+",
|
|
choices=("positive", "hard_negative", "exclude"),
|
|
default=("positive", "hard_negative", "exclude"),
|
|
)
|
|
parser.add_argument("--max-candidates", type=int)
|
|
parser.add_argument("--selection-seed", type=int, default=2027)
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
unknown = sorted(set(args.sessions) - set(ALL_SESSIONS))
|
|
if unknown:
|
|
raise ValueError(f"Unknown EasyCom sessions: {unknown}")
|
|
dataset_root = args.dataset_root.expanduser().resolve()
|
|
output_dir = args.output_dir.expanduser().resolve()
|
|
all_rows: list[ProtocolCandidate] = []
|
|
rules = Rules()
|
|
for session in args.sessions:
|
|
clips = build_clips(dataset_root, session, rules)
|
|
utterances = load_utterances(session, clips)
|
|
rows = derive_protocol_candidates(
|
|
utterances,
|
|
clips,
|
|
session=session,
|
|
wearer_id=args.camera_wearer_id,
|
|
window_seconds=args.window_seconds,
|
|
context_seconds=args.context_seconds,
|
|
)
|
|
all_rows.extend(rows)
|
|
counts = Counter((row.task, row.automatic_protocol_label) for row in rows)
|
|
print(f"{session}: candidates={len(rows)} counts={dict(sorted(counts.items()))}")
|
|
|
|
eligible = [
|
|
row
|
|
for row in all_rows
|
|
if row.task in set(args.tasks)
|
|
and row.automatic_protocol_label in set(args.automatic_labels)
|
|
]
|
|
selected = stratified_limit(eligible, args.max_candidates, seed=args.selection_seed)
|
|
output_path = output_dir / "easycom_eot_int_candidates.jsonl"
|
|
_write_jsonl(output_path, selected)
|
|
counts = Counter(
|
|
(row.split, row.task, row.automatic_protocol_label, row.automatic_hard_negative_type)
|
|
for row in selected
|
|
)
|
|
summary = {
|
|
"schema_version": SCHEMA_VERSION,
|
|
"protocol_version": PROTOCOL_VERSION,
|
|
"dataset": "EasyCom",
|
|
"semantic_warning": (
|
|
"Automatic labels are timing candidates for independent human audit; "
|
|
"they are not semantic EOT/INT ground truth."
|
|
),
|
|
"sessions": list(args.sessions),
|
|
"camera_wearer_id": args.camera_wearer_id,
|
|
"window_seconds": args.window_seconds,
|
|
"context_seconds": args.context_seconds,
|
|
"selection_seed": args.selection_seed,
|
|
"max_candidates": args.max_candidates,
|
|
"tasks": list(args.tasks),
|
|
"automatic_labels": list(args.automatic_labels),
|
|
"num_candidates_before_filter": len(all_rows),
|
|
"num_candidates_before_limit": len(eligible),
|
|
"num_candidates_written": len(selected),
|
|
"counts": [
|
|
{
|
|
"split": key[0],
|
|
"task": key[1],
|
|
"automatic_protocol_label": key[2],
|
|
"automatic_hard_negative_type": key[3],
|
|
"count": value,
|
|
}
|
|
for key, value in sorted(counts.items())
|
|
],
|
|
}
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
(output_dir / "summary.json").write_text(
|
|
json.dumps(summary, indent=2, ensure_ascii=False) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
print(f"Saved {len(selected)} manuscript-aligned review candidates to {output_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|