wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update llm assisted annotation

Browse Source
This commit is contained in:
Xin Wang
2026-09-14 23:04:21 +08:00
parent 068b7ab9e6
commit 06b5e6c385
12 changed files with 1348 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
test_adjudicate_easycom_eot_int_candidates.py
View File

@@ -8,11 +8,18 @@ import json
import tempfile
from adjudicate_easycom_eot_int_candidates import (
_transport_rows, build_user_prompt, read_candidates, validate_decisions,
PROMPT_VERSION, SYSTEM_PROMPT, _progress_line, _transport_rows, build_user_prompt, read_candidates,
requires_semantic_adjudication, validate_decisions,
)
class EOTINTAdjudicationTests(unittest.TestCase):
def test_progress_line_reports_items_percent_pass_and_elapsed_time(self) -> None:
line = _progress_line(25, 100, label="pass 1/2 · batch 4/13", elapsed_seconds=65)
self.assertIn("25/100 ( 25.0%)", line)
self.assertIn("pass 1/2 · batch 4/13", line)
self.assertTrue(line.endswith("01:05"))
def test_prompt_contains_task_and_context(self) -> None:
prompt = build_user_prompt(
[{"uid": "u1", "candidate_source": "WEARER_ONSET", "task": "eot_response",
@@ -24,6 +31,16 @@ class EOTINTAdjudicationTests(unittest.TestCase):
self.assertIn("wearer_duration", prompt)
self.assertIn("Yes.", prompt)
def test_system_prompt_treats_laughter_as_nonlexical_floor_feedback(self) -> None:
self.assertEqual(PROMPT_VERSION, "easycom-wearer-centric-eot-int-prelabel-v6")
self.assertIn("two immutable anchors", SYSTEM_PROMPT)
self.assertIn("t_w < t_o < t_y", SYSTEM_PROMPT)
self.assertIn("a transcription consisting of [L] denotes laughter-only vocalization", SYSTEM_PROMPT)
self.assertIn("EOT_WEARER_BACKCHANNEL", SYSTEM_PROMPT)
self.assertIn("INT_OTHER_BACKCHANNEL", SYSTEM_PROMPT)
self.assertIn("do not infer that a visible smile occurred", SYSTEM_PROMPT)
self.assertNotIn('"needs_human_review"', SYSTEM_PROMPT)
def test_decisions_require_exact_uid_and_vocabulary(self) -> None:
payload = {
"items": [
@@ -40,9 +57,21 @@ class EOTINTAdjudicationTests(unittest.TestCase):
}
decisions = validate_decisions(payload, ["u1"])
self.assertEqual(decisions[0]["subtype"], "INT_OTHER_BACKCHANNEL")
self.assertTrue(decisions[0]["needs_human_review"])
with self.assertRaisesRegex(ValueError, "mismatch"):
validate_decisions({"items": []}, ["u1"])
def test_model_needs_human_review_type_drift_is_ignored(self) -> None:
payload = {
"items": [{
"uid": "u1", "verdict": "UNSURE", "event_type": None,
"subtype": None, "confidence": 0.4, "reason": "Ambiguous context.",
"needs_human_review": "true",
}]
}
decision = validate_decisions(payload, ["u1"])[0]
self.assertTrue(decision["needs_human_review"])
def test_markdown_escaped_uid_is_normalized_but_still_unique(self) -> None:
item = {
"uid": "Session\\_4/wearer\\_onset/00000024",
@@ -79,6 +108,35 @@ class EOTINTAdjudicationTests(unittest.TestCase):
self.assertEqual([row["uid"] for row in transported], ["item-0000", "item-0001"])
self.assertEqual(mapping["item-0001"], rows[1]["uid"])
def test_old_nonoverlap_wearer_offset_is_not_sent_for_adjudication(self) -> None:
control = {
"candidate_source": "WEARER_OFFSET",
"temporal_evidence": {
"active_other_ids_before_offset": [],
"active_other_ids_after_offset": [],
},
}
crossing = {
"candidate_source": "WEARER_OFFSET",
"temporal_evidence": {
"active_other_ids_before_offset": [4],
"active_other_ids_after_offset": [4],
},
}
speaker_change = {
"candidate_source": "WEARER_OFFSET",
"temporal_evidence": {
"active_other_ids_before_offset": [3],
"active_other_ids_after_offset": [4],
},
}
self.assertFalse(requires_semantic_adjudication(control))
self.assertTrue(requires_semantic_adjudication(crossing))
self.assertFalse(requires_semantic_adjudication(speaker_change))
self.assertFalse(requires_semantic_adjudication({"candidate_source": "WEARER_OFFSET_CONTROL"}))
self.assertFalse(requires_semantic_adjudication({"candidate_source": "WEARER_ONSET_CONTROL"}))
self.assertFalse(requires_semantic_adjudication({"candidate_source": "WEARER_RESUME"}))
if __name__ == "__main__":
unittest.main()