Initial commit: Omni-only turn-taking experiment scaffold.

Add EasyCom prep, Qwen2.5-Omni feature extraction, streaming turn model prototype, and docs for the ICASSP 2027 ego-speak study.

Co-authored-by: Cursor <cursoragent@cursor.com>

smoke_test.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""Minimal single-GPU smoke test for a local Qwen2.5-Omni checkpoint.
+
+Examples:
+  python smoke_test.py --model /data1/yfzx/wangx/Models/Qwen2.5-Omni-3B
+  python smoke_test.py --model /data1/yfzx/wangx/Models/Qwen2.5-Omni-3B \
+      --video /data1/example.mp4
+
+Qwen2.5-Omni generation currently is more reliable on a single GPU for this
+use case.  In particular, do not replace the explicit device map below with
+``device_map=\"auto\"``: it can place cache tensors on different GPUs.
+"""
+
+import argparse
+import time
+
+import torch
+from qwen_omni_utils import process_mm_info
+from transformers import Qwen2_5OmniForConditionalGeneration, Qwen2_5OmniProcessor
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", required=True, help="Local model directory or HF model ID")
+    parser.add_argument("--video", help="Optional local .mp4 file for an audio-video test")
+    parser.add_argument("--device", default="cuda:0", help="One CUDA device, e.g. cuda:0")
+    parser.add_argument("--max-new-tokens", type=int, default=64)
+    parser.add_argument(
+        "--video-fps",
+        type=float,
+        default=1.0,
+        help="Frames per second sampled by the Omni video processor (default: 1.0)",
+    )
+    parser.add_argument(
+        "--video-max-pixels",
+        type=int,
+        default=256 * 28 * 28,
+        help="Maximum pixels per sampled video frame (default: 200704)",
+    )
+    parser.add_argument(
+        "--flash-attn2",
+        action="store_true",
+        help="Use FlashAttention-2; requires the flash_attn package.",
+    )
+    return parser.parse_args()
+
+
+def move_tensors_to_device(batch: dict, device: str) -> dict:
+    """Move processor tensors without accidentally converting integer IDs."""
+    return {
+        name: value.to(device) if torch.is_tensor(value) else value
+        for name, value in batch.items()
+    }
+
+
+def main() -> None:
+    args = parse_args()
+    if not torch.cuda.is_available():
+        raise RuntimeError("A CUDA GPU is required for this smoke test.")
+
+    print(f"GPU: {torch.cuda.get_device_name(args.device)}")
+    print(f"CUDA: {torch.version.cuda}")
+    print(f"PyTorch: {torch.__version__}")
+    print(f"Loading: {args.model}")
+
+    started = time.perf_counter()
+    # A 3B model fits comfortably on one L20 (48 GB).  Keep all weights and
+    # generation cache on the same device; device_map='auto' may split them.
+    model_load_kwargs = dict(
+        torch_dtype=torch.bfloat16,
+        device_map={"": args.device},
+    )
+    if args.flash_attn2:
+        model_load_kwargs["attn_implementation"] = "flash_attention_2"
+
+    model = Qwen2_5OmniForConditionalGeneration.from_pretrained(
+        args.model,
+        **model_load_kwargs,
+    )
+    model.disable_talker()  # We need Thinker text states only; saves VRAM.
+    model.eval()
+    processor = Qwen2_5OmniProcessor.from_pretrained(args.model)
+    print(f"Model loaded in {time.perf_counter() - started:.1f}s")
+
+    if args.video:
+        content = [
+            {
+                "type": "video",
+                "video": args.video,
+                # These are consumed by qwen_omni_utils while sampling the
+                # video.  They prevent quadratic vision attention from
+                # allocating tens of GB for a short, high-resolution clip.
+                "fps": args.video_fps,
+                "max_pixels": args.video_max_pixels,
+            },
+            {
+                "type": "text",
+                "text": "Briefly describe the people, speech, and interaction in this video.",
+            },
+        ]
+        print(
+            f"Video: {args.video} "
+            f"(sample_fps={args.video_fps}, max_pixels={args.video_max_pixels})"
+        )
+    else:
+        content = [
+            {"type": "text", "text": "Reply with exactly: Qwen Omni smoke test passed."},
+        ]
+
+    messages = [{"role": "user", "content": content}]
+    prompt = processor.apply_chat_template(
+        messages, tokenize=False, add_generation_prompt=True
+    )
+    audios, images, videos = process_mm_info(
+        messages, use_audio_in_video=bool(args.video)
+    )
+    inputs = processor(
+        text=prompt,
+        audio=audios,
+        images=images,
+        videos=videos,
+        return_tensors="pt",
+        padding=True,
+        use_audio_in_video=bool(args.video),
+    )
+    inputs = move_tensors_to_device(inputs, args.device)
+
+    started = time.perf_counter()
+    with torch.inference_mode():
+        output_ids = model.generate(
+            **inputs,
+            max_new_tokens=args.max_new_tokens,
+            do_sample=False,
+            return_audio=False,
+        )
+
+    # Remove the prompt tokens before decoding the generated completion.
+    generated_ids = output_ids[:, inputs["input_ids"].shape[1] :]
+    answer = processor.batch_decode(
+        generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
+    )[0]
+    print(f"Generation completed in {time.perf_counter() - started:.1f}s")
+    print(f"\nModel response:\n{answer.strip()}")
+
+
+if __name__ == "__main__":
+    main()