ai-video-fullstack/backend/tests/test_user_input.py

import unittest
from unittest.mock import AsyncMock

from pipecat.frames.frames import (
    InputTransportMessageFrame,
    LLMMessagesAppendFrame,
    UserImageRawFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from services.pipecat.call_lifecycle import CallEndCoordinator
from services.pipecat.pipeline import _multimodal_user_input_frame
from services.pipecat.processors import (
    UserInputError,
    UserInputProcessor,
    parse_user_input,
)


class UserInputParserTests(unittest.TestCase):
    def test_parses_text_and_current_camera_frame(self):
        value = parse_user_input(
            {
                "type": "user-input",
                "schema_version": 1,
                "input_id": "input_1",
                "parts": [
                    {"type": "input_text", "text": "帮我看看"},
                    {
                        "type": "input_image",
                        "source": {
                            "type": "camera_frame",
                            "frame": "current",
                        },
                    },
                ],
                "options": {
                    "run_immediately": True,
                    "interrupt": True,
                },
            }
        )

        self.assertIsNotNone(value)
        self.assertEqual(value.text, "帮我看看")
        self.assertTrue(value.has_camera_frame)
        self.assertEqual(value.transcript_text, "")

    def test_rejects_legacy_and_unsupported_image_sources(self):
        self.assertIsNone(parse_user_input({"type": "user-text", "text": "旧协议"}))
        with self.assertRaisesRegex(UserInputError, "当前摄像头"):
            parse_user_input(
                {
                    "type": "user-input",
                    "schema_version": 1,
                    "input_id": "input_2",
                    "parts": [
                        {
                            "type": "input_image",
                            "source": {
                                "type": "remote_url",
                                "url": "https://example.com/image.jpg",
                            },
                        }
                    ],
                }
            )

    def test_parses_uploaded_image_asset(self):
        value = parse_user_input(
            {
                "type": "user-input",
                "schema_version": 1,
                "input_id": "input_upload",
                "parts": [
                    {"type": "input_text", "text": "这是什么？"},
                    {
                        "type": "input_image",
                        "source": {
                            "type": "uploaded_asset",
                            "asset_token": "signed-token",
                        },
                    },
                ],
            }
        )

        self.assertIsNotNone(value)
        self.assertTrue(value.has_image)
        self.assertFalse(value.has_camera_frame)
        self.assertEqual(value.image_asset_token, "signed-token")
        self.assertEqual(value.prompt_text, "这是什么？")
        self.assertEqual(value.transcript_text, "")

    def test_image_only_input_has_no_synthetic_chat_text(self):
        value = parse_user_input(
            {
                "type": "user-input",
                "schema_version": 1,
                "input_id": "input_photo",
                "parts": [
                    {
                        "type": "input_image",
                        "source": {"type": "camera_frame", "frame": "current"},
                    }
                ],
            }
        )

        self.assertIsNotNone(value)
        self.assertEqual(value.transcript_text, "")
        self.assertEqual(value.prompt_text, "请根据用户刚提交的图片进行回复。")

    def test_native_image_uses_the_standard_multimodal_user_turn_path(self):
        image = UserImageRawFrame(
            image=bytes([220, 40, 40] * 16 * 16),
            size=(16, 16),
            format="RGB",
        )

        frame = _multimodal_user_input_frame(
            image,
            "请根据用户刚提交的图片进行回复。",
        )

        self.assertIsInstance(frame, LLMMessagesAppendFrame)
        self.assertTrue(frame.run_llm)
        self.assertEqual(frame.messages[0]["role"], "user")
        content = frame.messages[0]["content"]
        self.assertEqual(
            content[0],
            {"type": "text", "text": "请根据用户刚提交的图片进行回复。"},
        )
        self.assertEqual(content[1]["type"], "image_url")
        self.assertTrue(
            content[1]["image_url"]["url"].startswith(
                "data:image/jpeg;base64,"
            )
        )


class UserInputProcessorTests(unittest.IsolatedAsyncioTestCase):
    async def test_interrupting_text_cancels_pending_end_before_mute_check(self):
        async def queue_end(_reason: str) -> None:
            self.fail("撤销后不应挂断")

        call_end = CallEndCoordinator(queue_end)
        call_end.begin("user_goodbye", interruptible=True)
        processor = UserInputProcessor(
            should_ignore_input=lambda: not call_end.accepts_input,
            cancel_pending_end=call_end.cancel_pending_end,
        )
        processor._call_event_handler = AsyncMock()
        processor.broadcast_interruption = AsyncMock()

        await processor.process_frame(
            InputTransportMessageFrame(
                message={
                    "type": "user-input",
                    "schema_version": 1,
                    "input_id": "input_interrupt_end",
                    "parts": [{"type": "input_text", "text": "等等"}],
                    "options": {
                        "run_immediately": True,
                        "interrupt": True,
                    },
                }
            ),
            FrameDirection.DOWNSTREAM,
        )

        self.assertFalse(call_end.ending)
        processor._call_event_handler.assert_awaited_once()
        processor.broadcast_interruption.assert_awaited_once()


if __name__ == "__main__":
    unittest.main()