import unittest from pipecat.frames.frames import LLMMessagesAppendFrame, UserImageRawFrame from services.pipecat.pipeline import _multimodal_user_input_frame from services.pipecat.processors import UserInputError, parse_user_input class UserInputParserTests(unittest.TestCase): def test_parses_text_and_current_camera_frame(self): value = parse_user_input( { "type": "user-input", "schema_version": 1, "input_id": "input_1", "parts": [ {"type": "input_text", "text": "帮我看看"}, { "type": "input_image", "source": { "type": "camera_frame", "frame": "current", }, }, ], "options": { "run_immediately": True, "interrupt": True, }, } ) self.assertIsNotNone(value) self.assertEqual(value.text, "帮我看看") self.assertTrue(value.has_camera_frame) self.assertEqual(value.transcript_text, "") def test_rejects_legacy_and_unsupported_image_sources(self): self.assertIsNone(parse_user_input({"type": "user-text", "text": "旧协议"})) with self.assertRaisesRegex(UserInputError, "当前摄像头"): parse_user_input( { "type": "user-input", "schema_version": 1, "input_id": "input_2", "parts": [ { "type": "input_image", "source": { "type": "remote_url", "url": "https://example.com/image.jpg", }, } ], } ) def test_parses_uploaded_image_asset(self): value = parse_user_input( { "type": "user-input", "schema_version": 1, "input_id": "input_upload", "parts": [ {"type": "input_text", "text": "这是什么?"}, { "type": "input_image", "source": { "type": "uploaded_asset", "asset_token": "signed-token", }, }, ], } ) self.assertIsNotNone(value) self.assertTrue(value.has_image) self.assertFalse(value.has_camera_frame) self.assertEqual(value.image_asset_token, "signed-token") self.assertEqual(value.prompt_text, "这是什么?") self.assertEqual(value.transcript_text, "") def test_image_only_input_has_no_synthetic_chat_text(self): value = parse_user_input( { "type": "user-input", "schema_version": 1, "input_id": "input_photo", "parts": [ { "type": "input_image", "source": {"type": "camera_frame", "frame": "current"}, } ], } ) self.assertIsNotNone(value) self.assertEqual(value.transcript_text, "") self.assertEqual(value.prompt_text, "请根据用户刚提交的图片进行回复。") def test_native_image_uses_the_standard_multimodal_user_turn_path(self): image = UserImageRawFrame( image=bytes([220, 40, 40] * 16 * 16), size=(16, 16), format="RGB", ) frame = _multimodal_user_input_frame( image, "请根据用户刚提交的图片进行回复。", ) self.assertIsInstance(frame, LLMMessagesAppendFrame) self.assertTrue(frame.run_llm) self.assertEqual(frame.messages[0]["role"], "user") content = frame.messages[0]["content"] self.assertEqual( content[0], {"type": "text", "text": "请根据用户刚提交的图片进行回复。"}, ) self.assertEqual(content[1]["type"], "image_url") self.assertTrue( content[1]["image_url"]["url"].startswith( "data:image/jpeg;base64," ) ) if __name__ == "__main__": unittest.main()