wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: support uploaded images in debug voice preview

Browse Source 
Allow paste/drag temporary image assets so vision turns work without a live camera frame.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Xin Wang
2026-08-05 19:52:37 +08:00
parent 555c8f5fa6
commit e36ca308b8
16 changed files with 686 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/app.py
View File

@@ -26,6 +26,7 @@ from routes import (
auth,
conversations,
health,
input_assets,
knowledge_bases,
mcp_servers,
model_registry,
@@ -60,6 +61,7 @@ app.add_middleware(
app.include_router(health.router)
app.include_router(auth.router)
app.include_router(conversations.router)
app.include_router(input_assets.router)
app.include_router(assistants.router)
app.include_router(knowledge_bases.router)
app.include_router(mcp_servers.router)

2
backend/requirements.txt
View File

@@ -6,7 +6,7 @@ pipecat-ai[webrtc,websocket,silero,openai,mcp]==1.7.0
Pillow>=11.1.0,<13
# FastGPT 类型助手:本地 SDK(包 /api/v1/chat/completions 流式 + chatId 会话)
fastgpt-client @ file:///Users/wangx/Code/AI-VideoAssistant-Project/fastgpt-python-sdk
# fastgpt-client @ file:///Users/wangx/Code/AI-VideoAssistant-Project/fastgpt-python-sdk
# Dify 类型助手:异步流式 Runtime API SDK
dify-client-python==1.0.3

59
backend/routes/input_assets.py Normal file
View File

@@ -0,0 +1,59 @@
"""Authenticated upload endpoint for pending debug-chat image attachments."""
from __future__ import annotations
import asyncio
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
from pydantic import BaseModel
import settings
from services.auth import require_admin
from services.input_assets import discard_input_image, store_input_image
router = APIRouter(
prefix="/api/input-assets",
tags=["input-assets"],
dependencies=[Depends(require_admin)],
)
class InputImageAssetOut(BaseModel):
assetToken: str
width: int
height: int
sizeBytes: int
@router.post("/image", response_model=InputImageAssetOut)
async def upload_input_image(file: UploadFile = File(...)):
data = await file.read(settings.INPUT_IMAGE_MAX_BYTES + 1)
if len(data) > settings.INPUT_IMAGE_MAX_BYTES:
limit_mb = settings.INPUT_IMAGE_MAX_BYTES // 1024 // 1024
raise HTTPException(413, f"图片不能超过 {limit_mb} MB")
if not data:
raise HTTPException(400, "图片文件不能为空")
try:
stored = await asyncio.to_thread(store_input_image, data)
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
except Exception as exc:
raise HTTPException(503, "图片暂时无法上传，请确认对象存储可用") from exc
return InputImageAssetOut(
assetToken=stored.token,
width=stored.width,
height=stored.height,
sizeBytes=stored.size_bytes,
)
@router.delete("/{asset_token}")
async def delete_input_image(asset_token: str):
try:
await asyncio.to_thread(discard_input_image, asset_token)
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
except Exception as exc:
raise HTTPException(503, "图片附件暂时无法清理") from exc
return {"ok": True}

12
backend/services/conversation_history.py
View File

@@ -176,6 +176,8 @@ class ConversationRecorder:
*,
input_id: str,
timestamp: object,
content: str = "",
source: str = "camera_capture",
mime_type: str = "image/jpeg",
) -> None:
"""Persist a captured frame without delaying the realtime model turn."""
@@ -185,6 +187,8 @@ class ConversationRecorder:
data,
input_id=input_id,
timestamp=timestamp,
content=content,
source=source,
mime_type=mime_type,
),
name=f"conversation-image:{self.session_id}:{input_id}",
@@ -198,6 +202,8 @@ class ConversationRecorder:
*,
input_id: str,
timestamp: object,
content: str,
source: str,
mime_type: str,
) -> None:
message_id = f"msg_{uuid4().hex[:20]}"
@@ -221,11 +227,11 @@ class ConversationRecorder:
sequence=next_sequence,
role="user",
content_type="image",
content="",
content=content.strip(),
occurred_at=_parse_timestamp(timestamp),
extra={
"input_id": input_id,
"source": "camera_capture",
"source": source,
},
)
)
@@ -238,7 +244,7 @@ class ConversationRecorder:
storage_uri=storage_uri(key),
mime_type=mime_type,
size_bytes=len(data),
extra={"input_id": input_id},
extra={"input_id": input_id, "source": source},
)
)
conversation = await db.get(ConversationSession, self.session_id)

132
backend/services/input_assets.py Normal file
View File

@@ -0,0 +1,132 @@
"""Short-lived uploaded images consumed by the realtime user-input protocol."""
from __future__ import annotations
import base64
import hashlib
import hmac
import json
import time
from dataclasses import dataclass
from io import BytesIO
from uuid import uuid4
from PIL import Image, ImageOps, UnidentifiedImageError
import settings
from services.object_storage import delete_object, get_object, put_object
INPUT_ASSET_PREFIX = "conversation-inputs/"
@dataclass(frozen=True)
class StoredInputImage:
token: str
width: int
height: int
size_bytes: int
def _b64encode(data: bytes) -> str:
return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii")
def _b64decode(data: str) -> bytes:
padding = "=" * (-len(data) % 4)
return base64.urlsafe_b64decode(f"{data}{padding}".encode("ascii"))
def _signature(payload: str) -> str:
digest = hmac.new(
settings.AUTH_SECRET_KEY.encode("utf-8"),
payload.encode("ascii"),
hashlib.sha256,
).digest()
return _b64encode(digest)
def _normalize_image(data: bytes) -> tuple[bytes, int, int]:
try:
source = Image.open(BytesIO(data))
width, height = source.size
if width <= 0 or height <= 0:
raise ValueError("图片尺寸无效")
if width * height > settings.INPUT_IMAGE_MAX_PIXELS:
raise ValueError("图片像素尺寸过大")
source.load()
except (UnidentifiedImageError, OSError) as exc:
raise ValueError("文件不是可识别的图片") from exc
image = ImageOps.exif_transpose(source)
image.thumbnail(
(settings.INPUT_IMAGE_MAX_EDGE, settings.INPUT_IMAGE_MAX_EDGE),
Image.Resampling.LANCZOS,
)
if image.mode in {"RGBA", "LA"} or "transparency" in image.info:
rgba = image.convert("RGBA")
background = Image.new("RGB", rgba.size, "white")
background.paste(rgba, mask=rgba.getchannel("A"))
image = background
else:
image = image.convert("RGB")
buffer = BytesIO()
image.save(buffer, format="JPEG", quality=85, optimize=True)
return buffer.getvalue(), image.width, image.height
def store_input_image(data: bytes) -> StoredInputImage:
normalized, width, height = _normalize_image(data)
key = f"{INPUT_ASSET_PREFIX}{uuid4().hex}.jpg"
put_object(key, normalized, "image/jpeg")
expires_at = int(time.time()) + settings.INPUT_IMAGE_TOKEN_TTL_SECONDS
encoded_payload = _b64encode(
json.dumps(
{"key": key, "exp": expires_at},
separators=(",", ":"),
).encode("utf-8")
)
token = f"{encoded_payload}.{_signature(encoded_payload)}"
return StoredInputImage(
token=token,
width=width,
height=height,
size_bytes=len(normalized),
)
def _key_from_token(token: str) -> str:
if not token or "." not in token:
raise ValueError("图片附件 token 无效")
encoded_payload, signature = token.rsplit(".", 1)
if not hmac.compare_digest(_signature(encoded_payload), signature):
raise ValueError("图片附件 token 签名无效")
try:
payload = json.loads(_b64decode(encoded_payload))
key = str(payload.get("key") or "")
expires_at = int(payload.get("exp") or 0)
except (ValueError, TypeError, json.JSONDecodeError) as exc:
raise ValueError("图片附件 token 内容无效") from exc
if expires_at < int(time.time()):
raise ValueError("图片附件已过期，请重新添加")
if not key.startswith(INPUT_ASSET_PREFIX):
raise ValueError("图片附件存储位置无效")
return key
def consume_input_image(token: str) -> bytes:
"""Read a signed image once and remove its temporary object."""
key = _key_from_token(token)
data = get_object(key)
try:
delete_object(key)
except Exception:
# Consumption succeeded. A stale temporary object must not break the turn.
pass
return data
def discard_input_image(token: str) -> None:
delete_object(_key_from_token(token))

51
backend/services/pipecat/pipeline.py
View File

@@ -27,6 +27,7 @@ from services.pipecat.service_factory import (
)
from db.session import SessionLocal
from services.knowledge import search as search_knowledge
from services.input_assets import consume_input_image
from services.client_tools import ClientToolBroker
from services.tool_policy import policy_for_tool
from services.vision import (
@@ -36,6 +37,7 @@ from services.vision import (
config_with_main_llm_as_vision,
config_with_vision_resource,
image_data_uri,
image_frame_from_jpeg,
image_jpeg_bytes,
)
from services.workflow_engine import WorkflowEngine
@@ -734,7 +736,7 @@ async def run_pipeline(
greeting = await brain.greeting(cfg)
async def submit_user_input(value: UserInput) -> None:
if not value.has_camera_frame:
if not value.has_image:
if not value.run_immediately:
brain.record_user_message(value.text)
await worker.queue_frame(
@@ -749,28 +751,49 @@ async def run_pipeline(
raise ValueError("P0 图片输入必须立即触发回复")
if not vision_enabled:
raise ValueError("当前助手未启用视觉能力")
user_id = vision_state.get("client_id")
if not user_id:
raise ValueError("当前没有可用的摄像头视频流")
native_vision = active_vision_uses_main_llm()
analysis_cfg = None if native_vision else active_vision_config()
request = UserImageRequestFrame(
user_id=user_id,
text=value.prompt_text,
append_to_context=False,
)
try:
image_frame = await vision_capture.request_image(llm, request)
except asyncio.TimeoutError as exc:
raise ValueError("等待摄像头视频帧超时") from exc
if value.image_asset_token:
try:
image_bytes = await asyncio.to_thread(
consume_input_image,
value.image_asset_token,
)
image_frame = await asyncio.to_thread(
image_frame_from_jpeg,
image_bytes,
)
except ValueError:
raise
except Exception as exc:
raise ValueError("上传图片暂时无法读取") from exc
else:
user_id = vision_state.get("client_id")
if not user_id:
raise ValueError("当前没有可用的摄像头视频流")
request = UserImageRequestFrame(
user_id=user_id,
text=value.prompt_text,
append_to_context=False,
)
try:
image_frame = await vision_capture.request_image(llm, request)
except asyncio.TimeoutError as exc:
raise ValueError("等待摄像头视频帧超时") from exc
image_bytes = await asyncio.to_thread(image_jpeg_bytes, image_frame)
if recorder:
image_bytes = await asyncio.to_thread(image_jpeg_bytes, image_frame)
recorder.record_image_later(
image_bytes,
input_id=value.input_id,
timestamp=time_now_iso8601(),
content=value.text,
source=(
"uploaded_asset"
if value.image_asset_token
else "camera_capture"
),
)
if native_vision:

11
backend/services/pipecat/pipeline_events.py
View File

@@ -149,11 +149,12 @@ def bind_cascade_pipeline_events(
@text_input.event_handler("on_user_input")
async def on_user_input(_processor, user_input: UserInput):
await queue_transcript(
"user",
user_input.transcript_text,
time_now_iso8601(),
)
if not user_input.has_image:
await queue_transcript(
"user",
user_input.transcript_text,
time_now_iso8601(),
)
if user_input.run_immediately and user_input.interrupt:
pending_user_inputs.append(user_input)
return

41
backend/services/pipecat/processors.py
View File

@@ -50,9 +50,14 @@ class UserInput:
input_id: str
text: str
has_camera_frame: bool
image_asset_token: str | None
run_immediately: bool
interrupt: bool
@property
def has_image(self) -> bool:
return self.has_camera_frame or bool(self.image_asset_token)
@property
def prompt_text(self) -> str:
if self.text:
@@ -61,10 +66,9 @@ class UserInput:
@property
def transcript_text(self) -> str:
# Images are represented as structured media records. A synthetic
# sentence would be shown as chat text and persisted as if the user had
# said it, while adding no visual information for the model.
return self.text
# Image turns are persisted as one structured media message, including
# any optional text caption. Do not create a duplicate transcript row.
return "" if self.has_image else self.text
class UserInputError(ValueError):
@@ -155,6 +159,7 @@ def parse_user_input(message) -> UserInput | None:
text = ""
has_camera_frame = False
image_asset_token: str | None = None
for part in parts:
if not isinstance(part, dict):
raise UserInputError("user-input part 格式不正确", input_id=input_id)
@@ -166,23 +171,30 @@ def parse_user_input(message) -> UserInput | None:
if not text:
raise UserInputError("input_text 不能为空", input_id=input_id)
elif part_type == "input_image":
if has_camera_frame:
if has_camera_frame or image_asset_token:
raise UserInputError("P0 只支持一张图片", input_id=input_id)
source = part.get("source")
if (
not isinstance(source, dict)
or source.get("type") != "camera_frame"
or source.get("frame") != "current"
):
if not isinstance(source, dict):
raise UserInputError("input_image source 格式不正确", input_id=input_id)
if source.get("type") == "camera_frame" and source.get("frame") == "current":
has_camera_frame = True
elif source.get("type") == "uploaded_asset":
token = str(source.get("asset_token") or "").strip()
if not token or len(token) > 4096:
raise UserInputError(
"上传图片缺少有效的 asset_token",
input_id=input_id,
)
image_asset_token = token
else:
raise UserInputError(
"P0 只支持当前摄像头画面",
"仅支持当前摄像头画面或已上传图片",
input_id=input_id,
)
has_camera_frame = True
else:
raise UserInputError(f"不支持的输入类型: {part_type}", input_id=input_id)
if not text and not has_camera_frame:
if not text and not (has_camera_frame or image_asset_token):
raise UserInputError("user-input 没有有效内容", input_id=input_id)
options = message.get("options")
@@ -193,6 +205,7 @@ def parse_user_input(message) -> UserInput | None:
input_id=input_id,
text=text,
has_camera_frame=has_camera_frame,
image_asset_token=image_asset_token,
run_immediately=run_immediately,
interrupt=interrupt,
)
@@ -565,7 +578,7 @@ class RealtimeUserInputProcessor(FrameProcessor):
"当前工作流节点暂不接收用户输入",
)
return
if user_input.has_camera_frame:
if user_input.has_image:
await self._emit_error(
user_input.input_id,
"Realtime 模式暂不支持图片输入",

15
backend/services/vision.py
View File

@@ -49,6 +49,21 @@ def image_data_uri(frame: UserImageRawFrame) -> str:
return f"data:image/jpeg;base64,{encoded}"
def image_frame_from_jpeg(data: bytes) -> UserImageRawFrame:
"""Decode one normalized uploaded JPEG into Pipecat's raw image frame."""
try:
image = Image.open(BytesIO(data)).convert("RGB")
image.load()
except (OSError, ValueError) as exc:
raise ValueError("上传图片无法解码") from exc
return UserImageRawFrame(
image=image.tobytes(),
size=image.size,
format="RGB",
)
async def analyze_image_with_vision_model(
cfg: AssistantConfig,
frame: UserImageRawFrame,

8
backend/settings.py
View File

@@ -59,3 +59,11 @@ S3_BUCKET = os.getenv("S3_BUCKET", "ai-video")
S3_REGION = os.getenv("S3_REGION", "us-east-1")
KNOWLEDGE_MAX_FILE_BYTES = int(os.getenv("KNOWLEDGE_MAX_FILE_BYTES", "20971520"))
KNOWLEDGE_TOP_K = int(os.getenv("KNOWLEDGE_TOP_K", "5"))
# ---- Realtime user-input image attachments ----
INPUT_IMAGE_MAX_BYTES = int(os.getenv("INPUT_IMAGE_MAX_BYTES", "10485760"))
INPUT_IMAGE_MAX_PIXELS = int(os.getenv("INPUT_IMAGE_MAX_PIXELS", "24000000"))
INPUT_IMAGE_MAX_EDGE = int(os.getenv("INPUT_IMAGE_MAX_EDGE", "2048"))
INPUT_IMAGE_TOKEN_TTL_SECONDS = int(
os.getenv("INPUT_IMAGE_TOKEN_TTL_SECONDS", "600")
)

4
backend/tests/test_conversation_history.py
View File

@@ -139,6 +139,8 @@ class ConversationRecorderTest(unittest.IsolatedAsyncioTestCase):
input_id="input_photo",
timestamp="2026-08-05T10:00:00+08:00",
mime_type="image/jpeg",
content="帮我看看",
source="uploaded_asset",
)
self.assertTrue(session.committed)
@@ -146,7 +148,9 @@ class ConversationRecorderTest(unittest.IsolatedAsyncioTestCase):
message, artifact = session.added
self.assertEqual(message.content_type, "image")
self.assertEqual(message.role, "user")
self.assertEqual(message.content, "帮我看看")
self.assertEqual(message.extra["input_id"], "input_photo")
self.assertEqual(message.extra["source"], "uploaded_asset")
self.assertEqual(artifact.message_id, message.id)
self.assertEqual(artifact.kind, "image")
self.assertEqual(artifact.size_bytes, len(b"jpeg-data"))

58
backend/tests/test_input_assets.py Normal file
View File

@@ -0,0 +1,58 @@
from __future__ import annotations
import unittest
from io import BytesIO
from unittest.mock import patch
from PIL import Image
from services.input_assets import consume_input_image, store_input_image
def png_bytes(size: tuple[int, int] = (32, 24)) -> bytes:
output = BytesIO()
Image.new("RGBA", size, (20, 80, 160, 180)).save(output, format="PNG")
return output.getvalue()
class InputAssetTests(unittest.TestCase):
def test_store_normalizes_and_consume_removes_temporary_object(self):
with (
patch("services.input_assets.put_object") as put_object,
patch("services.input_assets.get_object") as get_object,
patch("services.input_assets.delete_object") as delete_object,
):
stored = store_input_image(png_bytes())
key, normalized, mime_type = put_object.call_args.args
self.assertTrue(key.startswith("conversation-inputs/"))
self.assertEqual(mime_type, "image/jpeg")
self.assertTrue(normalized.startswith(b"\xff\xd8"))
self.assertEqual((stored.width, stored.height), (32, 24))
self.assertEqual(stored.size_bytes, len(normalized))
get_object.return_value = normalized
consumed = consume_input_image(stored.token)
self.assertEqual(consumed, normalized)
get_object.assert_called_once_with(key)
delete_object.assert_called_once_with(key)
def test_tampered_token_is_rejected_before_storage_read(self):
with patch("services.input_assets.put_object"):
stored = store_input_image(png_bytes())
tampered = f"{stored.token}x"
with patch("services.input_assets.get_object") as get_object:
with self.assertRaisesRegex(ValueError, "签名无效"):
consume_input_image(tampered)
get_object.assert_not_called()
def test_non_image_is_rejected(self):
with self.assertRaisesRegex(ValueError, "可识别的图片"):
store_input_image(b"not an image")
if __name__ == "__main__":
unittest.main()

32
backend/tests/test_user_input.py
View File

@@ -32,7 +32,7 @@ class UserInputParserTests(unittest.TestCase):
self.assertIsNotNone(value)
self.assertEqual(value.text, "帮我看看")
self.assertTrue(value.has_camera_frame)
self.assertEqual(value.transcript_text, "帮我看看")
self.assertEqual(value.transcript_text, "")
def test_rejects_legacy_and_unsupported_image_sources(self):
self.assertIsNone(parse_user_input({"type": "user-text", "text": "旧协议"}))
@@ -46,14 +46,40 @@ class UserInputParserTests(unittest.TestCase):
{
"type": "input_image",
"source": {
"type": "uploaded_asset",
"asset_id": "asset_1",
"type": "remote_url",
"url": "https://example.com/image.jpg",
},
}
],
}
)
def test_parses_uploaded_image_asset(self):
value = parse_user_input(
{
"type": "user-input",
"schema_version": 1,
"input_id": "input_upload",
"parts": [
{"type": "input_text", "text": "这是什么？"},
{
"type": "input_image",
"source": {
"type": "uploaded_asset",
"asset_token": "signed-token",
},
},
],
}
)
self.assertIsNotNone(value)
self.assertTrue(value.has_image)
self.assertFalse(value.has_camera_frame)
self.assertEqual(value.image_asset_token, "signed-token")
self.assertEqual(value.prompt_text, "这是什么？")
self.assertEqual(value.transcript_text, "")
def test_image_only_input_has_no_synthetic_chat_text(self):
value = parse_user_input(
{