Compare commits

...

3 Commits

Author SHA1 Message Date
Xin Wang
e36ca308b8 feat: support uploaded images in debug voice preview
Allow paste/drag temporary image assets so vision turns work without a live camera frame.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-05 19:52:37 +08:00
Xin Wang
555c8f5fa6 Merge remote-tracking branch 'origin/main' into codex/photo-history 2026-08-05 16:14:17 +08:00
Xin Wang
317a0600bc feat: persist photos and enrich conversation history 2026-08-05 16:14:03 +08:00
26 changed files with 1820 additions and 201 deletions

View File

@@ -26,6 +26,7 @@ from routes import (
auth,
conversations,
health,
input_assets,
knowledge_bases,
mcp_servers,
model_registry,
@@ -60,6 +61,7 @@ app.add_middleware(
app.include_router(health.router)
app.include_router(auth.router)
app.include_router(conversations.router)
app.include_router(input_assets.router)
app.include_router(assistants.router)
app.include_router(knowledge_bases.router)
app.include_router(mcp_servers.router)

View File

@@ -6,7 +6,7 @@ pipecat-ai[webrtc,websocket,silero,openai,mcp]==1.7.0
Pillow>=11.1.0,<13
# FastGPT 类型助手:本地 SDK(包 /api/v1/chat/completions 流式 + chatId 会话)
fastgpt-client @ file:///Users/wangx/Code/AI-VideoAssistant-Project/fastgpt-python-sdk
# fastgpt-client @ file:///Users/wangx/Code/AI-VideoAssistant-Project/fastgpt-python-sdk
# Dify 类型助手:异步流式 Runtime API SDK
dify-client-python==1.0.3

View File

@@ -1,15 +1,20 @@
"""对话历史查询 API。"""
from db.models import ConversationMessage, ConversationSession
import asyncio
from db.models import ConversationArtifact, ConversationMessage, ConversationSession
from db.session import get_session
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi import APIRouter, Depends, HTTPException, Query, Response
from loguru import logger
from schemas import (
ConversationArtifactOut,
ConversationDetailOut,
ConversationListOut,
ConversationMessageOut,
ConversationOut,
)
from services.auth import require_admin
from services.object_storage import delete_object, get_object, key_from_storage_uri
from sqlalchemy import func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -100,6 +105,17 @@ async def get_conversation(
.order_by(ConversationMessage.sequence)
)
).scalars().all()
artifacts = (
await session.execute(
select(ConversationArtifact)
.where(ConversationArtifact.session_id == conversation_id)
.order_by(ConversationArtifact.created_at)
)
).scalars().all()
artifacts_by_message: dict[str, list[ConversationArtifact]] = {}
for artifact in artifacts:
if artifact.message_id:
artifacts_by_message.setdefault(artifact.message_id, []).append(artifact)
return ConversationDetailOut(
**_session_out(conversation).model_dump(),
extra=conversation.extra or {},
@@ -112,12 +128,52 @@ async def get_conversation(
content=message.content,
occurred_at=message.occurred_at,
extra=message.extra or {},
artifacts=[
ConversationArtifactOut(
id=artifact.id,
kind=artifact.kind,
content_url=(
f"/api/conversations/{conversation_id}/artifacts/"
f"{artifact.id}/content"
),
mime_type=artifact.mime_type,
size_bytes=artifact.size_bytes,
duration_ms=artifact.duration_ms,
extra=artifact.extra or {},
)
for artifact in artifacts_by_message.get(message.id, [])
],
)
for message in messages
],
)
@router.get("/{conversation_id}/artifacts/{artifact_id}/content")
async def get_conversation_artifact(
conversation_id: str,
artifact_id: str,
session: AsyncSession = Depends(get_session),
):
artifact = await session.get(ConversationArtifact, artifact_id)
if not artifact or artifact.session_id != conversation_id:
raise HTTPException(404, "会话附件不存在")
try:
key = key_from_storage_uri(artifact.storage_uri)
data = await asyncio.to_thread(get_object, key)
except Exception as exc:
logger.warning(f"读取会话附件失败: {artifact_id}: {exc}")
raise HTTPException(404, "会话附件不可用") from exc
return Response(
content=data,
media_type=artifact.mime_type or "application/octet-stream",
headers={
"Cache-Control": "private, max-age=300",
"Content-Disposition": f'inline; filename="{artifact.id}"',
},
)
@router.delete("/{conversation_id}")
async def delete_conversation(
conversation_id: str,
@@ -126,6 +182,19 @@ async def delete_conversation(
conversation = await session.get(ConversationSession, conversation_id)
if not conversation:
raise HTTPException(404, "对话记录不存在")
storage_uris = (
await session.execute(
select(ConversationArtifact.storage_uri).where(
ConversationArtifact.session_id == conversation_id
)
)
).scalars().all()
await session.delete(conversation)
await session.commit()
for storage_uri in storage_uris:
try:
key = key_from_storage_uri(storage_uri)
await asyncio.to_thread(delete_object, key)
except Exception as exc:
logger.warning(f"清理已删除会话的附件失败: {storage_uri}: {exc}")
return {"ok": True}

View File

@@ -0,0 +1,59 @@
"""Authenticated upload endpoint for pending debug-chat image attachments."""
from __future__ import annotations
import asyncio
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
from pydantic import BaseModel
import settings
from services.auth import require_admin
from services.input_assets import discard_input_image, store_input_image
router = APIRouter(
prefix="/api/input-assets",
tags=["input-assets"],
dependencies=[Depends(require_admin)],
)
class InputImageAssetOut(BaseModel):
assetToken: str
width: int
height: int
sizeBytes: int
@router.post("/image", response_model=InputImageAssetOut)
async def upload_input_image(file: UploadFile = File(...)):
data = await file.read(settings.INPUT_IMAGE_MAX_BYTES + 1)
if len(data) > settings.INPUT_IMAGE_MAX_BYTES:
limit_mb = settings.INPUT_IMAGE_MAX_BYTES // 1024 // 1024
raise HTTPException(413, f"图片不能超过 {limit_mb} MB")
if not data:
raise HTTPException(400, "图片文件不能为空")
try:
stored = await asyncio.to_thread(store_input_image, data)
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
except Exception as exc:
raise HTTPException(503, "图片暂时无法上传,请确认对象存储可用") from exc
return InputImageAssetOut(
assetToken=stored.token,
width=stored.width,
height=stored.height,
sizeBytes=stored.size_bytes,
)
@router.delete("/{asset_token}")
async def delete_input_image(asset_token: str):
try:
await asyncio.to_thread(discard_input_image, asset_token)
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
except Exception as exc:
raise HTTPException(503, "图片附件暂时无法清理") from exc
return {"ok": True}

View File

@@ -467,6 +467,16 @@ class ModelResourceTestResult(CamelModel):
# ---------- 对话历史 ----------
class ConversationArtifactOut(CamelModel):
id: str
kind: str
content_url: str
mime_type: str
size_bytes: int | None = None
duration_ms: int | None = None
extra: dict[str, Any] = Field(default_factory=dict)
class ConversationMessageOut(CamelModel):
id: str
sequence: int
@@ -475,6 +485,7 @@ class ConversationMessageOut(CamelModel):
content: str
occurred_at: datetime
extra: dict[str, Any] = Field(default_factory=dict)
artifacts: list[ConversationArtifactOut] = Field(default_factory=list)
class ConversationOut(CamelModel):

View File

@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio
from collections.abc import Awaitable
from time import monotonic
from typing import Any
from uuid import uuid4
@@ -503,12 +504,45 @@ class PromptBrain(BaseBrain):
await params.result_callback(result)
async def call_tool(params: FunctionCallParams) -> None:
invocation_id = f"tool_{uuid4().hex[:20]}"
started_at = monotonic()
await self._emit_trace(
"tool_started",
invocationId=invocation_id,
toolId=tool.id,
toolName=tool.name,
functionName=tool.function_name,
toolType=tool.type,
)
try:
result = await self._tools.execute(tool, dict(params.arguments or {}))
if result["updated_variables"]:
self._refresh_prompt()
await self._emit_trace(
"tool_completed" if result.get("status") == "ok" else "tool_failed",
invocationId=invocation_id,
toolId=tool.id,
toolName=tool.name,
functionName=tool.function_name,
toolType=tool.type,
status=str(result.get("status") or "unknown"),
durationMs=max(0, round((monotonic() - started_at) * 1000)),
updatedVariables=list(result.get("updated_variables") or []),
resultKeys=sorted(str(name) for name in result if name != "data"),
)
await return_result(params, result)
except (ToolExecutionError, ValueError) as exc:
await self._emit_trace(
"tool_failed",
invocationId=invocation_id,
toolId=tool.id,
toolName=tool.name,
functionName=tool.function_name,
toolType=tool.type,
status="error",
durationMs=max(0, round((monotonic() - started_at) * 1000)),
error=str(exc)[:2048],
)
await return_result(
params,
{"status": "error", "message": f"工具调用失败: {exc}"},
@@ -522,6 +556,26 @@ class PromptBrain(BaseBrain):
)
return schema, call_tool
async def _emit_trace(self, event: str, **details: Any) -> None:
runtime = self._runtime
if runtime is None:
return
try:
await runtime.queue_frame(
OutputTransportMessageUrgentFrame(
message={
"type": "workflow-event",
"eventId": f"wfe_{uuid4().hex[:20]}",
"event": event,
"timestamp": time_now_iso8601(),
"sessionId": runtime.session_id,
**details,
}
)
)
except Exception as exc: # noqa: BLE001 - trace must not alter execution
logger.warning(f"发送 Prompt 工具轨迹失败,不影响当前调用: {exc}")
def _make_end_call_tool(self, tool, runtime: BrainRuntime):
config = (tool.definition or {}).get("config") or {}
message_type = str(config.get("message_type") or "none")

View File

@@ -6,7 +6,9 @@ import asyncio
from collections.abc import Awaitable
from copy import deepcopy
from dataclasses import dataclass, replace
from time import monotonic
from typing import Any
from uuid import uuid4
from loguru import logger
from models import AssistantConfig, RuntimeTool
@@ -764,10 +766,46 @@ class WorkflowBrain(BaseBrain):
async def handler(args, _flow_manager):
transition_id = self._state.transition_id
invocation_id = f"tool_{uuid4().hex[:20]}"
started_at = monotonic()
await self._emit_trace(
"tool_started",
nodeId=node_id,
invocationId=invocation_id,
toolId=tool.id,
toolName=tool.name,
functionName=tool.function_name,
toolType=tool.type,
)
try:
result = await self._tools.execute(tool, dict(args or {}))
except ToolExecutionError as exc:
await self._emit_trace(
"tool_failed",
nodeId=node_id,
invocationId=invocation_id,
toolId=tool.id,
toolName=tool.name,
functionName=tool.function_name,
toolType=tool.type,
status="error",
durationMs=max(0, round((monotonic() - started_at) * 1000)),
error=str(exc)[:2048],
)
return {"status": "error", "message": str(exc)}
await self._emit_trace(
"tool_completed" if result.get("status") == "ok" else "tool_failed",
nodeId=node_id,
invocationId=invocation_id,
toolId=tool.id,
toolName=tool.name,
functionName=tool.function_name,
toolType=tool.type,
status=str(result.get("status") or "unknown"),
durationMs=max(0, round((monotonic() - started_at) * 1000)),
updatedVariables=list(result.get("updated_variables") or []),
resultKeys=sorted(str(name) for name in result if name != "data"),
)
if (
self._state.current_node_id != node_id
or self._state.transition_id != transition_id

View File

@@ -7,6 +7,7 @@ from collections import deque
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from typing import Any, Literal, Protocol
from uuid import uuid4
from loguru import logger
from pipecat.frames.frames import (
@@ -17,6 +18,7 @@ from pipecat.frames.frames import (
StopFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.time import time_now_iso8601
class ClientToolError(RuntimeError):
@@ -29,6 +31,7 @@ ClientToolResponseWaitMode = Literal["timeout", "session"]
@dataclass(frozen=True)
class _PendingClientToolCall:
future: asyncio.Future[dict[str, Any]]
function_name: str
interrupt_on_result: bool = False
@@ -80,8 +83,6 @@ class ClientToolBroker(FrameProcessor):
response_wait_mode: ClientToolResponseWaitMode = "timeout",
interrupt_on_result: bool = False,
) -> dict[str, Any]:
from uuid import uuid4
if self._closed_message is not None:
raise ClientToolError(self._closed_message)
@@ -99,9 +100,21 @@ class ClientToolBroker(FrameProcessor):
OutputTransportMessageUrgentFrame(message=message)
)
except Exception as exc:
await self._emit_trace(
"client_tool_failed",
toolCallId=tool_call_id,
functionName=function_name,
error="客户端工具调用发送失败",
)
raise ClientToolError(
f"客户端工具调用发送失败: {function_name}"
) from exc
await self._emit_trace(
"client_tool_completed",
toolCallId=tool_call_id,
functionName=function_name,
status="dispatched",
)
return {
"status": "ok",
"data": {"dispatched": True},
@@ -111,6 +124,7 @@ class ClientToolBroker(FrameProcessor):
future: asyncio.Future[dict[str, Any]] = loop.create_future()
self._pending[tool_call_id] = _PendingClientToolCall(
future=future,
function_name=function_name,
interrupt_on_result=interrupt_on_result,
)
try:
@@ -121,6 +135,12 @@ class ClientToolBroker(FrameProcessor):
return await future
return await asyncio.wait_for(future, timeout=timeout_seconds)
except TimeoutError as exc:
await self._emit_trace(
"client_tool_failed",
toolCallId=tool_call_id,
functionName=function_name,
error="客户端工具调用超时",
)
raise ClientToolError(f"客户端工具调用超时: {function_name}") from exc
except ClientToolError:
raise
@@ -163,10 +183,28 @@ class ClientToolBroker(FrameProcessor):
future = pending.future
status = str(message.get("status") or "error")
result_data = message.get("data")
user_action = (
str(result_data.get("action") or "")
if isinstance(result_data, dict)
else ""
)
await self._emit_trace(
"client_tool_completed" if status == "ok" else "client_tool_failed",
toolCallId=tool_call_id,
functionName=pending.function_name,
status=status,
**({"userAction": user_action} if user_action else {}),
**(
{"error": str(message.get("message") or "客户端工具执行失败")}
if status != "ok"
else {}
),
)
if status == "ok":
response = {
"status": "ok",
"data": message.get("data"),
"data": result_data,
}
if pending.interrupt_on_result:
# Match text input exactly: broadcasting only starts the
@@ -206,6 +244,23 @@ class ClientToolBroker(FrameProcessor):
}
)
async def _emit_trace(self, event: str, **details: Any) -> None:
"""Publish sanitized client interaction metadata for conversation history."""
try:
await self.push_frame(
OutputTransportMessageUrgentFrame(
message={
"type": "workflow-event",
"eventId": f"wfe_{uuid4().hex[:20]}",
"event": event,
"timestamp": time_now_iso8601(),
**details,
}
)
)
except Exception as exc: # noqa: BLE001 - tracing must not alter the tool call
logger.warning(f"发送客户端工具轨迹失败,不影响工具调用: {exc}")
def on_interruption_processed(self) -> None:
"""Release one result after the aggregator acknowledges interruption."""
while self._deferred_results:

View File

@@ -9,9 +9,10 @@ from copy import deepcopy
from datetime import UTC, datetime
from uuid import uuid4
from db.models import ConversationMessage, ConversationSession
from db.models import ConversationArtifact, ConversationMessage, ConversationSession
from db.session import SessionLocal
from loguru import logger
from services.object_storage import delete_object, put_object, storage_uri
MAX_WORKFLOW_TRACE_EVENTS = 1000
@@ -36,6 +37,7 @@ class ConversationRecorder:
self._trace_sequence = 0
self._lock = asyncio.Lock()
self._seen_events: set[str] = set()
self._pending_artifacts: set[asyncio.Task[None]] = set()
@classmethod
async def start(
@@ -168,6 +170,95 @@ class ConversationRecorder:
except Exception as exc:
logger.error(f"保存对话文本失败,不影响本次通话: {exc}")
def record_image_later(
self,
data: bytes,
*,
input_id: str,
timestamp: object,
content: str = "",
source: str = "camera_capture",
mime_type: str = "image/jpeg",
) -> None:
"""Persist a captured frame without delaying the realtime model turn."""
task = asyncio.create_task(
self._record_image(
data,
input_id=input_id,
timestamp=timestamp,
content=content,
source=source,
mime_type=mime_type,
),
name=f"conversation-image:{self.session_id}:{input_id}",
)
self._pending_artifacts.add(task)
task.add_done_callback(self._pending_artifacts.discard)
async def _record_image(
self,
data: bytes,
*,
input_id: str,
timestamp: object,
content: str,
source: str,
mime_type: str,
) -> None:
message_id = f"msg_{uuid4().hex[:20]}"
artifact_id = f"artifact_{uuid4().hex[:20]}"
extension = ".jpg" if mime_type == "image/jpeg" else ".bin"
key = f"conversations/{self.session_id}/{artifact_id}{extension}"
try:
await asyncio.to_thread(put_object, key, data, mime_type)
except Exception as exc:
logger.error(f"保存会话图片到对象存储失败,不影响本次通话: {exc}")
return
async with self._lock:
next_sequence = self._sequence + 1
try:
async with SessionLocal() as db:
db.add(
ConversationMessage(
id=message_id,
session_id=self.session_id,
sequence=next_sequence,
role="user",
content_type="image",
content=content.strip(),
occurred_at=_parse_timestamp(timestamp),
extra={
"input_id": input_id,
"source": source,
},
)
)
db.add(
ConversationArtifact(
id=artifact_id,
session_id=self.session_id,
message_id=message_id,
kind="image",
storage_uri=storage_uri(key),
mime_type=mime_type,
size_bytes=len(data),
extra={"input_id": input_id, "source": source},
)
)
conversation = await db.get(ConversationSession, self.session_id)
if conversation:
conversation.message_count = next_sequence
await db.commit()
self._sequence = next_sequence
except Exception as exc:
logger.error(f"保存会话图片索引失败,不影响本次通话: {exc}")
try:
await asyncio.to_thread(delete_object, key)
except Exception:
logger.warning(f"清理未关联的会话图片失败: {key}")
async def finish(self, *, status: str = "completed") -> None:
"""Finish the session even if the owning pipeline is being cancelled."""
finish_task = asyncio.create_task(
@@ -184,6 +275,9 @@ class ConversationRecorder:
raise
async def _finish(self, *, status: str) -> None:
pending_artifacts = list(self._pending_artifacts)
if pending_artifacts:
await asyncio.gather(*pending_artifacts, return_exceptions=True)
async with self._lock:
try:
async with SessionLocal() as db:

View File

@@ -0,0 +1,132 @@
"""Short-lived uploaded images consumed by the realtime user-input protocol."""
from __future__ import annotations
import base64
import hashlib
import hmac
import json
import time
from dataclasses import dataclass
from io import BytesIO
from uuid import uuid4
from PIL import Image, ImageOps, UnidentifiedImageError
import settings
from services.object_storage import delete_object, get_object, put_object
INPUT_ASSET_PREFIX = "conversation-inputs/"
@dataclass(frozen=True)
class StoredInputImage:
token: str
width: int
height: int
size_bytes: int
def _b64encode(data: bytes) -> str:
return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii")
def _b64decode(data: str) -> bytes:
padding = "=" * (-len(data) % 4)
return base64.urlsafe_b64decode(f"{data}{padding}".encode("ascii"))
def _signature(payload: str) -> str:
digest = hmac.new(
settings.AUTH_SECRET_KEY.encode("utf-8"),
payload.encode("ascii"),
hashlib.sha256,
).digest()
return _b64encode(digest)
def _normalize_image(data: bytes) -> tuple[bytes, int, int]:
try:
source = Image.open(BytesIO(data))
width, height = source.size
if width <= 0 or height <= 0:
raise ValueError("图片尺寸无效")
if width * height > settings.INPUT_IMAGE_MAX_PIXELS:
raise ValueError("图片像素尺寸过大")
source.load()
except (UnidentifiedImageError, OSError) as exc:
raise ValueError("文件不是可识别的图片") from exc
image = ImageOps.exif_transpose(source)
image.thumbnail(
(settings.INPUT_IMAGE_MAX_EDGE, settings.INPUT_IMAGE_MAX_EDGE),
Image.Resampling.LANCZOS,
)
if image.mode in {"RGBA", "LA"} or "transparency" in image.info:
rgba = image.convert("RGBA")
background = Image.new("RGB", rgba.size, "white")
background.paste(rgba, mask=rgba.getchannel("A"))
image = background
else:
image = image.convert("RGB")
buffer = BytesIO()
image.save(buffer, format="JPEG", quality=85, optimize=True)
return buffer.getvalue(), image.width, image.height
def store_input_image(data: bytes) -> StoredInputImage:
normalized, width, height = _normalize_image(data)
key = f"{INPUT_ASSET_PREFIX}{uuid4().hex}.jpg"
put_object(key, normalized, "image/jpeg")
expires_at = int(time.time()) + settings.INPUT_IMAGE_TOKEN_TTL_SECONDS
encoded_payload = _b64encode(
json.dumps(
{"key": key, "exp": expires_at},
separators=(",", ":"),
).encode("utf-8")
)
token = f"{encoded_payload}.{_signature(encoded_payload)}"
return StoredInputImage(
token=token,
width=width,
height=height,
size_bytes=len(normalized),
)
def _key_from_token(token: str) -> str:
if not token or "." not in token:
raise ValueError("图片附件 token 无效")
encoded_payload, signature = token.rsplit(".", 1)
if not hmac.compare_digest(_signature(encoded_payload), signature):
raise ValueError("图片附件 token 签名无效")
try:
payload = json.loads(_b64decode(encoded_payload))
key = str(payload.get("key") or "")
expires_at = int(payload.get("exp") or 0)
except (ValueError, TypeError, json.JSONDecodeError) as exc:
raise ValueError("图片附件 token 内容无效") from exc
if expires_at < int(time.time()):
raise ValueError("图片附件已过期,请重新添加")
if not key.startswith(INPUT_ASSET_PREFIX):
raise ValueError("图片附件存储位置无效")
return key
def consume_input_image(token: str) -> bytes:
"""Read a signed image once and remove its temporary object."""
key = _key_from_token(token)
data = get_object(key)
try:
delete_object(key)
except Exception:
# Consumption succeeded. A stale temporary object must not break the turn.
pass
return data
def discard_input_image(token: str) -> None:
delete_object(_key_from_token(token))

View File

@@ -6,9 +6,6 @@ from pathlib import Path
import re
import uuid
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
from docx import Document as DocxDocument
from openai import AsyncOpenAI
from pypdf import PdfReader
@@ -17,38 +14,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
import settings
from db.models import KnowledgeBase, KnowledgeChunk, KnowledgeDocument, ModelResource
def _s3_client():
return boto3.client(
"s3",
endpoint_url=settings.S3_ENDPOINT_URL,
aws_access_key_id=settings.S3_ACCESS_KEY,
aws_secret_access_key=settings.S3_SECRET_KEY,
region_name=settings.S3_REGION,
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
)
def _ensure_bucket_and_put(key: str, data: bytes, mime_type: str) -> None:
client = _s3_client()
try:
client.head_bucket(Bucket=settings.S3_BUCKET)
except ClientError as exc:
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
if status != 404:
raise
client.create_bucket(Bucket=settings.S3_BUCKET)
client.put_object(Bucket=settings.S3_BUCKET, Key=key, Body=data, ContentType=mime_type)
def _delete_object(key: str) -> None:
_s3_client().delete_object(Bucket=settings.S3_BUCKET, Key=key)
def _get_object(key: str) -> bytes:
response = _s3_client().get_object(Bucket=settings.S3_BUCKET, Key=key)
return response["Body"].read()
from services.object_storage import delete_object, get_object, put_object
def extract_text(filename: str, data: bytes) -> str:
@@ -122,7 +88,7 @@ async def create_document(
safe_name = Path(name).name
extension = ".txt" if source_type == "text" else Path(safe_name).suffix
storage_key = f"knowledge/{kb.id}/{document_id}/source{extension}"
await asyncio.to_thread(_ensure_bucket_and_put, storage_key, raw_data, mime_type)
await asyncio.to_thread(put_object, storage_key, raw_data, mime_type)
document = KnowledgeDocument(
id=document_id,
@@ -158,7 +124,7 @@ async def process_document(document_id: str) -> None:
await session.commit()
try:
data = await asyncio.to_thread(_get_object, document.storage_key)
data = await asyncio.to_thread(get_object, document.storage_key)
text = (
data.decode("utf-8", errors="replace")
if document.source_type == "text"
@@ -245,4 +211,4 @@ async def search(
async def delete_storage_object(document: KnowledgeDocument) -> None:
if document.storage_key:
await asyncio.to_thread(_delete_object, document.storage_key)
await asyncio.to_thread(delete_object, document.storage_key)

View File

@@ -0,0 +1,60 @@
"""Small S3-compatible object-storage adapter shared by backend features."""
from __future__ import annotations
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
import settings
def _client():
return boto3.client(
"s3",
endpoint_url=settings.S3_ENDPOINT_URL,
aws_access_key_id=settings.S3_ACCESS_KEY,
aws_secret_access_key=settings.S3_SECRET_KEY,
region_name=settings.S3_REGION,
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
)
def put_object(key: str, data: bytes, mime_type: str) -> None:
client = _client()
try:
client.head_bucket(Bucket=settings.S3_BUCKET)
except ClientError as exc:
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
if status != 404:
raise
client.create_bucket(Bucket=settings.S3_BUCKET)
client.put_object(
Bucket=settings.S3_BUCKET,
Key=key,
Body=data,
ContentType=mime_type,
)
def get_object(key: str) -> bytes:
response = _client().get_object(Bucket=settings.S3_BUCKET, Key=key)
return response["Body"].read()
def delete_object(key: str) -> None:
_client().delete_object(Bucket=settings.S3_BUCKET, Key=key)
def storage_uri(key: str) -> str:
return f"s3://{settings.S3_BUCKET}/{key.lstrip('/')}"
def key_from_storage_uri(uri: str) -> str:
prefix = f"s3://{settings.S3_BUCKET}/"
if not uri.startswith(prefix):
raise ValueError("不支持的会话附件存储地址")
key = uri[len(prefix) :].strip("/")
if not key:
raise ValueError("会话附件存储地址缺少对象键")
return key

View File

@@ -27,6 +27,7 @@ from services.pipecat.service_factory import (
)
from db.session import SessionLocal
from services.knowledge import search as search_knowledge
from services.input_assets import consume_input_image
from services.client_tools import ClientToolBroker
from services.tool_policy import policy_for_tool
from services.vision import (
@@ -36,6 +37,8 @@ from services.vision import (
config_with_main_llm_as_vision,
config_with_vision_resource,
image_data_uri,
image_frame_from_jpeg,
image_jpeg_bytes,
)
from services.workflow_engine import WorkflowEngine
@@ -58,6 +61,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.llm_service import FunctionCallParams
from pipecat.utils.time import time_now_iso8601
from services.pipecat.turn_config import (
ConfigurableLLMUserAggregator,
create_user_turn_strategies,
@@ -732,7 +736,7 @@ async def run_pipeline(
greeting = await brain.greeting(cfg)
async def submit_user_input(value: UserInput) -> None:
if not value.has_camera_frame:
if not value.has_image:
if not value.run_immediately:
brain.record_user_message(value.text)
await worker.queue_frame(
@@ -747,21 +751,50 @@ async def run_pipeline(
raise ValueError("P0 图片输入必须立即触发回复")
if not vision_enabled:
raise ValueError("当前助手未启用视觉能力")
user_id = vision_state.get("client_id")
if not user_id:
raise ValueError("当前没有可用的摄像头视频流")
native_vision = active_vision_uses_main_llm()
analysis_cfg = None if native_vision else active_vision_config()
request = UserImageRequestFrame(
user_id=user_id,
text=value.prompt_text,
append_to_context=False,
)
try:
image_frame = await vision_capture.request_image(llm, request)
except asyncio.TimeoutError as exc:
raise ValueError("等待摄像头视频帧超时") from exc
if value.image_asset_token:
try:
image_bytes = await asyncio.to_thread(
consume_input_image,
value.image_asset_token,
)
image_frame = await asyncio.to_thread(
image_frame_from_jpeg,
image_bytes,
)
except ValueError:
raise
except Exception as exc:
raise ValueError("上传图片暂时无法读取") from exc
else:
user_id = vision_state.get("client_id")
if not user_id:
raise ValueError("当前没有可用的摄像头视频流")
request = UserImageRequestFrame(
user_id=user_id,
text=value.prompt_text,
append_to_context=False,
)
try:
image_frame = await vision_capture.request_image(llm, request)
except asyncio.TimeoutError as exc:
raise ValueError("等待摄像头视频帧超时") from exc
image_bytes = await asyncio.to_thread(image_jpeg_bytes, image_frame)
if recorder:
recorder.record_image_later(
image_bytes,
input_id=value.input_id,
timestamp=time_now_iso8601(),
content=value.text,
source=(
"uploaded_asset"
if value.image_asset_token
else "camera_capture"
),
)
if native_vision:
input_frame = await asyncio.to_thread(

View File

@@ -149,11 +149,12 @@ def bind_cascade_pipeline_events(
@text_input.event_handler("on_user_input")
async def on_user_input(_processor, user_input: UserInput):
await queue_transcript(
"user",
user_input.transcript_text,
time_now_iso8601(),
)
if not user_input.has_image:
await queue_transcript(
"user",
user_input.transcript_text,
time_now_iso8601(),
)
if user_input.run_immediately and user_input.interrupt:
pending_user_inputs.append(user_input)
return

View File

@@ -50,9 +50,14 @@ class UserInput:
input_id: str
text: str
has_camera_frame: bool
image_asset_token: str | None
run_immediately: bool
interrupt: bool
@property
def has_image(self) -> bool:
return self.has_camera_frame or bool(self.image_asset_token)
@property
def prompt_text(self) -> str:
if self.text:
@@ -61,9 +66,9 @@ class UserInput:
@property
def transcript_text(self) -> str:
if not self.has_camera_frame:
return self.text
return f"{self.text}\n已发送一张图片".strip()
# Image turns are persisted as one structured media message, including
# any optional text caption. Do not create a duplicate transcript row.
return "" if self.has_image else self.text
class UserInputError(ValueError):
@@ -154,6 +159,7 @@ def parse_user_input(message) -> UserInput | None:
text = ""
has_camera_frame = False
image_asset_token: str | None = None
for part in parts:
if not isinstance(part, dict):
raise UserInputError("user-input part 格式不正确", input_id=input_id)
@@ -165,23 +171,30 @@ def parse_user_input(message) -> UserInput | None:
if not text:
raise UserInputError("input_text 不能为空", input_id=input_id)
elif part_type == "input_image":
if has_camera_frame:
if has_camera_frame or image_asset_token:
raise UserInputError("P0 只支持一张图片", input_id=input_id)
source = part.get("source")
if (
not isinstance(source, dict)
or source.get("type") != "camera_frame"
or source.get("frame") != "current"
):
if not isinstance(source, dict):
raise UserInputError("input_image source 格式不正确", input_id=input_id)
if source.get("type") == "camera_frame" and source.get("frame") == "current":
has_camera_frame = True
elif source.get("type") == "uploaded_asset":
token = str(source.get("asset_token") or "").strip()
if not token or len(token) > 4096:
raise UserInputError(
"上传图片缺少有效的 asset_token",
input_id=input_id,
)
image_asset_token = token
else:
raise UserInputError(
"P0 只支持当前摄像头画面",
"支持当前摄像头画面或已上传图片",
input_id=input_id,
)
has_camera_frame = True
else:
raise UserInputError(f"不支持的输入类型: {part_type}", input_id=input_id)
if not text and not has_camera_frame:
if not text and not (has_camera_frame or image_asset_token):
raise UserInputError("user-input 没有有效内容", input_id=input_id)
options = message.get("options")
@@ -192,6 +205,7 @@ def parse_user_input(message) -> UserInput | None:
input_id=input_id,
text=text,
has_camera_frame=has_camera_frame,
image_asset_token=image_asset_token,
run_immediately=run_immediately,
interrupt=interrupt,
)
@@ -564,7 +578,7 @@ class RealtimeUserInputProcessor(FrameProcessor):
"当前工作流节点暂不接收用户输入",
)
return
if user_input.has_camera_frame:
if user_input.has_image:
await self._emit_error(
user_input.input_id,
"Realtime 模式暂不支持图片输入",

View File

@@ -30,8 +30,8 @@ def _require(value: str, label: str) -> str:
raise ValueError(f"缺少模型资源配置: {label}")
def image_data_uri(frame: UserImageRawFrame) -> str:
"""Encode one Pipecat camera frame for a vision chat-completion request."""
def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes:
"""Encode one Pipecat camera frame as a storage- and model-ready JPEG."""
if not frame.format:
raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型")
buffer = BytesIO()
@@ -40,10 +40,30 @@ def image_data_uri(frame: UserImageRawFrame) -> str:
format="JPEG",
quality=85,
)
encoded = base64.b64encode(buffer.getvalue()).decode("utf-8")
return buffer.getvalue()
def image_data_uri(frame: UserImageRawFrame) -> str:
"""Encode one Pipecat camera frame for a vision chat-completion request."""
encoded = base64.b64encode(image_jpeg_bytes(frame)).decode("utf-8")
return f"data:image/jpeg;base64,{encoded}"
def image_frame_from_jpeg(data: bytes) -> UserImageRawFrame:
"""Decode one normalized uploaded JPEG into Pipecat's raw image frame."""
try:
image = Image.open(BytesIO(data)).convert("RGB")
image.load()
except (OSError, ValueError) as exc:
raise ValueError("上传图片无法解码") from exc
return UserImageRawFrame(
image=image.tobytes(),
size=image.size,
format="RGB",
)
async def analyze_image_with_vision_model(
cfg: AssistantConfig,
frame: UserImageRawFrame,

View File

@@ -59,3 +59,11 @@ S3_BUCKET = os.getenv("S3_BUCKET", "ai-video")
S3_REGION = os.getenv("S3_REGION", "us-east-1")
KNOWLEDGE_MAX_FILE_BYTES = int(os.getenv("KNOWLEDGE_MAX_FILE_BYTES", "20971520"))
KNOWLEDGE_TOP_K = int(os.getenv("KNOWLEDGE_TOP_K", "5"))
# ---- Realtime user-input image attachments ----
INPUT_IMAGE_MAX_BYTES = int(os.getenv("INPUT_IMAGE_MAX_BYTES", "10485760"))
INPUT_IMAGE_MAX_PIXELS = int(os.getenv("INPUT_IMAGE_MAX_PIXELS", "24000000"))
INPUT_IMAGE_MAX_EDGE = int(os.getenv("INPUT_IMAGE_MAX_EDGE", "2048"))
INPUT_IMAGE_TOKEN_TTL_SECONDS = int(
os.getenv("INPUT_IMAGE_TOKEN_TTL_SECONDS", "600")
)

View File

@@ -105,6 +105,58 @@ class ConversationRecorderTest(unittest.IsolatedAsyncioTestCase):
self.assertEqual(saved["sessionId"], "conv_test")
self.assertEqual(saved["sequence"], 1)
async def test_image_is_persisted_as_message_with_artifact(self):
conversation = SimpleNamespace(message_count=0)
class FakeSession:
def __init__(self):
self.added = []
self.committed = False
async def __aenter__(self):
return self
async def __aexit__(self, *_args):
return None
def add(self, value):
self.added.append(value)
async def get(self, _model, _session_id):
return conversation
async def commit(self):
self.committed = True
session = FakeSession()
recorder = ConversationRecorder("conv_test")
with (
patch("services.conversation_history.SessionLocal", return_value=session),
patch("services.conversation_history.put_object") as put_object,
):
await recorder._record_image(
b"jpeg-data",
input_id="input_photo",
timestamp="2026-08-05T10:00:00+08:00",
mime_type="image/jpeg",
content="帮我看看",
source="uploaded_asset",
)
self.assertTrue(session.committed)
self.assertEqual(len(session.added), 2)
message, artifact = session.added
self.assertEqual(message.content_type, "image")
self.assertEqual(message.role, "user")
self.assertEqual(message.content, "帮我看看")
self.assertEqual(message.extra["input_id"], "input_photo")
self.assertEqual(message.extra["source"], "uploaded_asset")
self.assertEqual(artifact.message_id, message.id)
self.assertEqual(artifact.kind, "image")
self.assertEqual(artifact.size_bytes, len(b"jpeg-data"))
self.assertEqual(conversation.message_count, 1)
put_object.assert_called_once()
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,58 @@
from __future__ import annotations
import unittest
from io import BytesIO
from unittest.mock import patch
from PIL import Image
from services.input_assets import consume_input_image, store_input_image
def png_bytes(size: tuple[int, int] = (32, 24)) -> bytes:
output = BytesIO()
Image.new("RGBA", size, (20, 80, 160, 180)).save(output, format="PNG")
return output.getvalue()
class InputAssetTests(unittest.TestCase):
def test_store_normalizes_and_consume_removes_temporary_object(self):
with (
patch("services.input_assets.put_object") as put_object,
patch("services.input_assets.get_object") as get_object,
patch("services.input_assets.delete_object") as delete_object,
):
stored = store_input_image(png_bytes())
key, normalized, mime_type = put_object.call_args.args
self.assertTrue(key.startswith("conversation-inputs/"))
self.assertEqual(mime_type, "image/jpeg")
self.assertTrue(normalized.startswith(b"\xff\xd8"))
self.assertEqual((stored.width, stored.height), (32, 24))
self.assertEqual(stored.size_bytes, len(normalized))
get_object.return_value = normalized
consumed = consume_input_image(stored.token)
self.assertEqual(consumed, normalized)
get_object.assert_called_once_with(key)
delete_object.assert_called_once_with(key)
def test_tampered_token_is_rejected_before_storage_read(self):
with patch("services.input_assets.put_object"):
stored = store_input_image(png_bytes())
tampered = f"{stored.token}x"
with patch("services.input_assets.get_object") as get_object:
with self.assertRaisesRegex(ValueError, "签名无效"):
consume_input_image(tampered)
get_object.assert_not_called()
def test_non_image_is_rejected(self):
with self.assertRaisesRegex(ValueError, "可识别的图片"):
store_input_image(b"not an image")
if __name__ == "__main__":
unittest.main()

View File

@@ -32,7 +32,7 @@ class UserInputParserTests(unittest.TestCase):
self.assertIsNotNone(value)
self.assertEqual(value.text, "帮我看看")
self.assertTrue(value.has_camera_frame)
self.assertEqual(value.transcript_text, "帮我看看\n已发送一张图片")
self.assertEqual(value.transcript_text, "")
def test_rejects_legacy_and_unsupported_image_sources(self):
self.assertIsNone(parse_user_input({"type": "user-text", "text": "旧协议"}))
@@ -46,14 +46,59 @@ class UserInputParserTests(unittest.TestCase):
{
"type": "input_image",
"source": {
"type": "uploaded_asset",
"asset_id": "asset_1",
"type": "remote_url",
"url": "https://example.com/image.jpg",
},
}
],
}
)
def test_parses_uploaded_image_asset(self):
value = parse_user_input(
{
"type": "user-input",
"schema_version": 1,
"input_id": "input_upload",
"parts": [
{"type": "input_text", "text": "这是什么?"},
{
"type": "input_image",
"source": {
"type": "uploaded_asset",
"asset_token": "signed-token",
},
},
],
}
)
self.assertIsNotNone(value)
self.assertTrue(value.has_image)
self.assertFalse(value.has_camera_frame)
self.assertEqual(value.image_asset_token, "signed-token")
self.assertEqual(value.prompt_text, "这是什么?")
self.assertEqual(value.transcript_text, "")
def test_image_only_input_has_no_synthetic_chat_text(self):
value = parse_user_input(
{
"type": "user-input",
"schema_version": 1,
"input_id": "input_photo",
"parts": [
{
"type": "input_image",
"source": {"type": "camera_frame", "frame": "current"},
}
],
}
)
self.assertIsNotNone(value)
self.assertEqual(value.transcript_text, "")
self.assertEqual(value.prompt_text, "请根据用户刚提交的图片进行回复。")
def test_native_image_uses_the_standard_multimodal_user_turn_path(self):
image = UserImageRawFrame(
image=bytes([220, 40, 40] * 16 * 16),

View File

@@ -7,6 +7,7 @@ import {
Braces,
Check,
Copy,
ImageIcon,
Loader2,
MessageSquareText,
Mic,
@@ -51,16 +52,46 @@ import {
useVoicePreview,
type ChatMessage,
type ClientToolDefinition,
type UserInputPart,
type VoicePreview,
type VoicePreviewStatus,
} from "@/hooks/use-voice-preview";
import type { DynamicVariableDefinition } from "@/lib/api";
import {
inputAssetsApi,
type DynamicVariableDefinition,
} from "@/lib/api";
type VizStyle = "aura" | "nebula" | "bars" | "wave";
// 调试面板顶部主视图:聊天记录 / 视频流
type DebugView = "chat" | "video";
type DebugInputMode = "mic" | "text";
type PendingDebugImage = {
file: File;
previewUrl: string;
};
const DEBUG_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
function fileToDataUrl(file: File): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(String(reader.result));
reader.onerror = () => reject(new Error("无法读取图片"));
reader.readAsDataURL(file);
});
}
function hasDraggedImage(dataTransfer: DataTransfer): boolean {
return (
Array.from(dataTransfer.items).some((item) =>
item.type.startsWith("image/"),
) ||
Array.from(dataTransfer.files).some((file) =>
file.type.startsWith("image/"),
)
);
}
const VIZ_OPTIONS: { style: VizStyle; label: string; icon: React.ReactNode }[] =
[
@@ -766,6 +797,8 @@ function DebugVoicePanel({
remoteStream,
messages,
sendText,
sendUserInput,
appendUserImage,
connect,
disconnect,
audioRef,
@@ -773,6 +806,11 @@ function DebugVoicePanel({
const recording = status === "connecting" || status === "connected";
const [textDraft, setTextDraft] = useState("");
const [inputMode, setInputMode] = useState<DebugInputMode>("mic");
const [pendingImage, setPendingImage] =
useState<PendingDebugImage | null>(null);
const [inputError, setInputError] = useState("");
const [sendingInput, setSendingInput] = useState(false);
const [draggingImage, setDraggingImage] = useState(false);
const [clientDialogContainer, setClientDialogContainer] =
useState<HTMLDivElement | null>(null);
const [messageDialogOpen, setMessageDialogOpen] = useState(false);
@@ -790,9 +828,110 @@ function DebugVoicePanel({
: "";
const startDisabled = status === "connecting" || Boolean(startBlockedMessage);
function handleSendText() {
if (sendText(textDraft)) {
useEffect(() => {
return () => {
if (pendingImage) URL.revokeObjectURL(pendingImage.previewUrl);
};
}, [pendingImage]);
function stageImage(file: File) {
setInputError("");
if (!vision) {
setInputError("请先为助手开启视觉理解,再添加图片。");
return;
}
if (!file.type.startsWith("image/")) {
setInputError("只能添加图片文件。");
return;
}
if (file.size > DEBUG_IMAGE_MAX_BYTES) {
setInputError("图片不能超过 10 MB。");
return;
}
setPendingImage({ file, previewUrl: URL.createObjectURL(file) });
setInputMode("text");
}
async function handleSendInput() {
const text = textDraft.trim();
if (!pendingImage) {
if (sendText(text)) {
setTextDraft("");
setInputError("");
}
return;
}
if (status !== "connected" || sendingInput) return;
setSendingInput(true);
setInputError("");
let assetToken = "";
try {
const imageUrl = await fileToDataUrl(pendingImage.file);
const asset = await inputAssetsApi.uploadImage(pendingImage.file);
assetToken = asset.assetToken;
const parts: UserInputPart[] = [];
if (text) parts.push({ type: "input_text", text });
parts.push({
type: "input_image",
source: { type: "uploaded_asset", asset_token: assetToken },
});
const timestamp = new Date().toISOString();
const result = await sendUserInput(parts);
appendUserImage(result.inputId, imageUrl, timestamp, text);
setTextDraft("");
setPendingImage(null);
} catch (sendError) {
if (assetToken) {
void inputAssetsApi.remove(assetToken).catch(() => {});
}
setInputError(
sendError instanceof Error ? sendError.message : "图片发送失败,请重试。",
);
} finally {
setSendingInput(false);
}
}
function handlePaste(event: React.ClipboardEvent<HTMLTextAreaElement>) {
const image = Array.from(event.clipboardData.items)
.find((item) => item.type.startsWith("image/"))
?.getAsFile();
if (!image) return;
event.preventDefault();
stageImage(image);
}
function handleDragOver(event: React.DragEvent<HTMLDivElement>) {
if (!hasDraggedImage(event.dataTransfer)) return;
event.preventDefault();
event.dataTransfer.dropEffect = "copy";
setDraggingImage(true);
}
function handleDragLeave(event: React.DragEvent<HTMLDivElement>) {
const nextTarget = event.relatedTarget;
if (nextTarget instanceof Node && event.currentTarget.contains(nextTarget)) {
return;
}
setDraggingImage(false);
}
function handleDrop(event: React.DragEvent<HTMLDivElement>) {
if (!hasDraggedImage(event.dataTransfer)) return;
event.preventDefault();
setDraggingImage(false);
const image = Array.from(event.dataTransfer.files).find((file) =>
file.type.startsWith("image/"),
);
if (image) stageImage(image);
}
function removePendingImage() {
if (!sendingInput) {
setPendingImage(null);
setInputError("");
}
}
@@ -968,61 +1107,136 @@ function DebugVoicePanel({
</div>
<div className="shrink-0 border-t border-hairline bg-card p-3">
<div className="flex items-center gap-2">
<div
className={[
"flex h-10 min-w-0 items-center gap-1 rounded-[1.4rem] border border-hairline-strong bg-background px-2",
"flex-1",
].join(" ")}
>
<div className="flex shrink-0 items-center gap-0.5 rounded-full bg-canvas-soft p-0.5">
<DebugInputModeButton
selected={inputMode === "mic"}
label="选择麦克风设备"
onClick={() => setInputMode("mic")}
>
<Mic size={15} />
</DebugInputModeButton>
<DebugInputModeButton
selected={inputMode === "text"}
label="文字输入"
onClick={() => setInputMode("text")}
>
<MessageSquareText size={15} />
</DebugInputModeButton>
<div className="flex items-end gap-2">
<div className="min-w-0 flex-1">
<div
onDragOver={handleDragOver}
onDragLeave={handleDragLeave}
onDrop={handleDrop}
className={[
"relative flex min-h-10 min-w-0 items-end gap-1 overflow-hidden rounded-[1.4rem] border bg-background px-2 transition-colors",
draggingImage
? "border-foreground"
: "border-hairline-strong",
].join(" ")}
>
<div className="mb-1 flex shrink-0 items-center gap-0.5 rounded-full bg-canvas-soft p-0.5">
<DebugInputModeButton
selected={inputMode === "mic"}
label="选择麦克风设备"
onClick={() => setInputMode("mic")}
>
<Mic size={15} />
</DebugInputModeButton>
<DebugInputModeButton
selected={inputMode === "text"}
label="文字输入"
onClick={() => setInputMode("text")}
>
<MessageSquareText size={15} />
</DebugInputModeButton>
</div>
{inputMode === "mic" ? (
<div className="flex min-w-0 flex-1">
<MicrophoneDeviceField preview={preview} />
</div>
) : (
<div className="min-w-0 flex-1 py-1">
{pendingImage && (
<div className="flex items-center gap-2 px-2 pb-1 pt-0.5">
<div className="group relative h-14 w-14 shrink-0 overflow-hidden rounded-xl border border-hairline bg-canvas-soft">
{/* eslint-disable-next-line @next/next/no-img-element */}
<img
src={pendingImage.previewUrl}
alt="待发送图片预览"
className="h-full w-full object-cover"
/>
<button
type="button"
aria-label="移除待发送图片"
title="移除图片"
disabled={sendingInput}
onClick={removePendingImage}
className="absolute right-1 top-1 flex h-5 w-5 items-center justify-center rounded-full bg-background/90 text-foreground shadow-sm transition-colors hover:bg-background disabled:cursor-not-allowed"
>
<X size={12} />
</button>
{sendingInput && (
<span className="absolute inset-0 flex items-center justify-center bg-background/65">
<Loader2 size={18} className="animate-spin" />
</span>
)}
</div>
<div className="min-w-0">
<p className="truncate text-xs font-medium text-foreground">
{pendingImage.file.name || "粘贴的图片"}
</p>
<p className="mt-0.5 text-[11px] text-muted-soft">
Enter
</p>
</div>
</div>
)}
<Textarea
rows={1}
value={textDraft}
disabled={status !== "connected" || sendingInput}
onChange={(event) => {
setTextDraft(event.target.value);
if (inputError) setInputError("");
}}
onPaste={handlePaste}
onKeyDown={(event) => {
if (
event.key === "Enter" &&
!event.shiftKey &&
!event.nativeEvent.isComposing
) {
event.preventDefault();
void handleSendInput();
}
}}
placeholder={
status === "connected"
? vision
? "输入文字,或粘贴 / 拖入图片…"
: "输入文字发送给助手,将打断当前播报…"
: "开始对话后可输入文字…"
}
className="max-h-24 min-h-8 resize-none overflow-y-auto border-transparent bg-transparent px-2 py-1 text-sm leading-6 text-foreground shadow-none outline-none placeholder:text-muted-soft focus-visible:ring-0 disabled:opacity-100"
/>
</div>
)}
{draggingImage && (
<div className="pointer-events-none absolute inset-0 z-20 flex items-center justify-center gap-2 rounded-[1.4rem] bg-background/95 text-xs font-medium text-foreground">
<ImageIcon size={16} />
</div>
)}
</div>
{inputMode === "mic" ? (
<MicrophoneDeviceField preview={preview} />
) : (
<Textarea
rows={1}
value={textDraft}
disabled={status !== "connected"}
onChange={(event) => setTextDraft(event.target.value)}
onKeyDown={(event) => {
if (event.key === "Enter" && !event.shiftKey && !event.nativeEvent.isComposing) {
event.preventDefault();
handleSendText();
}
}}
placeholder={
status === "connected"
? "输入文字发送给助手,将打断当前播报…"
: "开始对话后可输入文字…"
}
className="h-8 min-h-8 flex-1 resize-none overflow-hidden border-transparent bg-transparent px-2 py-1 text-sm leading-6 text-foreground shadow-none outline-none placeholder:text-muted-soft focus-visible:ring-0 disabled:opacity-100"
/>
{inputError && (
<p className="px-2 pt-1 text-[11px] leading-4 text-destructive">
{inputError}
</p>
)}
</div>
{inputMode === "text" && (
<Button
size="icon"
className="h-10 w-10 shrink-0 rounded-full"
aria-label="发送调试消息"
disabled={status !== "connected" || !textDraft.trim()}
onClick={handleSendText}
aria-label={sendingInput ? "正在发送图片" : "发送调试消息"}
disabled={
status !== "connected" ||
sendingInput ||
(!textDraft.trim() && !pendingImage)
}
onClick={() => void handleSendInput()}
>
<Send size={16} />
{sendingInput ? (
<Loader2 size={16} className="animate-spin" />
) : (
<Send size={16} />
)}
</Button>
)}
{!showIdleHub && (
@@ -1168,7 +1382,10 @@ function DebugVisionWorkspace({
} | null>(null);
const latestMessage = [...messages]
.reverse()
.find((message) => message.content.trim());
.find(
(message) =>
message.content.trim() || (message.attachments?.length ?? 0) > 0,
);
useEffect(() => {
const container = containerRef.current;
@@ -1215,7 +1432,10 @@ function DebugVisionWorkspace({
{latestMessage?.role === "user" ? "我:" : "助手:"}
</span>
<span>
{latestMessage?.content || "暂无消息,点击返回聊天记录"}
{latestMessage?.content ||
(latestMessage?.attachments?.length
? "发送了一张照片"
: "暂无消息,点击返回聊天记录")}
</span>
</span>
</button>
@@ -1463,7 +1683,7 @@ function DebugTranscriptPanel({
{time ? ` · ${time}` : ""}
</span>
<div className="whitespace-pre-wrap rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
{message.content}
{message.content || (message.streaming ? "…" : "")}
</div>
</div>
) : (
@@ -1474,8 +1694,26 @@ function DebugTranscriptPanel({
<span className="px-1 text-[11px] text-muted-soft">
{time ? ` · ${time}` : ""}
</span>
<div className="whitespace-pre-wrap rounded-2xl rounded-tr-sm bg-primary px-4 py-2.5 text-sm leading-6 text-primary-foreground">
{message.content}
<div
className={[
"overflow-hidden whitespace-pre-wrap rounded-2xl rounded-tr-sm bg-primary text-sm leading-6 text-primary-foreground",
message.attachments?.length ? "p-1" : "px-4 py-2.5",
].join(" ")}
>
{message.attachments?.map((attachment) => (
// eslint-disable-next-line @next/next/no-img-element
<img
key={attachment.id}
src={attachment.url}
alt={attachment.alt}
className="max-h-72 w-full rounded-[0.8rem] object-cover"
/>
))}
{message.content && (
<div className={message.attachments?.length ? "px-3 py-2" : ""}>
{message.content}
</div>
)}
</div>
</div>
);

View File

@@ -1,13 +1,25 @@
"use client";
import {
Activity,
ArrowRight,
Camera,
CheckCircle2,
ChevronDown,
ChevronUp,
CircleDot,
Eye,
GitBranch,
ImageIcon,
Loader2,
MessageSquareText,
MonitorSmartphone,
MoreHorizontal,
Server,
Trash2,
Wrench,
XCircle,
type LucideIcon,
} from "lucide-react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
@@ -37,9 +49,11 @@ import {
} from "@/components/layout/list-page-layout";
import { SearchInput } from "@/components/ui/search-input";
import {
API_BASE,
conversationsApi,
type Conversation,
type ConversationDetail,
type ConversationMessage,
} from "@/lib/api";
import { cn } from "@/lib/utils";
@@ -93,6 +107,249 @@ function statusLabel(status: string): string {
return "已结束";
}
type TraceEvent = Record<string, unknown>;
type WorkflowNode = { label: string; type: string };
type TraceTone = "default" | "success" | "error" | "user";
type TracePresentation = {
title: string;
description: string;
icon: LucideIcon;
tone: TraceTone;
};
function recordValue(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
}
function textValue(value: unknown): string {
return typeof value === "string" ? value : "";
}
function numberValue(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) ? value : null;
}
function workflowNodes(detail: ConversationDetail): Map<string, WorkflowNode> {
const snapshot = recordValue(detail.extra.workflow?.snapshot);
const nodes = Array.isArray(snapshot.nodes) ? snapshot.nodes : [];
return new Map(
nodes.flatMap((rawNode) => {
const node = recordValue(rawNode);
const id = textValue(node.id);
if (!id) return [];
const data = recordValue(node.data);
const type = textValue(node.type) || "node";
return [
[
id,
{
label: textValue(data.name) || type,
type,
},
] as const,
];
}),
);
}
function nodeLabel(nodes: Map<string, WorkflowNode>, nodeId: unknown): string {
const id = textValue(nodeId);
const node = nodes.get(id);
return node ? `${node.label}` : id ? `${id}` : "当前节点";
}
function toolLabel(toolType: unknown): string {
if (toolType === "client") return "客户端工具";
if (toolType === "http") return "HTTP 服务端工具";
if (toolType === "mcp") return "MCP 服务端工具";
if (toolType === "system") return "系统工具";
return "工具";
}
function tracePresentation(
event: TraceEvent,
nodes: Map<string, WorkflowNode>,
startedEvent?: TraceEvent,
): TracePresentation {
const eventName = textValue(event.event);
const node = nodeLabel(nodes, event.nodeId);
const outcome = recordValue(event.outcome);
const duration = numberValue(outcome.durationMs);
const tool = toolLabel(event.toolType ?? startedEvent?.toolType);
switch (eventName) {
case "node_entered":
return {
title: `进入节点 ${node}`,
description: `节点类型:${textValue(event.nodeType) || nodes.get(textValue(event.nodeId))?.type || "未知"}`,
icon: CircleDot,
tone: "default",
};
case "node_exited":
return {
title: `离开节点 ${node}`,
description: "当前节点处理完成,准备选择下一步。",
icon: ArrowRight,
tone: "default",
};
case "edge_selected":
return {
title: `节点转移:${nodeLabel(nodes, event.sourceNodeId)}${nodeLabel(nodes, event.targetNodeId)}`,
description: `路由方式:${textValue(event.edgeMode) || "always"}`,
icon: GitBranch,
tone: "default",
};
case "action_started":
return {
title: `开始调用${tool}`,
description: `${node} · ${textValue(event.toolId) || "未命名工具"}`,
icon: tool === "客户端工具" ? MonitorSmartphone : Server,
tone: "default",
};
case "action_completed":
return {
title: `${tool}执行成功`,
description: `${node}${duration === null ? "" : ` · ${duration} ms`}`,
icon: CheckCircle2,
tone: "success",
};
case "action_failed":
case "action_cancelled":
return {
title: `${tool}${eventName === "action_failed" ? "执行失败" : "已取消"}`,
description: textValue(recordValue(outcome.error).message) || node,
icon: XCircle,
tone: "error",
};
case "tool_started":
return {
title: `开始调用${tool}`,
description: `${node} · ${textValue(event.toolName) || textValue(event.functionName) || textValue(event.toolId) || "未命名工具"}`,
icon: tool === "客户端工具" ? MonitorSmartphone : Server,
tone: "default",
};
case "tool_completed":
return {
title: `${tool}调用成功`,
description: `${textValue(event.toolName) || textValue(event.functionName) || node}${numberValue(event.durationMs) === null ? "" : ` · ${numberValue(event.durationMs)} ms`}`,
icon: CheckCircle2,
tone: "success",
};
case "tool_failed":
return {
title: `${tool}调用失败`,
description: textValue(event.error) || textValue(event.toolName) || node,
icon: XCircle,
tone: "error",
};
case "client_tool_started":
return {
title:
event.functionName === "show_message"
? "向客户端显示交互消息"
: "向客户端下发工具调用",
description: textValue(event.functionName) || "客户端工具",
icon: MonitorSmartphone,
tone: "default",
};
case "client_tool_completed": {
const userAction = textValue(event.userAction);
return {
title: userAction === "confirmed" ? "用户已确认" : "客户端交互已完成",
description: userAction
? `用户操作:${userAction}`
: textValue(event.functionName) || textValue(event.status) || "执行成功",
icon: CheckCircle2,
tone: userAction ? "user" : "success",
};
}
case "client_tool_failed":
return {
title: "客户端交互失败",
description: textValue(event.error) || textValue(event.functionName),
icon: XCircle,
tone: "error",
};
case "message_started":
return event.requiresConfirmation
? {
title: "等待用户确认",
description: `${node}向客户端显示了确认消息。`,
icon: MonitorSmartphone,
tone: "user",
}
: {
title: "开始播放固定消息",
description: node,
icon: Activity,
tone: "default",
};
case "message_completed": {
const action = textValue(event.action);
return {
title: action === "confirmed" ? "用户已确认" : "消息步骤已完成",
description: `${node}${action ? ` · 操作:${action}` : ""}`,
icon: CheckCircle2,
tone: action ? "user" : "success",
};
}
case "message_interrupted":
return {
title: "用户输入打断消息",
description: node,
icon: Activity,
tone: "user",
};
case "message_failed":
return {
title: "消息步骤失败",
description: textValue(event.error) || node,
icon: XCircle,
tone: "error",
};
case "variables_updated": {
const names = Array.isArray(event.variableNames)
? event.variableNames.filter((name): name is string => typeof name === "string")
: [];
return {
title: "会话变量已更新",
description: names.length ? names.join("、") : node,
icon: Wrench,
tone: "default",
};
}
case "variables_snapshot":
return {
title: "记录会话变量快照",
description: node,
icon: Wrench,
tone: "default",
};
case "workflow_ended":
return {
title: "工作流已结束",
description: `${node} · ${textValue(event.outcome) || "success"}`,
icon: CheckCircle2,
tone: "success",
};
default:
return {
title: eventName || "运行事件",
description: node,
icon: Activity,
tone: "default",
};
}
}
function artifactUrl(path: string): string {
if (/^(https?:|data:|blob:)/.test(path)) return path;
return `${API_BASE}${path.startsWith("/") ? path : `/${path}`}`;
}
export function HistoryPage() {
const [rows, setRows] = useState<Conversation[]>([]);
const [total, setTotal] = useState(0);
@@ -370,7 +627,7 @@ export function HistoryPage() {
<>
<ListPageLayout
title="历史记录"
description="查看每次语音或文字会话中最终确认的用户转写和助手回复。"
description="按会话查看对话、照片、工具调用、用户操作与工作流运行轨迹。"
>
<ListPageSection>
<ListToolbar
@@ -423,7 +680,7 @@ export function HistoryPage() {
</ListPageLayout>
<Dialog open={dialogOpen} onOpenChange={setDialogOpen}>
<DialogContent className="max-h-[calc(100vh-3rem)] grid-rows-[auto_minmax(0,1fr)_auto] overflow-hidden sm:max-w-4xl">
<DialogContent className="max-h-[calc(100vh-3rem)] grid-rows-[auto_minmax(0,1fr)_auto] overflow-hidden sm:max-w-5xl">
<DialogHeader>
<DialogTitle className="flex items-center gap-2">
<MessageSquareText size={18} />
@@ -432,7 +689,7 @@ export function HistoryPage() {
<DialogDescription>
{detail
? `${formatDate(detail.startedAt)} · ${channelLabel(detail.channel)} · ${detail.messageCount} 条消息`
: "查看本次会话中最终确认的文本消息。"}
: "查看本次会话的完整对话与运行记录。"}
</DialogDescription>
</DialogHeader>
@@ -464,56 +721,7 @@ export function HistoryPage() {
</div>
</section>
<section className="rounded-xl border border-hairline bg-surface-strong/20">
<div className="border-b border-hairline px-4 py-3 text-sm font-medium">
</div>
<div className="space-y-5 p-4 sm:p-5">
{detail.messages.map((message) => (
<div
key={message.id}
className={cn(
"flex",
message.role === "user"
? "justify-end"
: "justify-start",
)}
>
<div className="max-w-[85%]">
<div
className={cn(
"mb-1.5 text-xs text-muted-soft",
message.role === "user" && "text-right",
)}
>
{message.role === "user" ? "用户" : "助手"} ·{" "}
{formatDate(message.occurredAt)}
</div>
<div
className={cn(
"whitespace-pre-wrap rounded-2xl px-4 py-3 text-left leading-6",
message.role === "user"
? "rounded-br-md bg-primary text-primary-foreground"
: "rounded-bl-md border border-hairline bg-background text-foreground shadow-sm",
)}
>
{message.content}
</div>
{message.extra.interrupted && (
<div className="mt-1.5 text-xs text-muted-foreground">
</div>
)}
</div>
</div>
))}
{detail.messages.length === 0 && (
<div className="py-12 text-center text-muted-foreground">
</div>
)}
</div>
</section>
<ConversationTimeline detail={detail} />
</div>
)}
</div>
@@ -537,3 +745,237 @@ function Metadata({ label, value }: { label: string; value: string }) {
</div>
);
}
type TimelineEntry =
| {
kind: "message";
timestamp: string;
order: number;
message: ConversationMessage;
}
| {
kind: "trace";
timestamp: string;
order: number;
event: TraceEvent;
};
function timestampOrder(value: string): number {
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? Number.MAX_SAFE_INTEGER : parsed;
}
function ConversationTimeline({ detail }: { detail: ConversationDetail }) {
const nodes = workflowNodes(detail);
const trace = (detail.extra.workflowTrace ?? []).map(recordValue);
const startedByInvocation = new Map<string, TraceEvent>();
trace.forEach((event) => {
if (event.event === "action_started" || event.event === "tool_started") {
const invocationId = textValue(event.invocationId);
if (invocationId) startedByInvocation.set(invocationId, event);
}
});
const entries: TimelineEntry[] = [
...detail.messages.map((message) => ({
kind: "message" as const,
timestamp: message.occurredAt,
order: message.sequence,
message,
})),
...trace.map((event, index) => ({
kind: "trace" as const,
timestamp: textValue(event.timestamp),
order: numberValue(event.sequence) ?? index + 1,
event,
})),
].sort(
(a, b) =>
timestampOrder(a.timestamp) - timestampOrder(b.timestamp) ||
a.order - b.order,
);
const imageCount = detail.messages.reduce(
(count, message) =>
count + (message.artifacts ?? []).filter((item) => item.kind === "image").length,
0,
);
const toolCount = trace.filter(
(event) => event.event === "action_started" || event.event === "tool_started",
).length;
const transitionCount = trace.filter((event) => event.event === "edge_selected").length;
return (
<section className="overflow-hidden rounded-xl border border-hairline bg-surface-strong/20">
<div className="flex flex-wrap items-center justify-between gap-3 border-b border-hairline px-4 py-3">
<div className="text-sm font-medium">线</div>
<div className="flex flex-wrap gap-2">
<Badge variant="secondary" className="gap-1.5 bg-surface-strong text-muted-foreground">
<MessageSquareText size={12} />
{detail.messages.length}
</Badge>
{imageCount > 0 && (
<Badge variant="secondary" className="gap-1.5 bg-surface-strong text-muted-foreground">
<ImageIcon size={12} />
{imageCount}
</Badge>
)}
{toolCount > 0 && (
<Badge variant="secondary" className="gap-1.5 bg-surface-strong text-muted-foreground">
<Wrench size={12} />
{toolCount}
</Badge>
)}
{transitionCount > 0 && (
<Badge variant="secondary" className="gap-1.5 bg-surface-strong text-muted-foreground">
<GitBranch size={12} />
{transitionCount}
</Badge>
)}
</div>
</div>
<div className="space-y-4 p-4 sm:p-5">
{entries.map((entry, index) =>
entry.kind === "message" ? (
<TimelineMessage key={`message-${entry.message.id}`} message={entry.message} />
) : (
<TimelineTrace
key={`trace-${textValue(entry.event.eventId) || index}`}
event={entry.event}
nodes={nodes}
startedEvent={startedByInvocation.get(
textValue(recordValue(entry.event.outcome).invocationId) ||
textValue(entry.event.invocationId),
)}
/>
),
)}
{entries.length === 0 && (
<div className="py-12 text-center text-muted-foreground">
</div>
)}
</div>
</section>
);
}
function TimelineMessage({ message }: { message: ConversationMessage }) {
const isUser = message.role === "user";
const images = (message.artifacts ?? []).filter((item) => item.kind === "image");
const isImageMessage = message.contentType === "image" || images.length > 0;
return (
<div className={cn("flex", isUser ? "justify-end" : "justify-start")}>
<div className="max-w-[88%] sm:max-w-[78%]">
<div
className={cn(
"mb-1.5 flex items-center gap-1.5 text-xs text-muted-soft",
isUser && "justify-end",
)}
>
{isImageMessage && <Camera size={12} />}
{isUser ? "用户" : "助手"} · {formatDate(message.occurredAt)}
</div>
<div
className={cn(
"overflow-hidden rounded-2xl text-left leading-6",
isUser
? "rounded-br-md bg-primary text-primary-foreground"
: "rounded-bl-md border border-hairline bg-background text-foreground shadow-sm",
isImageMessage ? "p-1" : "px-4 py-3",
)}
>
{images.map((image) => {
const url = artifactUrl(image.contentUrl);
return (
<a
key={image.id}
href={url}
target="_blank"
rel="noreferrer"
className="block overflow-hidden rounded-[0.8rem] bg-black/10"
title="在新窗口查看原图"
>
{/* eslint-disable-next-line @next/next/no-img-element */}
<img
src={url}
alt="用户在通话中拍摄的照片"
className="max-h-[28rem] w-full object-contain"
/>
</a>
);
})}
{isImageMessage && images.length === 0 && (
<div className="flex min-h-40 min-w-56 flex-col items-center justify-center gap-2 rounded-[0.8rem] bg-background/10 px-6 text-center text-sm opacity-75">
<ImageIcon size={22} />
</div>
)}
{message.content && (
<div className={isImageMessage ? "px-3 py-2" : ""}>{message.content}</div>
)}
</div>
{(message.extra.source || message.extra.node_id) && (
<div className={cn("mt-1.5 text-[11px] text-muted-soft", isUser && "text-right")}>
{[message.extra.source, message.extra.node_id].filter(Boolean).join(" · ")}
</div>
)}
{message.extra.interrupted && (
<div className="mt-1.5 text-xs text-muted-foreground">
</div>
)}
</div>
</div>
);
}
function TimelineTrace({
event,
nodes,
startedEvent,
}: {
event: TraceEvent;
nodes: Map<string, WorkflowNode>;
startedEvent?: TraceEvent;
}) {
const presentation = tracePresentation(event, nodes, startedEvent);
const Icon = presentation.icon;
const iconClass = {
default: "bg-surface-strong text-muted-foreground",
success: "bg-success/10 text-success",
error: "bg-destructive/10 text-destructive",
user: "bg-primary/10 text-primary",
}[presentation.tone];
return (
<div className="flex gap-3 rounded-xl border border-hairline-soft bg-background/65 p-3">
<div className={cn("flex size-8 shrink-0 items-center justify-center rounded-full", iconClass)}>
<Icon size={15} />
</div>
<div className="min-w-0 flex-1">
<div className="flex flex-wrap items-start justify-between gap-x-4 gap-y-1">
<div>
<div className="text-sm font-medium text-foreground">{presentation.title}</div>
<div className="mt-0.5 text-xs leading-5 text-muted-foreground">
{presentation.description}
</div>
</div>
<time className="shrink-0 text-[11px] tabular-nums text-muted-soft">
{formatDate(textValue(event.timestamp))}
</time>
</div>
<details className="mt-2 text-xs text-muted-foreground">
<summary className="w-fit cursor-pointer select-none hover:text-foreground">
</summary>
<pre className="mt-2 max-h-64 overflow-auto rounded-lg bg-surface-strong p-3 font-mono text-[11px] leading-5 text-foreground">
{JSON.stringify(event, null, 2)}
</pre>
</details>
</div>
</div>
);
}

View File

@@ -91,13 +91,31 @@ function MobileCallTranscript({ messages }: { messages: ChatMessage[] }) {
</span>
<div
className={[
"whitespace-pre-wrap rounded-2xl px-3.5 py-2.5 text-sm leading-6 shadow-sm",
"overflow-hidden whitespace-pre-wrap rounded-2xl text-sm leading-6 shadow-sm",
isAssistant
? "rounded-tl-sm bg-white/10 text-white/90"
: "rounded-tr-sm bg-white text-[#07101a]",
? "rounded-tl-sm bg-white/10 px-3.5 py-2.5 text-white/90"
: `rounded-tr-sm bg-white text-[#07101a] ${
message.attachments?.length ? "p-1" : "px-3.5 py-2.5"
}`,
].join(" ")}
>
{message.content || (message.streaming ? "…" : "")}
{message.attachments?.map((attachment) => (
// eslint-disable-next-line @next/next/no-img-element
<img
key={attachment.id}
src={attachment.url}
alt={attachment.alt}
className="max-h-[52dvh] w-full rounded-[0.8rem] object-cover"
/>
))}
{message.content && (
<div className={message.attachments?.length ? "px-2.5 py-2" : ""}>
{message.content}
</div>
)}
{!message.content && !message.attachments?.length && message.streaming
? "…"
: null}
</div>
</div>
);
@@ -166,7 +184,10 @@ function MobileCallVisualWorkspace({
} | null>(null);
const latestMessage = [...messages]
.reverse()
.find((message) => message.content.trim());
.find(
(message) =>
message.content.trim() || (message.attachments?.length ?? 0) > 0,
);
useEffect(() => {
const container = containerRef.current;
@@ -283,7 +304,10 @@ function MobileCallVisualWorkspace({
<span className="mr-1.5 font-medium text-white/55">
{latestMessage?.role === "user" ? "我:" : "助手:"}
</span>
{latestMessage?.content || "暂无消息,点击返回聊天记录"}
{latestMessage?.content ||
(latestMessage?.attachments?.length
? "发送了一张照片"
: "暂无消息,点击返回聊天记录")}
</span>
</button>
</>

View File

@@ -19,8 +19,64 @@ const PHOTO_BUTTON_DEFINITION = {
],
} as const;
const MAX_PREVIEW_EDGE = 1280;
async function capturePreviewImage(stream: MediaStream | null): Promise<string> {
const track = stream?.getVideoTracks()[0];
if (!stream || !track || track.readyState !== "live") {
throw new Error("当前没有可用的摄像头画面");
}
const video = document.createElement("video");
video.muted = true;
video.playsInline = true;
video.srcObject = stream;
try {
if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
await new Promise<void>((resolve, reject) => {
const timeout = window.setTimeout(
() => reject(new Error("等待摄像头预览超时")),
2_000,
);
video.addEventListener(
"loadeddata",
() => {
window.clearTimeout(timeout);
resolve();
},
{ once: true },
);
});
}
await video.play();
const sourceWidth = video.videoWidth || track.getSettings().width || 0;
const sourceHeight = video.videoHeight || track.getSettings().height || 0;
if (!sourceWidth || !sourceHeight) {
throw new Error("摄像头画面尺寸不可用");
}
const scale = Math.min(1, MAX_PREVIEW_EDGE / Math.max(sourceWidth, sourceHeight));
const canvas = document.createElement("canvas");
canvas.width = Math.max(1, Math.round(sourceWidth * scale));
canvas.height = Math.max(1, Math.round(sourceHeight * scale));
const context = canvas.getContext("2d");
if (!context) throw new Error("浏览器无法生成照片预览");
context.drawImage(video, 0, 0, canvas.width, canvas.height);
return canvas.toDataURL("image/jpeg", 0.85);
} finally {
video.pause();
video.srcObject = null;
}
}
export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
const { registerClientTool, sendUserInput, status } = preview;
const {
appendUserImage,
registerClientTool,
sendUserInput,
status,
videoStream,
} = preview;
const [visible, setVisible] = useState(false);
const [capturing, setCapturing] = useState(false);
const [error, setError] = useState<string | null>(null);
@@ -52,7 +108,9 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
setCapturing(true);
setError(null);
try {
await sendUserInput(
const timestamp = new Date().toISOString();
const imageUrl = await capturePreviewImage(videoStream);
const result = await sendUserInput(
[
{
type: "input_image",
@@ -61,6 +119,7 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
],
{ runImmediately: true, interrupt: true },
);
appendUserImage(result.inputId, imageUrl, timestamp);
} catch (captureError) {
setError(
captureError instanceof Error ? captureError.message : "拍照提交失败",
@@ -68,7 +127,7 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
} finally {
setCapturing(false);
}
}, [capturing, sendUserInput, status]);
}, [appendUserImage, capturing, sendUserInput, status, videoStream]);
return {
visible,

View File

@@ -45,6 +45,14 @@ export type ChatMessage = {
sequence: number;
turnId?: string;
streaming?: boolean;
attachments?: ChatAttachment[];
};
export type ChatAttachment = {
id: string;
type: "image";
url: string;
alt: string;
};
type AppMessage = Record<string, unknown> & { type?: string };
@@ -53,7 +61,9 @@ export type UserInputPart =
| { type: "input_text"; text: string }
| {
type: "input_image";
source: { type: "camera_frame"; frame: "current" };
source:
| { type: "camera_frame"; frame: "current" }
| { type: "uploaded_asset"; asset_token: string };
};
export type UserInputResult = {
inputId: string;
@@ -871,6 +881,39 @@ export function useVoicePreview(
[],
);
const appendUserImage = useCallback(
(
inputId: string,
imageUrl: string,
timestamp: string,
content = "",
) => {
messageSeqRef.current += 1;
const sequence = messageSeqRef.current;
setMessages((previous) =>
sortMessages([
...previous,
{
id: `user-image-${inputId}`,
role: "user",
content: content.trim(),
timestamp,
sequence,
attachments: [
{
id: `image-${inputId}`,
type: "image",
url: imageUrl,
alt: "用户提交的图片",
},
],
},
]),
);
},
[],
);
const updateSession = useCallback(
({
dynamicVariables,
@@ -962,6 +1005,7 @@ export function useVoicePreview(
supportsOutputSelection,
sendText,
sendUserInput,
appendUserImage,
updateSession,
registerClientTool,
connect,

View File

@@ -325,7 +325,24 @@ export type ConversationMessage = {
contentType: string;
content: string;
occurredAt: string;
extra: { interrupted?: boolean; turn_id?: string };
extra: {
interrupted?: boolean;
turn_id?: string;
source?: string;
node_id?: string;
input_id?: string;
};
artifacts: ConversationArtifact[];
};
export type ConversationArtifact = {
id: string;
kind: string;
contentUrl: string;
mimeType: string;
sizeBytes: number | null;
durationMs: number | null;
extra: Record<string, unknown>;
};
export type ConversationDetail = Conversation & {
@@ -372,6 +389,30 @@ export const conversationsApi = {
request<{ ok: boolean }>(`/api/conversations/${id}`, { method: "DELETE" }),
};
// ---------- 调试会话临时图片 ----------
export type InputImageAsset = {
assetToken: string;
width: number;
height: number;
sizeBytes: number;
};
export const inputAssetsApi = {
uploadImage: (file: File) => {
const body = new FormData();
body.append("file", file);
return request<InputImageAsset>("/api/input-assets/image", {
method: "POST",
body,
});
},
remove: (assetToken: string) =>
request<{ ok: boolean }>(
`/api/input-assets/${encodeURIComponent(assetToken)}`,
{ method: "DELETE" },
),
};
// ---------- 工具 ----------
export type ToolStatus = "active" | "archived" | "draft";
export type ToolExecutionMode = "immediate" | "async";