Compare commits
3 Commits
6348ae2af6
...
e36ca308b8
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e36ca308b8 | ||
|
|
555c8f5fa6 | ||
|
|
317a0600bc |
@@ -26,6 +26,7 @@ from routes import (
|
||||
auth,
|
||||
conversations,
|
||||
health,
|
||||
input_assets,
|
||||
knowledge_bases,
|
||||
mcp_servers,
|
||||
model_registry,
|
||||
@@ -60,6 +61,7 @@ app.add_middleware(
|
||||
app.include_router(health.router)
|
||||
app.include_router(auth.router)
|
||||
app.include_router(conversations.router)
|
||||
app.include_router(input_assets.router)
|
||||
app.include_router(assistants.router)
|
||||
app.include_router(knowledge_bases.router)
|
||||
app.include_router(mcp_servers.router)
|
||||
|
||||
@@ -6,7 +6,7 @@ pipecat-ai[webrtc,websocket,silero,openai,mcp]==1.7.0
|
||||
Pillow>=11.1.0,<13
|
||||
|
||||
# FastGPT 类型助手:本地 SDK(包 /api/v1/chat/completions 流式 + chatId 会话)
|
||||
fastgpt-client @ file:///Users/wangx/Code/AI-VideoAssistant-Project/fastgpt-python-sdk
|
||||
# fastgpt-client @ file:///Users/wangx/Code/AI-VideoAssistant-Project/fastgpt-python-sdk
|
||||
|
||||
# Dify 类型助手:异步流式 Runtime API SDK
|
||||
dify-client-python==1.0.3
|
||||
|
||||
@@ -1,15 +1,20 @@
|
||||
"""对话历史查询 API。"""
|
||||
|
||||
from db.models import ConversationMessage, ConversationSession
|
||||
import asyncio
|
||||
|
||||
from db.models import ConversationArtifact, ConversationMessage, ConversationSession
|
||||
from db.session import get_session
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Response
|
||||
from loguru import logger
|
||||
from schemas import (
|
||||
ConversationArtifactOut,
|
||||
ConversationDetailOut,
|
||||
ConversationListOut,
|
||||
ConversationMessageOut,
|
||||
ConversationOut,
|
||||
)
|
||||
from services.auth import require_admin
|
||||
from services.object_storage import delete_object, get_object, key_from_storage_uri
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
@@ -100,6 +105,17 @@ async def get_conversation(
|
||||
.order_by(ConversationMessage.sequence)
|
||||
)
|
||||
).scalars().all()
|
||||
artifacts = (
|
||||
await session.execute(
|
||||
select(ConversationArtifact)
|
||||
.where(ConversationArtifact.session_id == conversation_id)
|
||||
.order_by(ConversationArtifact.created_at)
|
||||
)
|
||||
).scalars().all()
|
||||
artifacts_by_message: dict[str, list[ConversationArtifact]] = {}
|
||||
for artifact in artifacts:
|
||||
if artifact.message_id:
|
||||
artifacts_by_message.setdefault(artifact.message_id, []).append(artifact)
|
||||
return ConversationDetailOut(
|
||||
**_session_out(conversation).model_dump(),
|
||||
extra=conversation.extra or {},
|
||||
@@ -112,12 +128,52 @@ async def get_conversation(
|
||||
content=message.content,
|
||||
occurred_at=message.occurred_at,
|
||||
extra=message.extra or {},
|
||||
artifacts=[
|
||||
ConversationArtifactOut(
|
||||
id=artifact.id,
|
||||
kind=artifact.kind,
|
||||
content_url=(
|
||||
f"/api/conversations/{conversation_id}/artifacts/"
|
||||
f"{artifact.id}/content"
|
||||
),
|
||||
mime_type=artifact.mime_type,
|
||||
size_bytes=artifact.size_bytes,
|
||||
duration_ms=artifact.duration_ms,
|
||||
extra=artifact.extra or {},
|
||||
)
|
||||
for artifact in artifacts_by_message.get(message.id, [])
|
||||
],
|
||||
)
|
||||
for message in messages
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{conversation_id}/artifacts/{artifact_id}/content")
|
||||
async def get_conversation_artifact(
|
||||
conversation_id: str,
|
||||
artifact_id: str,
|
||||
session: AsyncSession = Depends(get_session),
|
||||
):
|
||||
artifact = await session.get(ConversationArtifact, artifact_id)
|
||||
if not artifact or artifact.session_id != conversation_id:
|
||||
raise HTTPException(404, "会话附件不存在")
|
||||
try:
|
||||
key = key_from_storage_uri(artifact.storage_uri)
|
||||
data = await asyncio.to_thread(get_object, key)
|
||||
except Exception as exc:
|
||||
logger.warning(f"读取会话附件失败: {artifact_id}: {exc}")
|
||||
raise HTTPException(404, "会话附件不可用") from exc
|
||||
return Response(
|
||||
content=data,
|
||||
media_type=artifact.mime_type or "application/octet-stream",
|
||||
headers={
|
||||
"Cache-Control": "private, max-age=300",
|
||||
"Content-Disposition": f'inline; filename="{artifact.id}"',
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{conversation_id}")
|
||||
async def delete_conversation(
|
||||
conversation_id: str,
|
||||
@@ -126,6 +182,19 @@ async def delete_conversation(
|
||||
conversation = await session.get(ConversationSession, conversation_id)
|
||||
if not conversation:
|
||||
raise HTTPException(404, "对话记录不存在")
|
||||
storage_uris = (
|
||||
await session.execute(
|
||||
select(ConversationArtifact.storage_uri).where(
|
||||
ConversationArtifact.session_id == conversation_id
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
await session.delete(conversation)
|
||||
await session.commit()
|
||||
for storage_uri in storage_uris:
|
||||
try:
|
||||
key = key_from_storage_uri(storage_uri)
|
||||
await asyncio.to_thread(delete_object, key)
|
||||
except Exception as exc:
|
||||
logger.warning(f"清理已删除会话的附件失败: {storage_uri}: {exc}")
|
||||
return {"ok": True}
|
||||
|
||||
59
backend/routes/input_assets.py
Normal file
59
backend/routes/input_assets.py
Normal file
@@ -0,0 +1,59 @@
|
||||
"""Authenticated upload endpoint for pending debug-chat image attachments."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||
from pydantic import BaseModel
|
||||
|
||||
import settings
|
||||
from services.auth import require_admin
|
||||
from services.input_assets import discard_input_image, store_input_image
|
||||
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/input-assets",
|
||||
tags=["input-assets"],
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
|
||||
|
||||
class InputImageAssetOut(BaseModel):
|
||||
assetToken: str
|
||||
width: int
|
||||
height: int
|
||||
sizeBytes: int
|
||||
|
||||
|
||||
@router.post("/image", response_model=InputImageAssetOut)
|
||||
async def upload_input_image(file: UploadFile = File(...)):
|
||||
data = await file.read(settings.INPUT_IMAGE_MAX_BYTES + 1)
|
||||
if len(data) > settings.INPUT_IMAGE_MAX_BYTES:
|
||||
limit_mb = settings.INPUT_IMAGE_MAX_BYTES // 1024 // 1024
|
||||
raise HTTPException(413, f"图片不能超过 {limit_mb} MB")
|
||||
if not data:
|
||||
raise HTTPException(400, "图片文件不能为空")
|
||||
try:
|
||||
stored = await asyncio.to_thread(store_input_image, data)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
except Exception as exc:
|
||||
raise HTTPException(503, "图片暂时无法上传,请确认对象存储可用") from exc
|
||||
return InputImageAssetOut(
|
||||
assetToken=stored.token,
|
||||
width=stored.width,
|
||||
height=stored.height,
|
||||
sizeBytes=stored.size_bytes,
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{asset_token}")
|
||||
async def delete_input_image(asset_token: str):
|
||||
try:
|
||||
await asyncio.to_thread(discard_input_image, asset_token)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
except Exception as exc:
|
||||
raise HTTPException(503, "图片附件暂时无法清理") from exc
|
||||
return {"ok": True}
|
||||
@@ -467,6 +467,16 @@ class ModelResourceTestResult(CamelModel):
|
||||
|
||||
|
||||
# ---------- 对话历史 ----------
|
||||
class ConversationArtifactOut(CamelModel):
|
||||
id: str
|
||||
kind: str
|
||||
content_url: str
|
||||
mime_type: str
|
||||
size_bytes: int | None = None
|
||||
duration_ms: int | None = None
|
||||
extra: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ConversationMessageOut(CamelModel):
|
||||
id: str
|
||||
sequence: int
|
||||
@@ -475,6 +485,7 @@ class ConversationMessageOut(CamelModel):
|
||||
content: str
|
||||
occurred_at: datetime
|
||||
extra: dict[str, Any] = Field(default_factory=dict)
|
||||
artifacts: list[ConversationArtifactOut] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ConversationOut(CamelModel):
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Awaitable
|
||||
from time import monotonic
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
@@ -503,12 +504,45 @@ class PromptBrain(BaseBrain):
|
||||
await params.result_callback(result)
|
||||
|
||||
async def call_tool(params: FunctionCallParams) -> None:
|
||||
invocation_id = f"tool_{uuid4().hex[:20]}"
|
||||
started_at = monotonic()
|
||||
await self._emit_trace(
|
||||
"tool_started",
|
||||
invocationId=invocation_id,
|
||||
toolId=tool.id,
|
||||
toolName=tool.name,
|
||||
functionName=tool.function_name,
|
||||
toolType=tool.type,
|
||||
)
|
||||
try:
|
||||
result = await self._tools.execute(tool, dict(params.arguments or {}))
|
||||
if result["updated_variables"]:
|
||||
self._refresh_prompt()
|
||||
await self._emit_trace(
|
||||
"tool_completed" if result.get("status") == "ok" else "tool_failed",
|
||||
invocationId=invocation_id,
|
||||
toolId=tool.id,
|
||||
toolName=tool.name,
|
||||
functionName=tool.function_name,
|
||||
toolType=tool.type,
|
||||
status=str(result.get("status") or "unknown"),
|
||||
durationMs=max(0, round((monotonic() - started_at) * 1000)),
|
||||
updatedVariables=list(result.get("updated_variables") or []),
|
||||
resultKeys=sorted(str(name) for name in result if name != "data"),
|
||||
)
|
||||
await return_result(params, result)
|
||||
except (ToolExecutionError, ValueError) as exc:
|
||||
await self._emit_trace(
|
||||
"tool_failed",
|
||||
invocationId=invocation_id,
|
||||
toolId=tool.id,
|
||||
toolName=tool.name,
|
||||
functionName=tool.function_name,
|
||||
toolType=tool.type,
|
||||
status="error",
|
||||
durationMs=max(0, round((monotonic() - started_at) * 1000)),
|
||||
error=str(exc)[:2048],
|
||||
)
|
||||
await return_result(
|
||||
params,
|
||||
{"status": "error", "message": f"工具调用失败: {exc}"},
|
||||
@@ -522,6 +556,26 @@ class PromptBrain(BaseBrain):
|
||||
)
|
||||
return schema, call_tool
|
||||
|
||||
async def _emit_trace(self, event: str, **details: Any) -> None:
|
||||
runtime = self._runtime
|
||||
if runtime is None:
|
||||
return
|
||||
try:
|
||||
await runtime.queue_frame(
|
||||
OutputTransportMessageUrgentFrame(
|
||||
message={
|
||||
"type": "workflow-event",
|
||||
"eventId": f"wfe_{uuid4().hex[:20]}",
|
||||
"event": event,
|
||||
"timestamp": time_now_iso8601(),
|
||||
"sessionId": runtime.session_id,
|
||||
**details,
|
||||
}
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - trace must not alter execution
|
||||
logger.warning(f"发送 Prompt 工具轨迹失败,不影响当前调用: {exc}")
|
||||
|
||||
def _make_end_call_tool(self, tool, runtime: BrainRuntime):
|
||||
config = (tool.definition or {}).get("config") or {}
|
||||
message_type = str(config.get("message_type") or "none")
|
||||
|
||||
@@ -6,7 +6,9 @@ import asyncio
|
||||
from collections.abc import Awaitable
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass, replace
|
||||
from time import monotonic
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from loguru import logger
|
||||
from models import AssistantConfig, RuntimeTool
|
||||
@@ -764,10 +766,46 @@ class WorkflowBrain(BaseBrain):
|
||||
|
||||
async def handler(args, _flow_manager):
|
||||
transition_id = self._state.transition_id
|
||||
invocation_id = f"tool_{uuid4().hex[:20]}"
|
||||
started_at = monotonic()
|
||||
await self._emit_trace(
|
||||
"tool_started",
|
||||
nodeId=node_id,
|
||||
invocationId=invocation_id,
|
||||
toolId=tool.id,
|
||||
toolName=tool.name,
|
||||
functionName=tool.function_name,
|
||||
toolType=tool.type,
|
||||
)
|
||||
try:
|
||||
result = await self._tools.execute(tool, dict(args or {}))
|
||||
except ToolExecutionError as exc:
|
||||
await self._emit_trace(
|
||||
"tool_failed",
|
||||
nodeId=node_id,
|
||||
invocationId=invocation_id,
|
||||
toolId=tool.id,
|
||||
toolName=tool.name,
|
||||
functionName=tool.function_name,
|
||||
toolType=tool.type,
|
||||
status="error",
|
||||
durationMs=max(0, round((monotonic() - started_at) * 1000)),
|
||||
error=str(exc)[:2048],
|
||||
)
|
||||
return {"status": "error", "message": str(exc)}
|
||||
await self._emit_trace(
|
||||
"tool_completed" if result.get("status") == "ok" else "tool_failed",
|
||||
nodeId=node_id,
|
||||
invocationId=invocation_id,
|
||||
toolId=tool.id,
|
||||
toolName=tool.name,
|
||||
functionName=tool.function_name,
|
||||
toolType=tool.type,
|
||||
status=str(result.get("status") or "unknown"),
|
||||
durationMs=max(0, round((monotonic() - started_at) * 1000)),
|
||||
updatedVariables=list(result.get("updated_variables") or []),
|
||||
resultKeys=sorted(str(name) for name in result if name != "data"),
|
||||
)
|
||||
if (
|
||||
self._state.current_node_id != node_id
|
||||
or self._state.transition_id != transition_id
|
||||
|
||||
@@ -7,6 +7,7 @@ from collections import deque
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal, Protocol
|
||||
from uuid import uuid4
|
||||
|
||||
from loguru import logger
|
||||
from pipecat.frames.frames import (
|
||||
@@ -17,6 +18,7 @@ from pipecat.frames.frames import (
|
||||
StopFrame,
|
||||
)
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
from pipecat.utils.time import time_now_iso8601
|
||||
|
||||
|
||||
class ClientToolError(RuntimeError):
|
||||
@@ -29,6 +31,7 @@ ClientToolResponseWaitMode = Literal["timeout", "session"]
|
||||
@dataclass(frozen=True)
|
||||
class _PendingClientToolCall:
|
||||
future: asyncio.Future[dict[str, Any]]
|
||||
function_name: str
|
||||
interrupt_on_result: bool = False
|
||||
|
||||
|
||||
@@ -80,8 +83,6 @@ class ClientToolBroker(FrameProcessor):
|
||||
response_wait_mode: ClientToolResponseWaitMode = "timeout",
|
||||
interrupt_on_result: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
from uuid import uuid4
|
||||
|
||||
if self._closed_message is not None:
|
||||
raise ClientToolError(self._closed_message)
|
||||
|
||||
@@ -99,9 +100,21 @@ class ClientToolBroker(FrameProcessor):
|
||||
OutputTransportMessageUrgentFrame(message=message)
|
||||
)
|
||||
except Exception as exc:
|
||||
await self._emit_trace(
|
||||
"client_tool_failed",
|
||||
toolCallId=tool_call_id,
|
||||
functionName=function_name,
|
||||
error="客户端工具调用发送失败",
|
||||
)
|
||||
raise ClientToolError(
|
||||
f"客户端工具调用发送失败: {function_name}"
|
||||
) from exc
|
||||
await self._emit_trace(
|
||||
"client_tool_completed",
|
||||
toolCallId=tool_call_id,
|
||||
functionName=function_name,
|
||||
status="dispatched",
|
||||
)
|
||||
return {
|
||||
"status": "ok",
|
||||
"data": {"dispatched": True},
|
||||
@@ -111,6 +124,7 @@ class ClientToolBroker(FrameProcessor):
|
||||
future: asyncio.Future[dict[str, Any]] = loop.create_future()
|
||||
self._pending[tool_call_id] = _PendingClientToolCall(
|
||||
future=future,
|
||||
function_name=function_name,
|
||||
interrupt_on_result=interrupt_on_result,
|
||||
)
|
||||
try:
|
||||
@@ -121,6 +135,12 @@ class ClientToolBroker(FrameProcessor):
|
||||
return await future
|
||||
return await asyncio.wait_for(future, timeout=timeout_seconds)
|
||||
except TimeoutError as exc:
|
||||
await self._emit_trace(
|
||||
"client_tool_failed",
|
||||
toolCallId=tool_call_id,
|
||||
functionName=function_name,
|
||||
error="客户端工具调用超时",
|
||||
)
|
||||
raise ClientToolError(f"客户端工具调用超时: {function_name}") from exc
|
||||
except ClientToolError:
|
||||
raise
|
||||
@@ -163,10 +183,28 @@ class ClientToolBroker(FrameProcessor):
|
||||
|
||||
future = pending.future
|
||||
status = str(message.get("status") or "error")
|
||||
result_data = message.get("data")
|
||||
user_action = (
|
||||
str(result_data.get("action") or "")
|
||||
if isinstance(result_data, dict)
|
||||
else ""
|
||||
)
|
||||
await self._emit_trace(
|
||||
"client_tool_completed" if status == "ok" else "client_tool_failed",
|
||||
toolCallId=tool_call_id,
|
||||
functionName=pending.function_name,
|
||||
status=status,
|
||||
**({"userAction": user_action} if user_action else {}),
|
||||
**(
|
||||
{"error": str(message.get("message") or "客户端工具执行失败")}
|
||||
if status != "ok"
|
||||
else {}
|
||||
),
|
||||
)
|
||||
if status == "ok":
|
||||
response = {
|
||||
"status": "ok",
|
||||
"data": message.get("data"),
|
||||
"data": result_data,
|
||||
}
|
||||
if pending.interrupt_on_result:
|
||||
# Match text input exactly: broadcasting only starts the
|
||||
@@ -206,6 +244,23 @@ class ClientToolBroker(FrameProcessor):
|
||||
}
|
||||
)
|
||||
|
||||
async def _emit_trace(self, event: str, **details: Any) -> None:
|
||||
"""Publish sanitized client interaction metadata for conversation history."""
|
||||
try:
|
||||
await self.push_frame(
|
||||
OutputTransportMessageUrgentFrame(
|
||||
message={
|
||||
"type": "workflow-event",
|
||||
"eventId": f"wfe_{uuid4().hex[:20]}",
|
||||
"event": event,
|
||||
"timestamp": time_now_iso8601(),
|
||||
**details,
|
||||
}
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - tracing must not alter the tool call
|
||||
logger.warning(f"发送客户端工具轨迹失败,不影响工具调用: {exc}")
|
||||
|
||||
def on_interruption_processed(self) -> None:
|
||||
"""Release one result after the aggregator acknowledges interruption."""
|
||||
while self._deferred_results:
|
||||
|
||||
@@ -9,9 +9,10 @@ from copy import deepcopy
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from db.models import ConversationMessage, ConversationSession
|
||||
from db.models import ConversationArtifact, ConversationMessage, ConversationSession
|
||||
from db.session import SessionLocal
|
||||
from loguru import logger
|
||||
from services.object_storage import delete_object, put_object, storage_uri
|
||||
|
||||
|
||||
MAX_WORKFLOW_TRACE_EVENTS = 1000
|
||||
@@ -36,6 +37,7 @@ class ConversationRecorder:
|
||||
self._trace_sequence = 0
|
||||
self._lock = asyncio.Lock()
|
||||
self._seen_events: set[str] = set()
|
||||
self._pending_artifacts: set[asyncio.Task[None]] = set()
|
||||
|
||||
@classmethod
|
||||
async def start(
|
||||
@@ -168,6 +170,95 @@ class ConversationRecorder:
|
||||
except Exception as exc:
|
||||
logger.error(f"保存对话文本失败,不影响本次通话: {exc}")
|
||||
|
||||
def record_image_later(
|
||||
self,
|
||||
data: bytes,
|
||||
*,
|
||||
input_id: str,
|
||||
timestamp: object,
|
||||
content: str = "",
|
||||
source: str = "camera_capture",
|
||||
mime_type: str = "image/jpeg",
|
||||
) -> None:
|
||||
"""Persist a captured frame without delaying the realtime model turn."""
|
||||
|
||||
task = asyncio.create_task(
|
||||
self._record_image(
|
||||
data,
|
||||
input_id=input_id,
|
||||
timestamp=timestamp,
|
||||
content=content,
|
||||
source=source,
|
||||
mime_type=mime_type,
|
||||
),
|
||||
name=f"conversation-image:{self.session_id}:{input_id}",
|
||||
)
|
||||
self._pending_artifacts.add(task)
|
||||
task.add_done_callback(self._pending_artifacts.discard)
|
||||
|
||||
async def _record_image(
|
||||
self,
|
||||
data: bytes,
|
||||
*,
|
||||
input_id: str,
|
||||
timestamp: object,
|
||||
content: str,
|
||||
source: str,
|
||||
mime_type: str,
|
||||
) -> None:
|
||||
message_id = f"msg_{uuid4().hex[:20]}"
|
||||
artifact_id = f"artifact_{uuid4().hex[:20]}"
|
||||
extension = ".jpg" if mime_type == "image/jpeg" else ".bin"
|
||||
key = f"conversations/{self.session_id}/{artifact_id}{extension}"
|
||||
try:
|
||||
await asyncio.to_thread(put_object, key, data, mime_type)
|
||||
except Exception as exc:
|
||||
logger.error(f"保存会话图片到对象存储失败,不影响本次通话: {exc}")
|
||||
return
|
||||
|
||||
async with self._lock:
|
||||
next_sequence = self._sequence + 1
|
||||
try:
|
||||
async with SessionLocal() as db:
|
||||
db.add(
|
||||
ConversationMessage(
|
||||
id=message_id,
|
||||
session_id=self.session_id,
|
||||
sequence=next_sequence,
|
||||
role="user",
|
||||
content_type="image",
|
||||
content=content.strip(),
|
||||
occurred_at=_parse_timestamp(timestamp),
|
||||
extra={
|
||||
"input_id": input_id,
|
||||
"source": source,
|
||||
},
|
||||
)
|
||||
)
|
||||
db.add(
|
||||
ConversationArtifact(
|
||||
id=artifact_id,
|
||||
session_id=self.session_id,
|
||||
message_id=message_id,
|
||||
kind="image",
|
||||
storage_uri=storage_uri(key),
|
||||
mime_type=mime_type,
|
||||
size_bytes=len(data),
|
||||
extra={"input_id": input_id, "source": source},
|
||||
)
|
||||
)
|
||||
conversation = await db.get(ConversationSession, self.session_id)
|
||||
if conversation:
|
||||
conversation.message_count = next_sequence
|
||||
await db.commit()
|
||||
self._sequence = next_sequence
|
||||
except Exception as exc:
|
||||
logger.error(f"保存会话图片索引失败,不影响本次通话: {exc}")
|
||||
try:
|
||||
await asyncio.to_thread(delete_object, key)
|
||||
except Exception:
|
||||
logger.warning(f"清理未关联的会话图片失败: {key}")
|
||||
|
||||
async def finish(self, *, status: str = "completed") -> None:
|
||||
"""Finish the session even if the owning pipeline is being cancelled."""
|
||||
finish_task = asyncio.create_task(
|
||||
@@ -184,6 +275,9 @@ class ConversationRecorder:
|
||||
raise
|
||||
|
||||
async def _finish(self, *, status: str) -> None:
|
||||
pending_artifacts = list(self._pending_artifacts)
|
||||
if pending_artifacts:
|
||||
await asyncio.gather(*pending_artifacts, return_exceptions=True)
|
||||
async with self._lock:
|
||||
try:
|
||||
async with SessionLocal() as db:
|
||||
|
||||
132
backend/services/input_assets.py
Normal file
132
backend/services/input_assets.py
Normal file
@@ -0,0 +1,132 @@
|
||||
"""Short-lived uploaded images consumed by the realtime user-input protocol."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from io import BytesIO
|
||||
from uuid import uuid4
|
||||
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
import settings
|
||||
from services.object_storage import delete_object, get_object, put_object
|
||||
|
||||
|
||||
INPUT_ASSET_PREFIX = "conversation-inputs/"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StoredInputImage:
|
||||
token: str
|
||||
width: int
|
||||
height: int
|
||||
size_bytes: int
|
||||
|
||||
|
||||
def _b64encode(data: bytes) -> str:
|
||||
return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii")
|
||||
|
||||
|
||||
def _b64decode(data: str) -> bytes:
|
||||
padding = "=" * (-len(data) % 4)
|
||||
return base64.urlsafe_b64decode(f"{data}{padding}".encode("ascii"))
|
||||
|
||||
|
||||
def _signature(payload: str) -> str:
|
||||
digest = hmac.new(
|
||||
settings.AUTH_SECRET_KEY.encode("utf-8"),
|
||||
payload.encode("ascii"),
|
||||
hashlib.sha256,
|
||||
).digest()
|
||||
return _b64encode(digest)
|
||||
|
||||
|
||||
def _normalize_image(data: bytes) -> tuple[bytes, int, int]:
|
||||
try:
|
||||
source = Image.open(BytesIO(data))
|
||||
width, height = source.size
|
||||
if width <= 0 or height <= 0:
|
||||
raise ValueError("图片尺寸无效")
|
||||
if width * height > settings.INPUT_IMAGE_MAX_PIXELS:
|
||||
raise ValueError("图片像素尺寸过大")
|
||||
source.load()
|
||||
except (UnidentifiedImageError, OSError) as exc:
|
||||
raise ValueError("文件不是可识别的图片") from exc
|
||||
|
||||
image = ImageOps.exif_transpose(source)
|
||||
image.thumbnail(
|
||||
(settings.INPUT_IMAGE_MAX_EDGE, settings.INPUT_IMAGE_MAX_EDGE),
|
||||
Image.Resampling.LANCZOS,
|
||||
)
|
||||
if image.mode in {"RGBA", "LA"} or "transparency" in image.info:
|
||||
rgba = image.convert("RGBA")
|
||||
background = Image.new("RGB", rgba.size, "white")
|
||||
background.paste(rgba, mask=rgba.getchannel("A"))
|
||||
image = background
|
||||
else:
|
||||
image = image.convert("RGB")
|
||||
|
||||
buffer = BytesIO()
|
||||
image.save(buffer, format="JPEG", quality=85, optimize=True)
|
||||
return buffer.getvalue(), image.width, image.height
|
||||
|
||||
|
||||
def store_input_image(data: bytes) -> StoredInputImage:
|
||||
normalized, width, height = _normalize_image(data)
|
||||
key = f"{INPUT_ASSET_PREFIX}{uuid4().hex}.jpg"
|
||||
put_object(key, normalized, "image/jpeg")
|
||||
expires_at = int(time.time()) + settings.INPUT_IMAGE_TOKEN_TTL_SECONDS
|
||||
encoded_payload = _b64encode(
|
||||
json.dumps(
|
||||
{"key": key, "exp": expires_at},
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
)
|
||||
token = f"{encoded_payload}.{_signature(encoded_payload)}"
|
||||
return StoredInputImage(
|
||||
token=token,
|
||||
width=width,
|
||||
height=height,
|
||||
size_bytes=len(normalized),
|
||||
)
|
||||
|
||||
|
||||
def _key_from_token(token: str) -> str:
|
||||
if not token or "." not in token:
|
||||
raise ValueError("图片附件 token 无效")
|
||||
encoded_payload, signature = token.rsplit(".", 1)
|
||||
if not hmac.compare_digest(_signature(encoded_payload), signature):
|
||||
raise ValueError("图片附件 token 签名无效")
|
||||
try:
|
||||
payload = json.loads(_b64decode(encoded_payload))
|
||||
key = str(payload.get("key") or "")
|
||||
expires_at = int(payload.get("exp") or 0)
|
||||
except (ValueError, TypeError, json.JSONDecodeError) as exc:
|
||||
raise ValueError("图片附件 token 内容无效") from exc
|
||||
if expires_at < int(time.time()):
|
||||
raise ValueError("图片附件已过期,请重新添加")
|
||||
if not key.startswith(INPUT_ASSET_PREFIX):
|
||||
raise ValueError("图片附件存储位置无效")
|
||||
return key
|
||||
|
||||
|
||||
def consume_input_image(token: str) -> bytes:
|
||||
"""Read a signed image once and remove its temporary object."""
|
||||
|
||||
key = _key_from_token(token)
|
||||
data = get_object(key)
|
||||
try:
|
||||
delete_object(key)
|
||||
except Exception:
|
||||
# Consumption succeeded. A stale temporary object must not break the turn.
|
||||
pass
|
||||
return data
|
||||
|
||||
|
||||
def discard_input_image(token: str) -> None:
|
||||
delete_object(_key_from_token(token))
|
||||
@@ -6,9 +6,6 @@ from pathlib import Path
|
||||
import re
|
||||
import uuid
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from docx import Document as DocxDocument
|
||||
from openai import AsyncOpenAI
|
||||
from pypdf import PdfReader
|
||||
@@ -17,38 +14,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
import settings
|
||||
from db.models import KnowledgeBase, KnowledgeChunk, KnowledgeDocument, ModelResource
|
||||
|
||||
|
||||
def _s3_client():
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.S3_ENDPOINT_URL,
|
||||
aws_access_key_id=settings.S3_ACCESS_KEY,
|
||||
aws_secret_access_key=settings.S3_SECRET_KEY,
|
||||
region_name=settings.S3_REGION,
|
||||
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
|
||||
)
|
||||
|
||||
|
||||
def _ensure_bucket_and_put(key: str, data: bytes, mime_type: str) -> None:
|
||||
client = _s3_client()
|
||||
try:
|
||||
client.head_bucket(Bucket=settings.S3_BUCKET)
|
||||
except ClientError as exc:
|
||||
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
||||
if status != 404:
|
||||
raise
|
||||
client.create_bucket(Bucket=settings.S3_BUCKET)
|
||||
client.put_object(Bucket=settings.S3_BUCKET, Key=key, Body=data, ContentType=mime_type)
|
||||
|
||||
|
||||
def _delete_object(key: str) -> None:
|
||||
_s3_client().delete_object(Bucket=settings.S3_BUCKET, Key=key)
|
||||
|
||||
|
||||
def _get_object(key: str) -> bytes:
|
||||
response = _s3_client().get_object(Bucket=settings.S3_BUCKET, Key=key)
|
||||
return response["Body"].read()
|
||||
from services.object_storage import delete_object, get_object, put_object
|
||||
|
||||
|
||||
def extract_text(filename: str, data: bytes) -> str:
|
||||
@@ -122,7 +88,7 @@ async def create_document(
|
||||
safe_name = Path(name).name
|
||||
extension = ".txt" if source_type == "text" else Path(safe_name).suffix
|
||||
storage_key = f"knowledge/{kb.id}/{document_id}/source{extension}"
|
||||
await asyncio.to_thread(_ensure_bucket_and_put, storage_key, raw_data, mime_type)
|
||||
await asyncio.to_thread(put_object, storage_key, raw_data, mime_type)
|
||||
|
||||
document = KnowledgeDocument(
|
||||
id=document_id,
|
||||
@@ -158,7 +124,7 @@ async def process_document(document_id: str) -> None:
|
||||
await session.commit()
|
||||
|
||||
try:
|
||||
data = await asyncio.to_thread(_get_object, document.storage_key)
|
||||
data = await asyncio.to_thread(get_object, document.storage_key)
|
||||
text = (
|
||||
data.decode("utf-8", errors="replace")
|
||||
if document.source_type == "text"
|
||||
@@ -245,4 +211,4 @@ async def search(
|
||||
|
||||
async def delete_storage_object(document: KnowledgeDocument) -> None:
|
||||
if document.storage_key:
|
||||
await asyncio.to_thread(_delete_object, document.storage_key)
|
||||
await asyncio.to_thread(delete_object, document.storage_key)
|
||||
|
||||
60
backend/services/object_storage.py
Normal file
60
backend/services/object_storage.py
Normal file
@@ -0,0 +1,60 @@
|
||||
"""Small S3-compatible object-storage adapter shared by backend features."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
import settings
|
||||
|
||||
|
||||
def _client():
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.S3_ENDPOINT_URL,
|
||||
aws_access_key_id=settings.S3_ACCESS_KEY,
|
||||
aws_secret_access_key=settings.S3_SECRET_KEY,
|
||||
region_name=settings.S3_REGION,
|
||||
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
|
||||
)
|
||||
|
||||
|
||||
def put_object(key: str, data: bytes, mime_type: str) -> None:
|
||||
client = _client()
|
||||
try:
|
||||
client.head_bucket(Bucket=settings.S3_BUCKET)
|
||||
except ClientError as exc:
|
||||
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
||||
if status != 404:
|
||||
raise
|
||||
client.create_bucket(Bucket=settings.S3_BUCKET)
|
||||
client.put_object(
|
||||
Bucket=settings.S3_BUCKET,
|
||||
Key=key,
|
||||
Body=data,
|
||||
ContentType=mime_type,
|
||||
)
|
||||
|
||||
|
||||
def get_object(key: str) -> bytes:
|
||||
response = _client().get_object(Bucket=settings.S3_BUCKET, Key=key)
|
||||
return response["Body"].read()
|
||||
|
||||
|
||||
def delete_object(key: str) -> None:
|
||||
_client().delete_object(Bucket=settings.S3_BUCKET, Key=key)
|
||||
|
||||
|
||||
def storage_uri(key: str) -> str:
|
||||
return f"s3://{settings.S3_BUCKET}/{key.lstrip('/')}"
|
||||
|
||||
|
||||
def key_from_storage_uri(uri: str) -> str:
|
||||
prefix = f"s3://{settings.S3_BUCKET}/"
|
||||
if not uri.startswith(prefix):
|
||||
raise ValueError("不支持的会话附件存储地址")
|
||||
key = uri[len(prefix) :].strip("/")
|
||||
if not key:
|
||||
raise ValueError("会话附件存储地址缺少对象键")
|
||||
return key
|
||||
@@ -27,6 +27,7 @@ from services.pipecat.service_factory import (
|
||||
)
|
||||
from db.session import SessionLocal
|
||||
from services.knowledge import search as search_knowledge
|
||||
from services.input_assets import consume_input_image
|
||||
from services.client_tools import ClientToolBroker
|
||||
from services.tool_policy import policy_for_tool
|
||||
from services.vision import (
|
||||
@@ -36,6 +37,8 @@ from services.vision import (
|
||||
config_with_main_llm_as_vision,
|
||||
config_with_vision_resource,
|
||||
image_data_uri,
|
||||
image_frame_from_jpeg,
|
||||
image_jpeg_bytes,
|
||||
)
|
||||
from services.workflow_engine import WorkflowEngine
|
||||
|
||||
@@ -58,6 +61,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
|
||||
)
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
from pipecat.services.llm_service import FunctionCallParams
|
||||
from pipecat.utils.time import time_now_iso8601
|
||||
from services.pipecat.turn_config import (
|
||||
ConfigurableLLMUserAggregator,
|
||||
create_user_turn_strategies,
|
||||
@@ -732,7 +736,7 @@ async def run_pipeline(
|
||||
greeting = await brain.greeting(cfg)
|
||||
|
||||
async def submit_user_input(value: UserInput) -> None:
|
||||
if not value.has_camera_frame:
|
||||
if not value.has_image:
|
||||
if not value.run_immediately:
|
||||
brain.record_user_message(value.text)
|
||||
await worker.queue_frame(
|
||||
@@ -747,21 +751,50 @@ async def run_pipeline(
|
||||
raise ValueError("P0 图片输入必须立即触发回复")
|
||||
if not vision_enabled:
|
||||
raise ValueError("当前助手未启用视觉能力")
|
||||
user_id = vision_state.get("client_id")
|
||||
if not user_id:
|
||||
raise ValueError("当前没有可用的摄像头视频流")
|
||||
native_vision = active_vision_uses_main_llm()
|
||||
analysis_cfg = None if native_vision else active_vision_config()
|
||||
|
||||
request = UserImageRequestFrame(
|
||||
user_id=user_id,
|
||||
text=value.prompt_text,
|
||||
append_to_context=False,
|
||||
)
|
||||
try:
|
||||
image_frame = await vision_capture.request_image(llm, request)
|
||||
except asyncio.TimeoutError as exc:
|
||||
raise ValueError("等待摄像头视频帧超时") from exc
|
||||
if value.image_asset_token:
|
||||
try:
|
||||
image_bytes = await asyncio.to_thread(
|
||||
consume_input_image,
|
||||
value.image_asset_token,
|
||||
)
|
||||
image_frame = await asyncio.to_thread(
|
||||
image_frame_from_jpeg,
|
||||
image_bytes,
|
||||
)
|
||||
except ValueError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise ValueError("上传图片暂时无法读取") from exc
|
||||
else:
|
||||
user_id = vision_state.get("client_id")
|
||||
if not user_id:
|
||||
raise ValueError("当前没有可用的摄像头视频流")
|
||||
request = UserImageRequestFrame(
|
||||
user_id=user_id,
|
||||
text=value.prompt_text,
|
||||
append_to_context=False,
|
||||
)
|
||||
try:
|
||||
image_frame = await vision_capture.request_image(llm, request)
|
||||
except asyncio.TimeoutError as exc:
|
||||
raise ValueError("等待摄像头视频帧超时") from exc
|
||||
image_bytes = await asyncio.to_thread(image_jpeg_bytes, image_frame)
|
||||
|
||||
if recorder:
|
||||
recorder.record_image_later(
|
||||
image_bytes,
|
||||
input_id=value.input_id,
|
||||
timestamp=time_now_iso8601(),
|
||||
content=value.text,
|
||||
source=(
|
||||
"uploaded_asset"
|
||||
if value.image_asset_token
|
||||
else "camera_capture"
|
||||
),
|
||||
)
|
||||
|
||||
if native_vision:
|
||||
input_frame = await asyncio.to_thread(
|
||||
|
||||
@@ -149,11 +149,12 @@ def bind_cascade_pipeline_events(
|
||||
|
||||
@text_input.event_handler("on_user_input")
|
||||
async def on_user_input(_processor, user_input: UserInput):
|
||||
await queue_transcript(
|
||||
"user",
|
||||
user_input.transcript_text,
|
||||
time_now_iso8601(),
|
||||
)
|
||||
if not user_input.has_image:
|
||||
await queue_transcript(
|
||||
"user",
|
||||
user_input.transcript_text,
|
||||
time_now_iso8601(),
|
||||
)
|
||||
if user_input.run_immediately and user_input.interrupt:
|
||||
pending_user_inputs.append(user_input)
|
||||
return
|
||||
|
||||
@@ -50,9 +50,14 @@ class UserInput:
|
||||
input_id: str
|
||||
text: str
|
||||
has_camera_frame: bool
|
||||
image_asset_token: str | None
|
||||
run_immediately: bool
|
||||
interrupt: bool
|
||||
|
||||
@property
|
||||
def has_image(self) -> bool:
|
||||
return self.has_camera_frame or bool(self.image_asset_token)
|
||||
|
||||
@property
|
||||
def prompt_text(self) -> str:
|
||||
if self.text:
|
||||
@@ -61,9 +66,9 @@ class UserInput:
|
||||
|
||||
@property
|
||||
def transcript_text(self) -> str:
|
||||
if not self.has_camera_frame:
|
||||
return self.text
|
||||
return f"{self.text}\n已发送一张图片".strip()
|
||||
# Image turns are persisted as one structured media message, including
|
||||
# any optional text caption. Do not create a duplicate transcript row.
|
||||
return "" if self.has_image else self.text
|
||||
|
||||
|
||||
class UserInputError(ValueError):
|
||||
@@ -154,6 +159,7 @@ def parse_user_input(message) -> UserInput | None:
|
||||
|
||||
text = ""
|
||||
has_camera_frame = False
|
||||
image_asset_token: str | None = None
|
||||
for part in parts:
|
||||
if not isinstance(part, dict):
|
||||
raise UserInputError("user-input part 格式不正确", input_id=input_id)
|
||||
@@ -165,23 +171,30 @@ def parse_user_input(message) -> UserInput | None:
|
||||
if not text:
|
||||
raise UserInputError("input_text 不能为空", input_id=input_id)
|
||||
elif part_type == "input_image":
|
||||
if has_camera_frame:
|
||||
if has_camera_frame or image_asset_token:
|
||||
raise UserInputError("P0 只支持一张图片", input_id=input_id)
|
||||
source = part.get("source")
|
||||
if (
|
||||
not isinstance(source, dict)
|
||||
or source.get("type") != "camera_frame"
|
||||
or source.get("frame") != "current"
|
||||
):
|
||||
if not isinstance(source, dict):
|
||||
raise UserInputError("input_image source 格式不正确", input_id=input_id)
|
||||
if source.get("type") == "camera_frame" and source.get("frame") == "current":
|
||||
has_camera_frame = True
|
||||
elif source.get("type") == "uploaded_asset":
|
||||
token = str(source.get("asset_token") or "").strip()
|
||||
if not token or len(token) > 4096:
|
||||
raise UserInputError(
|
||||
"上传图片缺少有效的 asset_token",
|
||||
input_id=input_id,
|
||||
)
|
||||
image_asset_token = token
|
||||
else:
|
||||
raise UserInputError(
|
||||
"P0 只支持当前摄像头画面",
|
||||
"仅支持当前摄像头画面或已上传图片",
|
||||
input_id=input_id,
|
||||
)
|
||||
has_camera_frame = True
|
||||
else:
|
||||
raise UserInputError(f"不支持的输入类型: {part_type}", input_id=input_id)
|
||||
|
||||
if not text and not has_camera_frame:
|
||||
if not text and not (has_camera_frame or image_asset_token):
|
||||
raise UserInputError("user-input 没有有效内容", input_id=input_id)
|
||||
|
||||
options = message.get("options")
|
||||
@@ -192,6 +205,7 @@ def parse_user_input(message) -> UserInput | None:
|
||||
input_id=input_id,
|
||||
text=text,
|
||||
has_camera_frame=has_camera_frame,
|
||||
image_asset_token=image_asset_token,
|
||||
run_immediately=run_immediately,
|
||||
interrupt=interrupt,
|
||||
)
|
||||
@@ -564,7 +578,7 @@ class RealtimeUserInputProcessor(FrameProcessor):
|
||||
"当前工作流节点暂不接收用户输入",
|
||||
)
|
||||
return
|
||||
if user_input.has_camera_frame:
|
||||
if user_input.has_image:
|
||||
await self._emit_error(
|
||||
user_input.input_id,
|
||||
"Realtime 模式暂不支持图片输入",
|
||||
|
||||
@@ -30,8 +30,8 @@ def _require(value: str, label: str) -> str:
|
||||
raise ValueError(f"缺少模型资源配置: {label}")
|
||||
|
||||
|
||||
def image_data_uri(frame: UserImageRawFrame) -> str:
|
||||
"""Encode one Pipecat camera frame for a vision chat-completion request."""
|
||||
def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes:
|
||||
"""Encode one Pipecat camera frame as a storage- and model-ready JPEG."""
|
||||
if not frame.format:
|
||||
raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型")
|
||||
buffer = BytesIO()
|
||||
@@ -40,10 +40,30 @@ def image_data_uri(frame: UserImageRawFrame) -> str:
|
||||
format="JPEG",
|
||||
quality=85,
|
||||
)
|
||||
encoded = base64.b64encode(buffer.getvalue()).decode("utf-8")
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def image_data_uri(frame: UserImageRawFrame) -> str:
|
||||
"""Encode one Pipecat camera frame for a vision chat-completion request."""
|
||||
encoded = base64.b64encode(image_jpeg_bytes(frame)).decode("utf-8")
|
||||
return f"data:image/jpeg;base64,{encoded}"
|
||||
|
||||
|
||||
def image_frame_from_jpeg(data: bytes) -> UserImageRawFrame:
|
||||
"""Decode one normalized uploaded JPEG into Pipecat's raw image frame."""
|
||||
|
||||
try:
|
||||
image = Image.open(BytesIO(data)).convert("RGB")
|
||||
image.load()
|
||||
except (OSError, ValueError) as exc:
|
||||
raise ValueError("上传图片无法解码") from exc
|
||||
return UserImageRawFrame(
|
||||
image=image.tobytes(),
|
||||
size=image.size,
|
||||
format="RGB",
|
||||
)
|
||||
|
||||
|
||||
async def analyze_image_with_vision_model(
|
||||
cfg: AssistantConfig,
|
||||
frame: UserImageRawFrame,
|
||||
|
||||
@@ -59,3 +59,11 @@ S3_BUCKET = os.getenv("S3_BUCKET", "ai-video")
|
||||
S3_REGION = os.getenv("S3_REGION", "us-east-1")
|
||||
KNOWLEDGE_MAX_FILE_BYTES = int(os.getenv("KNOWLEDGE_MAX_FILE_BYTES", "20971520"))
|
||||
KNOWLEDGE_TOP_K = int(os.getenv("KNOWLEDGE_TOP_K", "5"))
|
||||
|
||||
# ---- Realtime user-input image attachments ----
|
||||
INPUT_IMAGE_MAX_BYTES = int(os.getenv("INPUT_IMAGE_MAX_BYTES", "10485760"))
|
||||
INPUT_IMAGE_MAX_PIXELS = int(os.getenv("INPUT_IMAGE_MAX_PIXELS", "24000000"))
|
||||
INPUT_IMAGE_MAX_EDGE = int(os.getenv("INPUT_IMAGE_MAX_EDGE", "2048"))
|
||||
INPUT_IMAGE_TOKEN_TTL_SECONDS = int(
|
||||
os.getenv("INPUT_IMAGE_TOKEN_TTL_SECONDS", "600")
|
||||
)
|
||||
|
||||
@@ -105,6 +105,58 @@ class ConversationRecorderTest(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(saved["sessionId"], "conv_test")
|
||||
self.assertEqual(saved["sequence"], 1)
|
||||
|
||||
async def test_image_is_persisted_as_message_with_artifact(self):
|
||||
conversation = SimpleNamespace(message_count=0)
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self):
|
||||
self.added = []
|
||||
self.committed = False
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_args):
|
||||
return None
|
||||
|
||||
def add(self, value):
|
||||
self.added.append(value)
|
||||
|
||||
async def get(self, _model, _session_id):
|
||||
return conversation
|
||||
|
||||
async def commit(self):
|
||||
self.committed = True
|
||||
|
||||
session = FakeSession()
|
||||
recorder = ConversationRecorder("conv_test")
|
||||
with (
|
||||
patch("services.conversation_history.SessionLocal", return_value=session),
|
||||
patch("services.conversation_history.put_object") as put_object,
|
||||
):
|
||||
await recorder._record_image(
|
||||
b"jpeg-data",
|
||||
input_id="input_photo",
|
||||
timestamp="2026-08-05T10:00:00+08:00",
|
||||
mime_type="image/jpeg",
|
||||
content="帮我看看",
|
||||
source="uploaded_asset",
|
||||
)
|
||||
|
||||
self.assertTrue(session.committed)
|
||||
self.assertEqual(len(session.added), 2)
|
||||
message, artifact = session.added
|
||||
self.assertEqual(message.content_type, "image")
|
||||
self.assertEqual(message.role, "user")
|
||||
self.assertEqual(message.content, "帮我看看")
|
||||
self.assertEqual(message.extra["input_id"], "input_photo")
|
||||
self.assertEqual(message.extra["source"], "uploaded_asset")
|
||||
self.assertEqual(artifact.message_id, message.id)
|
||||
self.assertEqual(artifact.kind, "image")
|
||||
self.assertEqual(artifact.size_bytes, len(b"jpeg-data"))
|
||||
self.assertEqual(conversation.message_count, 1)
|
||||
put_object.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
58
backend/tests/test_input_assets.py
Normal file
58
backend/tests/test_input_assets.py
Normal file
@@ -0,0 +1,58 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from io import BytesIO
|
||||
from unittest.mock import patch
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from services.input_assets import consume_input_image, store_input_image
|
||||
|
||||
|
||||
def png_bytes(size: tuple[int, int] = (32, 24)) -> bytes:
|
||||
output = BytesIO()
|
||||
Image.new("RGBA", size, (20, 80, 160, 180)).save(output, format="PNG")
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
class InputAssetTests(unittest.TestCase):
|
||||
def test_store_normalizes_and_consume_removes_temporary_object(self):
|
||||
with (
|
||||
patch("services.input_assets.put_object") as put_object,
|
||||
patch("services.input_assets.get_object") as get_object,
|
||||
patch("services.input_assets.delete_object") as delete_object,
|
||||
):
|
||||
stored = store_input_image(png_bytes())
|
||||
|
||||
key, normalized, mime_type = put_object.call_args.args
|
||||
self.assertTrue(key.startswith("conversation-inputs/"))
|
||||
self.assertEqual(mime_type, "image/jpeg")
|
||||
self.assertTrue(normalized.startswith(b"\xff\xd8"))
|
||||
self.assertEqual((stored.width, stored.height), (32, 24))
|
||||
self.assertEqual(stored.size_bytes, len(normalized))
|
||||
|
||||
get_object.return_value = normalized
|
||||
consumed = consume_input_image(stored.token)
|
||||
|
||||
self.assertEqual(consumed, normalized)
|
||||
get_object.assert_called_once_with(key)
|
||||
delete_object.assert_called_once_with(key)
|
||||
|
||||
def test_tampered_token_is_rejected_before_storage_read(self):
|
||||
with patch("services.input_assets.put_object"):
|
||||
stored = store_input_image(png_bytes())
|
||||
tampered = f"{stored.token}x"
|
||||
|
||||
with patch("services.input_assets.get_object") as get_object:
|
||||
with self.assertRaisesRegex(ValueError, "签名无效"):
|
||||
consume_input_image(tampered)
|
||||
|
||||
get_object.assert_not_called()
|
||||
|
||||
def test_non_image_is_rejected(self):
|
||||
with self.assertRaisesRegex(ValueError, "可识别的图片"):
|
||||
store_input_image(b"not an image")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -32,7 +32,7 @@ class UserInputParserTests(unittest.TestCase):
|
||||
self.assertIsNotNone(value)
|
||||
self.assertEqual(value.text, "帮我看看")
|
||||
self.assertTrue(value.has_camera_frame)
|
||||
self.assertEqual(value.transcript_text, "帮我看看\n已发送一张图片")
|
||||
self.assertEqual(value.transcript_text, "")
|
||||
|
||||
def test_rejects_legacy_and_unsupported_image_sources(self):
|
||||
self.assertIsNone(parse_user_input({"type": "user-text", "text": "旧协议"}))
|
||||
@@ -46,14 +46,59 @@ class UserInputParserTests(unittest.TestCase):
|
||||
{
|
||||
"type": "input_image",
|
||||
"source": {
|
||||
"type": "uploaded_asset",
|
||||
"asset_id": "asset_1",
|
||||
"type": "remote_url",
|
||||
"url": "https://example.com/image.jpg",
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
def test_parses_uploaded_image_asset(self):
|
||||
value = parse_user_input(
|
||||
{
|
||||
"type": "user-input",
|
||||
"schema_version": 1,
|
||||
"input_id": "input_upload",
|
||||
"parts": [
|
||||
{"type": "input_text", "text": "这是什么?"},
|
||||
{
|
||||
"type": "input_image",
|
||||
"source": {
|
||||
"type": "uploaded_asset",
|
||||
"asset_token": "signed-token",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
self.assertIsNotNone(value)
|
||||
self.assertTrue(value.has_image)
|
||||
self.assertFalse(value.has_camera_frame)
|
||||
self.assertEqual(value.image_asset_token, "signed-token")
|
||||
self.assertEqual(value.prompt_text, "这是什么?")
|
||||
self.assertEqual(value.transcript_text, "")
|
||||
|
||||
def test_image_only_input_has_no_synthetic_chat_text(self):
|
||||
value = parse_user_input(
|
||||
{
|
||||
"type": "user-input",
|
||||
"schema_version": 1,
|
||||
"input_id": "input_photo",
|
||||
"parts": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"source": {"type": "camera_frame", "frame": "current"},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
self.assertIsNotNone(value)
|
||||
self.assertEqual(value.transcript_text, "")
|
||||
self.assertEqual(value.prompt_text, "请根据用户刚提交的图片进行回复。")
|
||||
|
||||
def test_native_image_uses_the_standard_multimodal_user_turn_path(self):
|
||||
image = UserImageRawFrame(
|
||||
image=bytes([220, 40, 40] * 16 * 16),
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
Braces,
|
||||
Check,
|
||||
Copy,
|
||||
ImageIcon,
|
||||
Loader2,
|
||||
MessageSquareText,
|
||||
Mic,
|
||||
@@ -51,16 +52,46 @@ import {
|
||||
useVoicePreview,
|
||||
type ChatMessage,
|
||||
type ClientToolDefinition,
|
||||
type UserInputPart,
|
||||
type VoicePreview,
|
||||
type VoicePreviewStatus,
|
||||
} from "@/hooks/use-voice-preview";
|
||||
import type { DynamicVariableDefinition } from "@/lib/api";
|
||||
import {
|
||||
inputAssetsApi,
|
||||
type DynamicVariableDefinition,
|
||||
} from "@/lib/api";
|
||||
|
||||
type VizStyle = "aura" | "nebula" | "bars" | "wave";
|
||||
|
||||
// 调试面板顶部主视图:聊天记录 / 视频流
|
||||
type DebugView = "chat" | "video";
|
||||
type DebugInputMode = "mic" | "text";
|
||||
type PendingDebugImage = {
|
||||
file: File;
|
||||
previewUrl: string;
|
||||
};
|
||||
|
||||
const DEBUG_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
function fileToDataUrl(file: File): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => resolve(String(reader.result));
|
||||
reader.onerror = () => reject(new Error("无法读取图片"));
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
}
|
||||
|
||||
function hasDraggedImage(dataTransfer: DataTransfer): boolean {
|
||||
return (
|
||||
Array.from(dataTransfer.items).some((item) =>
|
||||
item.type.startsWith("image/"),
|
||||
) ||
|
||||
Array.from(dataTransfer.files).some((file) =>
|
||||
file.type.startsWith("image/"),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const VIZ_OPTIONS: { style: VizStyle; label: string; icon: React.ReactNode }[] =
|
||||
[
|
||||
@@ -766,6 +797,8 @@ function DebugVoicePanel({
|
||||
remoteStream,
|
||||
messages,
|
||||
sendText,
|
||||
sendUserInput,
|
||||
appendUserImage,
|
||||
connect,
|
||||
disconnect,
|
||||
audioRef,
|
||||
@@ -773,6 +806,11 @@ function DebugVoicePanel({
|
||||
const recording = status === "connecting" || status === "connected";
|
||||
const [textDraft, setTextDraft] = useState("");
|
||||
const [inputMode, setInputMode] = useState<DebugInputMode>("mic");
|
||||
const [pendingImage, setPendingImage] =
|
||||
useState<PendingDebugImage | null>(null);
|
||||
const [inputError, setInputError] = useState("");
|
||||
const [sendingInput, setSendingInput] = useState(false);
|
||||
const [draggingImage, setDraggingImage] = useState(false);
|
||||
const [clientDialogContainer, setClientDialogContainer] =
|
||||
useState<HTMLDivElement | null>(null);
|
||||
const [messageDialogOpen, setMessageDialogOpen] = useState(false);
|
||||
@@ -790,9 +828,110 @@ function DebugVoicePanel({
|
||||
: "";
|
||||
const startDisabled = status === "connecting" || Boolean(startBlockedMessage);
|
||||
|
||||
function handleSendText() {
|
||||
if (sendText(textDraft)) {
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (pendingImage) URL.revokeObjectURL(pendingImage.previewUrl);
|
||||
};
|
||||
}, [pendingImage]);
|
||||
|
||||
function stageImage(file: File) {
|
||||
setInputError("");
|
||||
if (!vision) {
|
||||
setInputError("请先为助手开启视觉理解,再添加图片。");
|
||||
return;
|
||||
}
|
||||
if (!file.type.startsWith("image/")) {
|
||||
setInputError("只能添加图片文件。");
|
||||
return;
|
||||
}
|
||||
if (file.size > DEBUG_IMAGE_MAX_BYTES) {
|
||||
setInputError("图片不能超过 10 MB。");
|
||||
return;
|
||||
}
|
||||
setPendingImage({ file, previewUrl: URL.createObjectURL(file) });
|
||||
setInputMode("text");
|
||||
}
|
||||
|
||||
async function handleSendInput() {
|
||||
const text = textDraft.trim();
|
||||
if (!pendingImage) {
|
||||
if (sendText(text)) {
|
||||
setTextDraft("");
|
||||
setInputError("");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (status !== "connected" || sendingInput) return;
|
||||
|
||||
setSendingInput(true);
|
||||
setInputError("");
|
||||
let assetToken = "";
|
||||
try {
|
||||
const imageUrl = await fileToDataUrl(pendingImage.file);
|
||||
const asset = await inputAssetsApi.uploadImage(pendingImage.file);
|
||||
assetToken = asset.assetToken;
|
||||
const parts: UserInputPart[] = [];
|
||||
if (text) parts.push({ type: "input_text", text });
|
||||
parts.push({
|
||||
type: "input_image",
|
||||
source: { type: "uploaded_asset", asset_token: assetToken },
|
||||
});
|
||||
|
||||
const timestamp = new Date().toISOString();
|
||||
const result = await sendUserInput(parts);
|
||||
appendUserImage(result.inputId, imageUrl, timestamp, text);
|
||||
setTextDraft("");
|
||||
setPendingImage(null);
|
||||
} catch (sendError) {
|
||||
if (assetToken) {
|
||||
void inputAssetsApi.remove(assetToken).catch(() => {});
|
||||
}
|
||||
setInputError(
|
||||
sendError instanceof Error ? sendError.message : "图片发送失败,请重试。",
|
||||
);
|
||||
} finally {
|
||||
setSendingInput(false);
|
||||
}
|
||||
}
|
||||
|
||||
function handlePaste(event: React.ClipboardEvent<HTMLTextAreaElement>) {
|
||||
const image = Array.from(event.clipboardData.items)
|
||||
.find((item) => item.type.startsWith("image/"))
|
||||
?.getAsFile();
|
||||
if (!image) return;
|
||||
event.preventDefault();
|
||||
stageImage(image);
|
||||
}
|
||||
|
||||
function handleDragOver(event: React.DragEvent<HTMLDivElement>) {
|
||||
if (!hasDraggedImage(event.dataTransfer)) return;
|
||||
event.preventDefault();
|
||||
event.dataTransfer.dropEffect = "copy";
|
||||
setDraggingImage(true);
|
||||
}
|
||||
|
||||
function handleDragLeave(event: React.DragEvent<HTMLDivElement>) {
|
||||
const nextTarget = event.relatedTarget;
|
||||
if (nextTarget instanceof Node && event.currentTarget.contains(nextTarget)) {
|
||||
return;
|
||||
}
|
||||
setDraggingImage(false);
|
||||
}
|
||||
|
||||
function handleDrop(event: React.DragEvent<HTMLDivElement>) {
|
||||
if (!hasDraggedImage(event.dataTransfer)) return;
|
||||
event.preventDefault();
|
||||
setDraggingImage(false);
|
||||
const image = Array.from(event.dataTransfer.files).find((file) =>
|
||||
file.type.startsWith("image/"),
|
||||
);
|
||||
if (image) stageImage(image);
|
||||
}
|
||||
|
||||
function removePendingImage() {
|
||||
if (!sendingInput) {
|
||||
setPendingImage(null);
|
||||
setInputError("");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -968,61 +1107,136 @@ function DebugVoicePanel({
|
||||
</div>
|
||||
|
||||
<div className="shrink-0 border-t border-hairline bg-card p-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<div
|
||||
className={[
|
||||
"flex h-10 min-w-0 items-center gap-1 rounded-[1.4rem] border border-hairline-strong bg-background px-2",
|
||||
"flex-1",
|
||||
].join(" ")}
|
||||
>
|
||||
<div className="flex shrink-0 items-center gap-0.5 rounded-full bg-canvas-soft p-0.5">
|
||||
<DebugInputModeButton
|
||||
selected={inputMode === "mic"}
|
||||
label="选择麦克风设备"
|
||||
onClick={() => setInputMode("mic")}
|
||||
>
|
||||
<Mic size={15} />
|
||||
</DebugInputModeButton>
|
||||
<DebugInputModeButton
|
||||
selected={inputMode === "text"}
|
||||
label="文字输入"
|
||||
onClick={() => setInputMode("text")}
|
||||
>
|
||||
<MessageSquareText size={15} />
|
||||
</DebugInputModeButton>
|
||||
<div className="flex items-end gap-2">
|
||||
<div className="min-w-0 flex-1">
|
||||
<div
|
||||
onDragOver={handleDragOver}
|
||||
onDragLeave={handleDragLeave}
|
||||
onDrop={handleDrop}
|
||||
className={[
|
||||
"relative flex min-h-10 min-w-0 items-end gap-1 overflow-hidden rounded-[1.4rem] border bg-background px-2 transition-colors",
|
||||
draggingImage
|
||||
? "border-foreground"
|
||||
: "border-hairline-strong",
|
||||
].join(" ")}
|
||||
>
|
||||
<div className="mb-1 flex shrink-0 items-center gap-0.5 rounded-full bg-canvas-soft p-0.5">
|
||||
<DebugInputModeButton
|
||||
selected={inputMode === "mic"}
|
||||
label="选择麦克风设备"
|
||||
onClick={() => setInputMode("mic")}
|
||||
>
|
||||
<Mic size={15} />
|
||||
</DebugInputModeButton>
|
||||
<DebugInputModeButton
|
||||
selected={inputMode === "text"}
|
||||
label="文字输入"
|
||||
onClick={() => setInputMode("text")}
|
||||
>
|
||||
<MessageSquareText size={15} />
|
||||
</DebugInputModeButton>
|
||||
</div>
|
||||
{inputMode === "mic" ? (
|
||||
<div className="flex min-w-0 flex-1">
|
||||
<MicrophoneDeviceField preview={preview} />
|
||||
</div>
|
||||
) : (
|
||||
<div className="min-w-0 flex-1 py-1">
|
||||
{pendingImage && (
|
||||
<div className="flex items-center gap-2 px-2 pb-1 pt-0.5">
|
||||
<div className="group relative h-14 w-14 shrink-0 overflow-hidden rounded-xl border border-hairline bg-canvas-soft">
|
||||
{/* eslint-disable-next-line @next/next/no-img-element */}
|
||||
<img
|
||||
src={pendingImage.previewUrl}
|
||||
alt="待发送图片预览"
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
aria-label="移除待发送图片"
|
||||
title="移除图片"
|
||||
disabled={sendingInput}
|
||||
onClick={removePendingImage}
|
||||
className="absolute right-1 top-1 flex h-5 w-5 items-center justify-center rounded-full bg-background/90 text-foreground shadow-sm transition-colors hover:bg-background disabled:cursor-not-allowed"
|
||||
>
|
||||
<X size={12} />
|
||||
</button>
|
||||
{sendingInput && (
|
||||
<span className="absolute inset-0 flex items-center justify-center bg-background/65">
|
||||
<Loader2 size={18} className="animate-spin" />
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="min-w-0">
|
||||
<p className="truncate text-xs font-medium text-foreground">
|
||||
{pendingImage.file.name || "粘贴的图片"}
|
||||
</p>
|
||||
<p className="mt-0.5 text-[11px] text-muted-soft">
|
||||
按 Enter 上传并发送
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<Textarea
|
||||
rows={1}
|
||||
value={textDraft}
|
||||
disabled={status !== "connected" || sendingInput}
|
||||
onChange={(event) => {
|
||||
setTextDraft(event.target.value);
|
||||
if (inputError) setInputError("");
|
||||
}}
|
||||
onPaste={handlePaste}
|
||||
onKeyDown={(event) => {
|
||||
if (
|
||||
event.key === "Enter" &&
|
||||
!event.shiftKey &&
|
||||
!event.nativeEvent.isComposing
|
||||
) {
|
||||
event.preventDefault();
|
||||
void handleSendInput();
|
||||
}
|
||||
}}
|
||||
placeholder={
|
||||
status === "connected"
|
||||
? vision
|
||||
? "输入文字,或粘贴 / 拖入图片…"
|
||||
: "输入文字发送给助手,将打断当前播报…"
|
||||
: "开始对话后可输入文字…"
|
||||
}
|
||||
className="max-h-24 min-h-8 resize-none overflow-y-auto border-transparent bg-transparent px-2 py-1 text-sm leading-6 text-foreground shadow-none outline-none placeholder:text-muted-soft focus-visible:ring-0 disabled:opacity-100"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
{draggingImage && (
|
||||
<div className="pointer-events-none absolute inset-0 z-20 flex items-center justify-center gap-2 rounded-[1.4rem] bg-background/95 text-xs font-medium text-foreground">
|
||||
<ImageIcon size={16} />
|
||||
松开以添加图片
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{inputMode === "mic" ? (
|
||||
<MicrophoneDeviceField preview={preview} />
|
||||
) : (
|
||||
<Textarea
|
||||
rows={1}
|
||||
value={textDraft}
|
||||
disabled={status !== "connected"}
|
||||
onChange={(event) => setTextDraft(event.target.value)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter" && !event.shiftKey && !event.nativeEvent.isComposing) {
|
||||
event.preventDefault();
|
||||
handleSendText();
|
||||
}
|
||||
}}
|
||||
placeholder={
|
||||
status === "connected"
|
||||
? "输入文字发送给助手,将打断当前播报…"
|
||||
: "开始对话后可输入文字…"
|
||||
}
|
||||
className="h-8 min-h-8 flex-1 resize-none overflow-hidden border-transparent bg-transparent px-2 py-1 text-sm leading-6 text-foreground shadow-none outline-none placeholder:text-muted-soft focus-visible:ring-0 disabled:opacity-100"
|
||||
/>
|
||||
{inputError && (
|
||||
<p className="px-2 pt-1 text-[11px] leading-4 text-destructive">
|
||||
{inputError}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
{inputMode === "text" && (
|
||||
<Button
|
||||
size="icon"
|
||||
className="h-10 w-10 shrink-0 rounded-full"
|
||||
aria-label="发送调试消息"
|
||||
disabled={status !== "connected" || !textDraft.trim()}
|
||||
onClick={handleSendText}
|
||||
aria-label={sendingInput ? "正在发送图片" : "发送调试消息"}
|
||||
disabled={
|
||||
status !== "connected" ||
|
||||
sendingInput ||
|
||||
(!textDraft.trim() && !pendingImage)
|
||||
}
|
||||
onClick={() => void handleSendInput()}
|
||||
>
|
||||
<Send size={16} />
|
||||
{sendingInput ? (
|
||||
<Loader2 size={16} className="animate-spin" />
|
||||
) : (
|
||||
<Send size={16} />
|
||||
)}
|
||||
</Button>
|
||||
)}
|
||||
{!showIdleHub && (
|
||||
@@ -1168,7 +1382,10 @@ function DebugVisionWorkspace({
|
||||
} | null>(null);
|
||||
const latestMessage = [...messages]
|
||||
.reverse()
|
||||
.find((message) => message.content.trim());
|
||||
.find(
|
||||
(message) =>
|
||||
message.content.trim() || (message.attachments?.length ?? 0) > 0,
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const container = containerRef.current;
|
||||
@@ -1215,7 +1432,10 @@ function DebugVisionWorkspace({
|
||||
{latestMessage?.role === "user" ? "我:" : "助手:"}
|
||||
</span>
|
||||
<span>
|
||||
{latestMessage?.content || "暂无消息,点击返回聊天记录"}
|
||||
{latestMessage?.content ||
|
||||
(latestMessage?.attachments?.length
|
||||
? "发送了一张照片"
|
||||
: "暂无消息,点击返回聊天记录")}
|
||||
</span>
|
||||
</span>
|
||||
</button>
|
||||
@@ -1463,7 +1683,7 @@ function DebugTranscriptPanel({
|
||||
助手{time ? ` · ${time}` : ""}
|
||||
</span>
|
||||
<div className="whitespace-pre-wrap rounded-2xl rounded-tl-sm bg-surface-strong px-4 py-2.5 text-sm leading-6 text-foreground">
|
||||
{message.content}
|
||||
{message.content || (message.streaming ? "…" : "")}
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
@@ -1474,8 +1694,26 @@ function DebugTranscriptPanel({
|
||||
<span className="px-1 text-[11px] text-muted-soft">
|
||||
我{time ? ` · ${time}` : ""}
|
||||
</span>
|
||||
<div className="whitespace-pre-wrap rounded-2xl rounded-tr-sm bg-primary px-4 py-2.5 text-sm leading-6 text-primary-foreground">
|
||||
{message.content}
|
||||
<div
|
||||
className={[
|
||||
"overflow-hidden whitespace-pre-wrap rounded-2xl rounded-tr-sm bg-primary text-sm leading-6 text-primary-foreground",
|
||||
message.attachments?.length ? "p-1" : "px-4 py-2.5",
|
||||
].join(" ")}
|
||||
>
|
||||
{message.attachments?.map((attachment) => (
|
||||
// eslint-disable-next-line @next/next/no-img-element
|
||||
<img
|
||||
key={attachment.id}
|
||||
src={attachment.url}
|
||||
alt={attachment.alt}
|
||||
className="max-h-72 w-full rounded-[0.8rem] object-cover"
|
||||
/>
|
||||
))}
|
||||
{message.content && (
|
||||
<div className={message.attachments?.length ? "px-3 py-2" : ""}>
|
||||
{message.content}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -1,13 +1,25 @@
|
||||
"use client";
|
||||
|
||||
import {
|
||||
Activity,
|
||||
ArrowRight,
|
||||
Camera,
|
||||
CheckCircle2,
|
||||
ChevronDown,
|
||||
ChevronUp,
|
||||
CircleDot,
|
||||
Eye,
|
||||
GitBranch,
|
||||
ImageIcon,
|
||||
Loader2,
|
||||
MessageSquareText,
|
||||
MonitorSmartphone,
|
||||
MoreHorizontal,
|
||||
Server,
|
||||
Trash2,
|
||||
Wrench,
|
||||
XCircle,
|
||||
type LucideIcon,
|
||||
} from "lucide-react";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
|
||||
@@ -37,9 +49,11 @@ import {
|
||||
} from "@/components/layout/list-page-layout";
|
||||
import { SearchInput } from "@/components/ui/search-input";
|
||||
import {
|
||||
API_BASE,
|
||||
conversationsApi,
|
||||
type Conversation,
|
||||
type ConversationDetail,
|
||||
type ConversationMessage,
|
||||
} from "@/lib/api";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
@@ -93,6 +107,249 @@ function statusLabel(status: string): string {
|
||||
return "已结束";
|
||||
}
|
||||
|
||||
type TraceEvent = Record<string, unknown>;
|
||||
type WorkflowNode = { label: string; type: string };
|
||||
type TraceTone = "default" | "success" | "error" | "user";
|
||||
|
||||
type TracePresentation = {
|
||||
title: string;
|
||||
description: string;
|
||||
icon: LucideIcon;
|
||||
tone: TraceTone;
|
||||
};
|
||||
|
||||
function recordValue(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === "object" && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function textValue(value: unknown): string {
|
||||
return typeof value === "string" ? value : "";
|
||||
}
|
||||
|
||||
function numberValue(value: unknown): number | null {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
function workflowNodes(detail: ConversationDetail): Map<string, WorkflowNode> {
|
||||
const snapshot = recordValue(detail.extra.workflow?.snapshot);
|
||||
const nodes = Array.isArray(snapshot.nodes) ? snapshot.nodes : [];
|
||||
return new Map(
|
||||
nodes.flatMap((rawNode) => {
|
||||
const node = recordValue(rawNode);
|
||||
const id = textValue(node.id);
|
||||
if (!id) return [];
|
||||
const data = recordValue(node.data);
|
||||
const type = textValue(node.type) || "node";
|
||||
return [
|
||||
[
|
||||
id,
|
||||
{
|
||||
label: textValue(data.name) || type,
|
||||
type,
|
||||
},
|
||||
] as const,
|
||||
];
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function nodeLabel(nodes: Map<string, WorkflowNode>, nodeId: unknown): string {
|
||||
const id = textValue(nodeId);
|
||||
const node = nodes.get(id);
|
||||
return node ? `「${node.label}」` : id ? `「${id}」` : "当前节点";
|
||||
}
|
||||
|
||||
function toolLabel(toolType: unknown): string {
|
||||
if (toolType === "client") return "客户端工具";
|
||||
if (toolType === "http") return "HTTP 服务端工具";
|
||||
if (toolType === "mcp") return "MCP 服务端工具";
|
||||
if (toolType === "system") return "系统工具";
|
||||
return "工具";
|
||||
}
|
||||
|
||||
function tracePresentation(
|
||||
event: TraceEvent,
|
||||
nodes: Map<string, WorkflowNode>,
|
||||
startedEvent?: TraceEvent,
|
||||
): TracePresentation {
|
||||
const eventName = textValue(event.event);
|
||||
const node = nodeLabel(nodes, event.nodeId);
|
||||
const outcome = recordValue(event.outcome);
|
||||
const duration = numberValue(outcome.durationMs);
|
||||
const tool = toolLabel(event.toolType ?? startedEvent?.toolType);
|
||||
|
||||
switch (eventName) {
|
||||
case "node_entered":
|
||||
return {
|
||||
title: `进入节点 ${node}`,
|
||||
description: `节点类型:${textValue(event.nodeType) || nodes.get(textValue(event.nodeId))?.type || "未知"}`,
|
||||
icon: CircleDot,
|
||||
tone: "default",
|
||||
};
|
||||
case "node_exited":
|
||||
return {
|
||||
title: `离开节点 ${node}`,
|
||||
description: "当前节点处理完成,准备选择下一步。",
|
||||
icon: ArrowRight,
|
||||
tone: "default",
|
||||
};
|
||||
case "edge_selected":
|
||||
return {
|
||||
title: `节点转移:${nodeLabel(nodes, event.sourceNodeId)} → ${nodeLabel(nodes, event.targetNodeId)}`,
|
||||
description: `路由方式:${textValue(event.edgeMode) || "always"}`,
|
||||
icon: GitBranch,
|
||||
tone: "default",
|
||||
};
|
||||
case "action_started":
|
||||
return {
|
||||
title: `开始调用${tool}`,
|
||||
description: `${node} · ${textValue(event.toolId) || "未命名工具"}`,
|
||||
icon: tool === "客户端工具" ? MonitorSmartphone : Server,
|
||||
tone: "default",
|
||||
};
|
||||
case "action_completed":
|
||||
return {
|
||||
title: `${tool}执行成功`,
|
||||
description: `${node}${duration === null ? "" : ` · ${duration} ms`}`,
|
||||
icon: CheckCircle2,
|
||||
tone: "success",
|
||||
};
|
||||
case "action_failed":
|
||||
case "action_cancelled":
|
||||
return {
|
||||
title: `${tool}${eventName === "action_failed" ? "执行失败" : "已取消"}`,
|
||||
description: textValue(recordValue(outcome.error).message) || node,
|
||||
icon: XCircle,
|
||||
tone: "error",
|
||||
};
|
||||
case "tool_started":
|
||||
return {
|
||||
title: `开始调用${tool}`,
|
||||
description: `${node} · ${textValue(event.toolName) || textValue(event.functionName) || textValue(event.toolId) || "未命名工具"}`,
|
||||
icon: tool === "客户端工具" ? MonitorSmartphone : Server,
|
||||
tone: "default",
|
||||
};
|
||||
case "tool_completed":
|
||||
return {
|
||||
title: `${tool}调用成功`,
|
||||
description: `${textValue(event.toolName) || textValue(event.functionName) || node}${numberValue(event.durationMs) === null ? "" : ` · ${numberValue(event.durationMs)} ms`}`,
|
||||
icon: CheckCircle2,
|
||||
tone: "success",
|
||||
};
|
||||
case "tool_failed":
|
||||
return {
|
||||
title: `${tool}调用失败`,
|
||||
description: textValue(event.error) || textValue(event.toolName) || node,
|
||||
icon: XCircle,
|
||||
tone: "error",
|
||||
};
|
||||
case "client_tool_started":
|
||||
return {
|
||||
title:
|
||||
event.functionName === "show_message"
|
||||
? "向客户端显示交互消息"
|
||||
: "向客户端下发工具调用",
|
||||
description: textValue(event.functionName) || "客户端工具",
|
||||
icon: MonitorSmartphone,
|
||||
tone: "default",
|
||||
};
|
||||
case "client_tool_completed": {
|
||||
const userAction = textValue(event.userAction);
|
||||
return {
|
||||
title: userAction === "confirmed" ? "用户已确认" : "客户端交互已完成",
|
||||
description: userAction
|
||||
? `用户操作:${userAction}`
|
||||
: textValue(event.functionName) || textValue(event.status) || "执行成功",
|
||||
icon: CheckCircle2,
|
||||
tone: userAction ? "user" : "success",
|
||||
};
|
||||
}
|
||||
case "client_tool_failed":
|
||||
return {
|
||||
title: "客户端交互失败",
|
||||
description: textValue(event.error) || textValue(event.functionName),
|
||||
icon: XCircle,
|
||||
tone: "error",
|
||||
};
|
||||
case "message_started":
|
||||
return event.requiresConfirmation
|
||||
? {
|
||||
title: "等待用户确认",
|
||||
description: `${node}向客户端显示了确认消息。`,
|
||||
icon: MonitorSmartphone,
|
||||
tone: "user",
|
||||
}
|
||||
: {
|
||||
title: "开始播放固定消息",
|
||||
description: node,
|
||||
icon: Activity,
|
||||
tone: "default",
|
||||
};
|
||||
case "message_completed": {
|
||||
const action = textValue(event.action);
|
||||
return {
|
||||
title: action === "confirmed" ? "用户已确认" : "消息步骤已完成",
|
||||
description: `${node}${action ? ` · 操作:${action}` : ""}`,
|
||||
icon: CheckCircle2,
|
||||
tone: action ? "user" : "success",
|
||||
};
|
||||
}
|
||||
case "message_interrupted":
|
||||
return {
|
||||
title: "用户输入打断消息",
|
||||
description: node,
|
||||
icon: Activity,
|
||||
tone: "user",
|
||||
};
|
||||
case "message_failed":
|
||||
return {
|
||||
title: "消息步骤失败",
|
||||
description: textValue(event.error) || node,
|
||||
icon: XCircle,
|
||||
tone: "error",
|
||||
};
|
||||
case "variables_updated": {
|
||||
const names = Array.isArray(event.variableNames)
|
||||
? event.variableNames.filter((name): name is string => typeof name === "string")
|
||||
: [];
|
||||
return {
|
||||
title: "会话变量已更新",
|
||||
description: names.length ? names.join("、") : node,
|
||||
icon: Wrench,
|
||||
tone: "default",
|
||||
};
|
||||
}
|
||||
case "variables_snapshot":
|
||||
return {
|
||||
title: "记录会话变量快照",
|
||||
description: node,
|
||||
icon: Wrench,
|
||||
tone: "default",
|
||||
};
|
||||
case "workflow_ended":
|
||||
return {
|
||||
title: "工作流已结束",
|
||||
description: `${node} · ${textValue(event.outcome) || "success"}`,
|
||||
icon: CheckCircle2,
|
||||
tone: "success",
|
||||
};
|
||||
default:
|
||||
return {
|
||||
title: eventName || "运行事件",
|
||||
description: node,
|
||||
icon: Activity,
|
||||
tone: "default",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function artifactUrl(path: string): string {
|
||||
if (/^(https?:|data:|blob:)/.test(path)) return path;
|
||||
return `${API_BASE}${path.startsWith("/") ? path : `/${path}`}`;
|
||||
}
|
||||
|
||||
export function HistoryPage() {
|
||||
const [rows, setRows] = useState<Conversation[]>([]);
|
||||
const [total, setTotal] = useState(0);
|
||||
@@ -370,7 +627,7 @@ export function HistoryPage() {
|
||||
<>
|
||||
<ListPageLayout
|
||||
title="历史记录"
|
||||
description="查看每次语音或文字会话中最终确认的用户转写和助手回复。"
|
||||
description="按会话查看对话、照片、工具调用、用户操作与工作流运行轨迹。"
|
||||
>
|
||||
<ListPageSection>
|
||||
<ListToolbar
|
||||
@@ -423,7 +680,7 @@ export function HistoryPage() {
|
||||
</ListPageLayout>
|
||||
|
||||
<Dialog open={dialogOpen} onOpenChange={setDialogOpen}>
|
||||
<DialogContent className="max-h-[calc(100vh-3rem)] grid-rows-[auto_minmax(0,1fr)_auto] overflow-hidden sm:max-w-4xl">
|
||||
<DialogContent className="max-h-[calc(100vh-3rem)] grid-rows-[auto_minmax(0,1fr)_auto] overflow-hidden sm:max-w-5xl">
|
||||
<DialogHeader>
|
||||
<DialogTitle className="flex items-center gap-2">
|
||||
<MessageSquareText size={18} />
|
||||
@@ -432,7 +689,7 @@ export function HistoryPage() {
|
||||
<DialogDescription>
|
||||
{detail
|
||||
? `${formatDate(detail.startedAt)} · ${channelLabel(detail.channel)} · ${detail.messageCount} 条消息`
|
||||
: "查看本次会话中最终确认的文本消息。"}
|
||||
: "查看本次会话的完整对话与运行记录。"}
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
@@ -464,56 +721,7 @@ export function HistoryPage() {
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="rounded-xl border border-hairline bg-surface-strong/20">
|
||||
<div className="border-b border-hairline px-4 py-3 text-sm font-medium">
|
||||
对话内容
|
||||
</div>
|
||||
<div className="space-y-5 p-4 sm:p-5">
|
||||
{detail.messages.map((message) => (
|
||||
<div
|
||||
key={message.id}
|
||||
className={cn(
|
||||
"flex",
|
||||
message.role === "user"
|
||||
? "justify-end"
|
||||
: "justify-start",
|
||||
)}
|
||||
>
|
||||
<div className="max-w-[85%]">
|
||||
<div
|
||||
className={cn(
|
||||
"mb-1.5 text-xs text-muted-soft",
|
||||
message.role === "user" && "text-right",
|
||||
)}
|
||||
>
|
||||
{message.role === "user" ? "用户" : "助手"} ·{" "}
|
||||
{formatDate(message.occurredAt)}
|
||||
</div>
|
||||
<div
|
||||
className={cn(
|
||||
"whitespace-pre-wrap rounded-2xl px-4 py-3 text-left leading-6",
|
||||
message.role === "user"
|
||||
? "rounded-br-md bg-primary text-primary-foreground"
|
||||
: "rounded-bl-md border border-hairline bg-background text-foreground shadow-sm",
|
||||
)}
|
||||
>
|
||||
{message.content}
|
||||
</div>
|
||||
{message.extra.interrupted && (
|
||||
<div className="mt-1.5 text-xs text-muted-foreground">
|
||||
回复在生成中被打断
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
{detail.messages.length === 0 && (
|
||||
<div className="py-12 text-center text-muted-foreground">
|
||||
本次会话没有产生文本消息
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
<ConversationTimeline detail={detail} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
@@ -537,3 +745,237 @@ function Metadata({ label, value }: { label: string; value: string }) {
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
type TimelineEntry =
|
||||
| {
|
||||
kind: "message";
|
||||
timestamp: string;
|
||||
order: number;
|
||||
message: ConversationMessage;
|
||||
}
|
||||
| {
|
||||
kind: "trace";
|
||||
timestamp: string;
|
||||
order: number;
|
||||
event: TraceEvent;
|
||||
};
|
||||
|
||||
function timestampOrder(value: string): number {
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? Number.MAX_SAFE_INTEGER : parsed;
|
||||
}
|
||||
|
||||
function ConversationTimeline({ detail }: { detail: ConversationDetail }) {
|
||||
const nodes = workflowNodes(detail);
|
||||
const trace = (detail.extra.workflowTrace ?? []).map(recordValue);
|
||||
const startedByInvocation = new Map<string, TraceEvent>();
|
||||
trace.forEach((event) => {
|
||||
if (event.event === "action_started" || event.event === "tool_started") {
|
||||
const invocationId = textValue(event.invocationId);
|
||||
if (invocationId) startedByInvocation.set(invocationId, event);
|
||||
}
|
||||
});
|
||||
|
||||
const entries: TimelineEntry[] = [
|
||||
...detail.messages.map((message) => ({
|
||||
kind: "message" as const,
|
||||
timestamp: message.occurredAt,
|
||||
order: message.sequence,
|
||||
message,
|
||||
})),
|
||||
...trace.map((event, index) => ({
|
||||
kind: "trace" as const,
|
||||
timestamp: textValue(event.timestamp),
|
||||
order: numberValue(event.sequence) ?? index + 1,
|
||||
event,
|
||||
})),
|
||||
].sort(
|
||||
(a, b) =>
|
||||
timestampOrder(a.timestamp) - timestampOrder(b.timestamp) ||
|
||||
a.order - b.order,
|
||||
);
|
||||
|
||||
const imageCount = detail.messages.reduce(
|
||||
(count, message) =>
|
||||
count + (message.artifacts ?? []).filter((item) => item.kind === "image").length,
|
||||
0,
|
||||
);
|
||||
const toolCount = trace.filter(
|
||||
(event) => event.event === "action_started" || event.event === "tool_started",
|
||||
).length;
|
||||
const transitionCount = trace.filter((event) => event.event === "edge_selected").length;
|
||||
|
||||
return (
|
||||
<section className="overflow-hidden rounded-xl border border-hairline bg-surface-strong/20">
|
||||
<div className="flex flex-wrap items-center justify-between gap-3 border-b border-hairline px-4 py-3">
|
||||
<div className="text-sm font-medium">完整时间线</div>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<Badge variant="secondary" className="gap-1.5 bg-surface-strong text-muted-foreground">
|
||||
<MessageSquareText size={12} />
|
||||
{detail.messages.length} 条对话
|
||||
</Badge>
|
||||
{imageCount > 0 && (
|
||||
<Badge variant="secondary" className="gap-1.5 bg-surface-strong text-muted-foreground">
|
||||
<ImageIcon size={12} />
|
||||
{imageCount} 张照片
|
||||
</Badge>
|
||||
)}
|
||||
{toolCount > 0 && (
|
||||
<Badge variant="secondary" className="gap-1.5 bg-surface-strong text-muted-foreground">
|
||||
<Wrench size={12} />
|
||||
{toolCount} 次工具
|
||||
</Badge>
|
||||
)}
|
||||
{transitionCount > 0 && (
|
||||
<Badge variant="secondary" className="gap-1.5 bg-surface-strong text-muted-foreground">
|
||||
<GitBranch size={12} />
|
||||
{transitionCount} 次转移
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="space-y-4 p-4 sm:p-5">
|
||||
{entries.map((entry, index) =>
|
||||
entry.kind === "message" ? (
|
||||
<TimelineMessage key={`message-${entry.message.id}`} message={entry.message} />
|
||||
) : (
|
||||
<TimelineTrace
|
||||
key={`trace-${textValue(entry.event.eventId) || index}`}
|
||||
event={entry.event}
|
||||
nodes={nodes}
|
||||
startedEvent={startedByInvocation.get(
|
||||
textValue(recordValue(entry.event.outcome).invocationId) ||
|
||||
textValue(entry.event.invocationId),
|
||||
)}
|
||||
/>
|
||||
),
|
||||
)}
|
||||
{entries.length === 0 && (
|
||||
<div className="py-12 text-center text-muted-foreground">
|
||||
本次会话没有产生可展示的记录
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
function TimelineMessage({ message }: { message: ConversationMessage }) {
|
||||
const isUser = message.role === "user";
|
||||
const images = (message.artifacts ?? []).filter((item) => item.kind === "image");
|
||||
const isImageMessage = message.contentType === "image" || images.length > 0;
|
||||
|
||||
return (
|
||||
<div className={cn("flex", isUser ? "justify-end" : "justify-start")}>
|
||||
<div className="max-w-[88%] sm:max-w-[78%]">
|
||||
<div
|
||||
className={cn(
|
||||
"mb-1.5 flex items-center gap-1.5 text-xs text-muted-soft",
|
||||
isUser && "justify-end",
|
||||
)}
|
||||
>
|
||||
{isImageMessage && <Camera size={12} />}
|
||||
{isUser ? "用户" : "助手"} · {formatDate(message.occurredAt)}
|
||||
</div>
|
||||
<div
|
||||
className={cn(
|
||||
"overflow-hidden rounded-2xl text-left leading-6",
|
||||
isUser
|
||||
? "rounded-br-md bg-primary text-primary-foreground"
|
||||
: "rounded-bl-md border border-hairline bg-background text-foreground shadow-sm",
|
||||
isImageMessage ? "p-1" : "px-4 py-3",
|
||||
)}
|
||||
>
|
||||
{images.map((image) => {
|
||||
const url = artifactUrl(image.contentUrl);
|
||||
return (
|
||||
<a
|
||||
key={image.id}
|
||||
href={url}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
className="block overflow-hidden rounded-[0.8rem] bg-black/10"
|
||||
title="在新窗口查看原图"
|
||||
>
|
||||
{/* eslint-disable-next-line @next/next/no-img-element */}
|
||||
<img
|
||||
src={url}
|
||||
alt="用户在通话中拍摄的照片"
|
||||
className="max-h-[28rem] w-full object-contain"
|
||||
/>
|
||||
</a>
|
||||
);
|
||||
})}
|
||||
{isImageMessage && images.length === 0 && (
|
||||
<div className="flex min-h-40 min-w-56 flex-col items-center justify-center gap-2 rounded-[0.8rem] bg-background/10 px-6 text-center text-sm opacity-75">
|
||||
<ImageIcon size={22} />
|
||||
图片附件不可用
|
||||
</div>
|
||||
)}
|
||||
{message.content && (
|
||||
<div className={isImageMessage ? "px-3 py-2" : ""}>{message.content}</div>
|
||||
)}
|
||||
</div>
|
||||
{(message.extra.source || message.extra.node_id) && (
|
||||
<div className={cn("mt-1.5 text-[11px] text-muted-soft", isUser && "text-right")}>
|
||||
{[message.extra.source, message.extra.node_id].filter(Boolean).join(" · ")}
|
||||
</div>
|
||||
)}
|
||||
{message.extra.interrupted && (
|
||||
<div className="mt-1.5 text-xs text-muted-foreground">
|
||||
回复在生成中被打断
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function TimelineTrace({
|
||||
event,
|
||||
nodes,
|
||||
startedEvent,
|
||||
}: {
|
||||
event: TraceEvent;
|
||||
nodes: Map<string, WorkflowNode>;
|
||||
startedEvent?: TraceEvent;
|
||||
}) {
|
||||
const presentation = tracePresentation(event, nodes, startedEvent);
|
||||
const Icon = presentation.icon;
|
||||
const iconClass = {
|
||||
default: "bg-surface-strong text-muted-foreground",
|
||||
success: "bg-success/10 text-success",
|
||||
error: "bg-destructive/10 text-destructive",
|
||||
user: "bg-primary/10 text-primary",
|
||||
}[presentation.tone];
|
||||
|
||||
return (
|
||||
<div className="flex gap-3 rounded-xl border border-hairline-soft bg-background/65 p-3">
|
||||
<div className={cn("flex size-8 shrink-0 items-center justify-center rounded-full", iconClass)}>
|
||||
<Icon size={15} />
|
||||
</div>
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="flex flex-wrap items-start justify-between gap-x-4 gap-y-1">
|
||||
<div>
|
||||
<div className="text-sm font-medium text-foreground">{presentation.title}</div>
|
||||
<div className="mt-0.5 text-xs leading-5 text-muted-foreground">
|
||||
{presentation.description}
|
||||
</div>
|
||||
</div>
|
||||
<time className="shrink-0 text-[11px] tabular-nums text-muted-soft">
|
||||
{formatDate(textValue(event.timestamp))}
|
||||
</time>
|
||||
</div>
|
||||
<details className="mt-2 text-xs text-muted-foreground">
|
||||
<summary className="w-fit cursor-pointer select-none hover:text-foreground">
|
||||
详细数据
|
||||
</summary>
|
||||
<pre className="mt-2 max-h-64 overflow-auto rounded-lg bg-surface-strong p-3 font-mono text-[11px] leading-5 text-foreground">
|
||||
{JSON.stringify(event, null, 2)}
|
||||
</pre>
|
||||
</details>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -91,13 +91,31 @@ function MobileCallTranscript({ messages }: { messages: ChatMessage[] }) {
|
||||
</span>
|
||||
<div
|
||||
className={[
|
||||
"whitespace-pre-wrap rounded-2xl px-3.5 py-2.5 text-sm leading-6 shadow-sm",
|
||||
"overflow-hidden whitespace-pre-wrap rounded-2xl text-sm leading-6 shadow-sm",
|
||||
isAssistant
|
||||
? "rounded-tl-sm bg-white/10 text-white/90"
|
||||
: "rounded-tr-sm bg-white text-[#07101a]",
|
||||
? "rounded-tl-sm bg-white/10 px-3.5 py-2.5 text-white/90"
|
||||
: `rounded-tr-sm bg-white text-[#07101a] ${
|
||||
message.attachments?.length ? "p-1" : "px-3.5 py-2.5"
|
||||
}`,
|
||||
].join(" ")}
|
||||
>
|
||||
{message.content || (message.streaming ? "…" : "")}
|
||||
{message.attachments?.map((attachment) => (
|
||||
// eslint-disable-next-line @next/next/no-img-element
|
||||
<img
|
||||
key={attachment.id}
|
||||
src={attachment.url}
|
||||
alt={attachment.alt}
|
||||
className="max-h-[52dvh] w-full rounded-[0.8rem] object-cover"
|
||||
/>
|
||||
))}
|
||||
{message.content && (
|
||||
<div className={message.attachments?.length ? "px-2.5 py-2" : ""}>
|
||||
{message.content}
|
||||
</div>
|
||||
)}
|
||||
{!message.content && !message.attachments?.length && message.streaming
|
||||
? "…"
|
||||
: null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
@@ -166,7 +184,10 @@ function MobileCallVisualWorkspace({
|
||||
} | null>(null);
|
||||
const latestMessage = [...messages]
|
||||
.reverse()
|
||||
.find((message) => message.content.trim());
|
||||
.find(
|
||||
(message) =>
|
||||
message.content.trim() || (message.attachments?.length ?? 0) > 0,
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const container = containerRef.current;
|
||||
@@ -283,7 +304,10 @@ function MobileCallVisualWorkspace({
|
||||
<span className="mr-1.5 font-medium text-white/55">
|
||||
{latestMessage?.role === "user" ? "我:" : "助手:"}
|
||||
</span>
|
||||
{latestMessage?.content || "暂无消息,点击返回聊天记录"}
|
||||
{latestMessage?.content ||
|
||||
(latestMessage?.attachments?.length
|
||||
? "发送了一张照片"
|
||||
: "暂无消息,点击返回聊天记录")}
|
||||
</span>
|
||||
</button>
|
||||
</>
|
||||
|
||||
@@ -19,8 +19,64 @@ const PHOTO_BUTTON_DEFINITION = {
|
||||
],
|
||||
} as const;
|
||||
|
||||
const MAX_PREVIEW_EDGE = 1280;
|
||||
|
||||
async function capturePreviewImage(stream: MediaStream | null): Promise<string> {
|
||||
const track = stream?.getVideoTracks()[0];
|
||||
if (!stream || !track || track.readyState !== "live") {
|
||||
throw new Error("当前没有可用的摄像头画面");
|
||||
}
|
||||
|
||||
const video = document.createElement("video");
|
||||
video.muted = true;
|
||||
video.playsInline = true;
|
||||
video.srcObject = stream;
|
||||
try {
|
||||
if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const timeout = window.setTimeout(
|
||||
() => reject(new Error("等待摄像头预览超时")),
|
||||
2_000,
|
||||
);
|
||||
video.addEventListener(
|
||||
"loadeddata",
|
||||
() => {
|
||||
window.clearTimeout(timeout);
|
||||
resolve();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
}
|
||||
await video.play();
|
||||
|
||||
const sourceWidth = video.videoWidth || track.getSettings().width || 0;
|
||||
const sourceHeight = video.videoHeight || track.getSettings().height || 0;
|
||||
if (!sourceWidth || !sourceHeight) {
|
||||
throw new Error("摄像头画面尺寸不可用");
|
||||
}
|
||||
const scale = Math.min(1, MAX_PREVIEW_EDGE / Math.max(sourceWidth, sourceHeight));
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = Math.max(1, Math.round(sourceWidth * scale));
|
||||
canvas.height = Math.max(1, Math.round(sourceHeight * scale));
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) throw new Error("浏览器无法生成照片预览");
|
||||
context.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||||
return canvas.toDataURL("image/jpeg", 0.85);
|
||||
} finally {
|
||||
video.pause();
|
||||
video.srcObject = null;
|
||||
}
|
||||
}
|
||||
|
||||
export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
|
||||
const { registerClientTool, sendUserInput, status } = preview;
|
||||
const {
|
||||
appendUserImage,
|
||||
registerClientTool,
|
||||
sendUserInput,
|
||||
status,
|
||||
videoStream,
|
||||
} = preview;
|
||||
const [visible, setVisible] = useState(false);
|
||||
const [capturing, setCapturing] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
@@ -52,7 +108,9 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
|
||||
setCapturing(true);
|
||||
setError(null);
|
||||
try {
|
||||
await sendUserInput(
|
||||
const timestamp = new Date().toISOString();
|
||||
const imageUrl = await capturePreviewImage(videoStream);
|
||||
const result = await sendUserInput(
|
||||
[
|
||||
{
|
||||
type: "input_image",
|
||||
@@ -61,6 +119,7 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
|
||||
],
|
||||
{ runImmediately: true, interrupt: true },
|
||||
);
|
||||
appendUserImage(result.inputId, imageUrl, timestamp);
|
||||
} catch (captureError) {
|
||||
setError(
|
||||
captureError instanceof Error ? captureError.message : "拍照提交失败",
|
||||
@@ -68,7 +127,7 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
|
||||
} finally {
|
||||
setCapturing(false);
|
||||
}
|
||||
}, [capturing, sendUserInput, status]);
|
||||
}, [appendUserImage, capturing, sendUserInput, status, videoStream]);
|
||||
|
||||
return {
|
||||
visible,
|
||||
|
||||
@@ -45,6 +45,14 @@ export type ChatMessage = {
|
||||
sequence: number;
|
||||
turnId?: string;
|
||||
streaming?: boolean;
|
||||
attachments?: ChatAttachment[];
|
||||
};
|
||||
|
||||
export type ChatAttachment = {
|
||||
id: string;
|
||||
type: "image";
|
||||
url: string;
|
||||
alt: string;
|
||||
};
|
||||
|
||||
type AppMessage = Record<string, unknown> & { type?: string };
|
||||
@@ -53,7 +61,9 @@ export type UserInputPart =
|
||||
| { type: "input_text"; text: string }
|
||||
| {
|
||||
type: "input_image";
|
||||
source: { type: "camera_frame"; frame: "current" };
|
||||
source:
|
||||
| { type: "camera_frame"; frame: "current" }
|
||||
| { type: "uploaded_asset"; asset_token: string };
|
||||
};
|
||||
export type UserInputResult = {
|
||||
inputId: string;
|
||||
@@ -871,6 +881,39 @@ export function useVoicePreview(
|
||||
[],
|
||||
);
|
||||
|
||||
const appendUserImage = useCallback(
|
||||
(
|
||||
inputId: string,
|
||||
imageUrl: string,
|
||||
timestamp: string,
|
||||
content = "",
|
||||
) => {
|
||||
messageSeqRef.current += 1;
|
||||
const sequence = messageSeqRef.current;
|
||||
setMessages((previous) =>
|
||||
sortMessages([
|
||||
...previous,
|
||||
{
|
||||
id: `user-image-${inputId}`,
|
||||
role: "user",
|
||||
content: content.trim(),
|
||||
timestamp,
|
||||
sequence,
|
||||
attachments: [
|
||||
{
|
||||
id: `image-${inputId}`,
|
||||
type: "image",
|
||||
url: imageUrl,
|
||||
alt: "用户提交的图片",
|
||||
},
|
||||
],
|
||||
},
|
||||
]),
|
||||
);
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const updateSession = useCallback(
|
||||
({
|
||||
dynamicVariables,
|
||||
@@ -962,6 +1005,7 @@ export function useVoicePreview(
|
||||
supportsOutputSelection,
|
||||
sendText,
|
||||
sendUserInput,
|
||||
appendUserImage,
|
||||
updateSession,
|
||||
registerClientTool,
|
||||
connect,
|
||||
|
||||
@@ -325,7 +325,24 @@ export type ConversationMessage = {
|
||||
contentType: string;
|
||||
content: string;
|
||||
occurredAt: string;
|
||||
extra: { interrupted?: boolean; turn_id?: string };
|
||||
extra: {
|
||||
interrupted?: boolean;
|
||||
turn_id?: string;
|
||||
source?: string;
|
||||
node_id?: string;
|
||||
input_id?: string;
|
||||
};
|
||||
artifacts: ConversationArtifact[];
|
||||
};
|
||||
|
||||
export type ConversationArtifact = {
|
||||
id: string;
|
||||
kind: string;
|
||||
contentUrl: string;
|
||||
mimeType: string;
|
||||
sizeBytes: number | null;
|
||||
durationMs: number | null;
|
||||
extra: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type ConversationDetail = Conversation & {
|
||||
@@ -372,6 +389,30 @@ export const conversationsApi = {
|
||||
request<{ ok: boolean }>(`/api/conversations/${id}`, { method: "DELETE" }),
|
||||
};
|
||||
|
||||
// ---------- 调试会话临时图片 ----------
|
||||
export type InputImageAsset = {
|
||||
assetToken: string;
|
||||
width: number;
|
||||
height: number;
|
||||
sizeBytes: number;
|
||||
};
|
||||
|
||||
export const inputAssetsApi = {
|
||||
uploadImage: (file: File) => {
|
||||
const body = new FormData();
|
||||
body.append("file", file);
|
||||
return request<InputImageAsset>("/api/input-assets/image", {
|
||||
method: "POST",
|
||||
body,
|
||||
});
|
||||
},
|
||||
remove: (assetToken: string) =>
|
||||
request<{ ok: boolean }>(
|
||||
`/api/input-assets/${encodeURIComponent(assetToken)}`,
|
||||
{ method: "DELETE" },
|
||||
),
|
||||
};
|
||||
|
||||
// ---------- 工具 ----------
|
||||
export type ToolStatus = "active" | "archived" | "draft";
|
||||
export type ToolExecutionMode = "immediate" | "async";
|
||||
|
||||
Reference in New Issue
Block a user