From 317a0600bc04850503687ff65f1d3248b5cfaca7 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Wed, 5 Aug 2026 16:14:03 +0800 Subject: [PATCH] feat: persist photos and enrich conversation history --- backend/routes/conversations.py | 73 ++- backend/schemas.py | 11 + backend/services/brains/prompt_brain.py | 54 ++ backend/services/brains/workflow_brain.py | 38 ++ backend/services/client_tools.py | 61 +- backend/services/conversation_history.py | 90 ++- backend/services/knowledge.py | 42 +- backend/services/object_storage.py | 60 ++ backend/services/pipecat/pipeline.py | 10 + backend/services/pipecat/processors.py | 7 +- backend/services/vision.py | 11 +- backend/tests/test_conversation_history.py | 48 ++ backend/tests/test_user_input.py | 21 +- .../assistant-editor/debug-preview.tsx | 34 +- frontend/src/components/pages/HistoryPage.tsx | 548 ++++++++++++++++-- .../src/components/pages/MobileCallPage.tsx | 36 +- frontend/src/hooks/use-photo-capture-tool.ts | 65 ++- frontend/src/hooks/use-voice-preview.ts | 37 ++ frontend/src/lib/api.ts | 19 +- 19 files changed, 1146 insertions(+), 119 deletions(-) create mode 100644 backend/services/object_storage.py diff --git a/backend/routes/conversations.py b/backend/routes/conversations.py index 0c15413..490ac21 100644 --- a/backend/routes/conversations.py +++ b/backend/routes/conversations.py @@ -1,15 +1,20 @@ """对话历史查询 API。""" -from db.models import ConversationMessage, ConversationSession +import asyncio + +from db.models import ConversationArtifact, ConversationMessage, ConversationSession from db.session import get_session -from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi import APIRouter, Depends, HTTPException, Query, Response +from loguru import logger from schemas import ( + ConversationArtifactOut, ConversationDetailOut, ConversationListOut, ConversationMessageOut, ConversationOut, ) from services.auth import require_admin +from services.object_storage import delete_object, get_object, key_from_storage_uri from sqlalchemy import func, or_, select from sqlalchemy.ext.asyncio import AsyncSession @@ -100,6 +105,17 @@ async def get_conversation( .order_by(ConversationMessage.sequence) ) ).scalars().all() + artifacts = ( + await session.execute( + select(ConversationArtifact) + .where(ConversationArtifact.session_id == conversation_id) + .order_by(ConversationArtifact.created_at) + ) + ).scalars().all() + artifacts_by_message: dict[str, list[ConversationArtifact]] = {} + for artifact in artifacts: + if artifact.message_id: + artifacts_by_message.setdefault(artifact.message_id, []).append(artifact) return ConversationDetailOut( **_session_out(conversation).model_dump(), extra=conversation.extra or {}, @@ -112,12 +128,52 @@ async def get_conversation( content=message.content, occurred_at=message.occurred_at, extra=message.extra or {}, + artifacts=[ + ConversationArtifactOut( + id=artifact.id, + kind=artifact.kind, + content_url=( + f"/api/conversations/{conversation_id}/artifacts/" + f"{artifact.id}/content" + ), + mime_type=artifact.mime_type, + size_bytes=artifact.size_bytes, + duration_ms=artifact.duration_ms, + extra=artifact.extra or {}, + ) + for artifact in artifacts_by_message.get(message.id, []) + ], ) for message in messages ], ) +@router.get("/{conversation_id}/artifacts/{artifact_id}/content") +async def get_conversation_artifact( + conversation_id: str, + artifact_id: str, + session: AsyncSession = Depends(get_session), +): + artifact = await session.get(ConversationArtifact, artifact_id) + if not artifact or artifact.session_id != conversation_id: + raise HTTPException(404, "会话附件不存在") + try: + key = key_from_storage_uri(artifact.storage_uri) + data = await asyncio.to_thread(get_object, key) + except Exception as exc: + logger.warning(f"读取会话附件失败: {artifact_id}: {exc}") + raise HTTPException(404, "会话附件不可用") from exc + return Response( + content=data, + media_type=artifact.mime_type or "application/octet-stream", + headers={ + "Cache-Control": "private, max-age=300", + "Content-Disposition": f'inline; filename="{artifact.id}"', + }, + ) + + @router.delete("/{conversation_id}") async def delete_conversation( conversation_id: str, @@ -126,6 +182,19 @@ async def delete_conversation( conversation = await session.get(ConversationSession, conversation_id) if not conversation: raise HTTPException(404, "对话记录不存在") + storage_uris = ( + await session.execute( + select(ConversationArtifact.storage_uri).where( + ConversationArtifact.session_id == conversation_id + ) + ) + ).scalars().all() await session.delete(conversation) await session.commit() + for storage_uri in storage_uris: + try: + key = key_from_storage_uri(storage_uri) + await asyncio.to_thread(delete_object, key) + except Exception as exc: + logger.warning(f"清理已删除会话的附件失败: {storage_uri}: {exc}") return {"ok": True} diff --git a/backend/schemas.py b/backend/schemas.py index d3e5aae..3634c8b 100644 --- a/backend/schemas.py +++ b/backend/schemas.py @@ -467,6 +467,16 @@ class ModelResourceTestResult(CamelModel): # ---------- 对话历史 ---------- +class ConversationArtifactOut(CamelModel): + id: str + kind: str + content_url: str + mime_type: str + size_bytes: int | None = None + duration_ms: int | None = None + extra: dict[str, Any] = Field(default_factory=dict) + + class ConversationMessageOut(CamelModel): id: str sequence: int @@ -475,6 +485,7 @@ class ConversationMessageOut(CamelModel): content: str occurred_at: datetime extra: dict[str, Any] = Field(default_factory=dict) + artifacts: list[ConversationArtifactOut] = Field(default_factory=list) class ConversationOut(CamelModel): diff --git a/backend/services/brains/prompt_brain.py b/backend/services/brains/prompt_brain.py index 1098b98..f33eb9b 100644 --- a/backend/services/brains/prompt_brain.py +++ b/backend/services/brains/prompt_brain.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio from collections.abc import Awaitable +from time import monotonic from typing import Any from uuid import uuid4 @@ -503,12 +504,45 @@ class PromptBrain(BaseBrain): await params.result_callback(result) async def call_tool(params: FunctionCallParams) -> None: + invocation_id = f"tool_{uuid4().hex[:20]}" + started_at = monotonic() + await self._emit_trace( + "tool_started", + invocationId=invocation_id, + toolId=tool.id, + toolName=tool.name, + functionName=tool.function_name, + toolType=tool.type, + ) try: result = await self._tools.execute(tool, dict(params.arguments or {})) if result["updated_variables"]: self._refresh_prompt() + await self._emit_trace( + "tool_completed" if result.get("status") == "ok" else "tool_failed", + invocationId=invocation_id, + toolId=tool.id, + toolName=tool.name, + functionName=tool.function_name, + toolType=tool.type, + status=str(result.get("status") or "unknown"), + durationMs=max(0, round((monotonic() - started_at) * 1000)), + updatedVariables=list(result.get("updated_variables") or []), + resultKeys=sorted(str(name) for name in result if name != "data"), + ) await return_result(params, result) except (ToolExecutionError, ValueError) as exc: + await self._emit_trace( + "tool_failed", + invocationId=invocation_id, + toolId=tool.id, + toolName=tool.name, + functionName=tool.function_name, + toolType=tool.type, + status="error", + durationMs=max(0, round((monotonic() - started_at) * 1000)), + error=str(exc)[:2048], + ) await return_result( params, {"status": "error", "message": f"工具调用失败: {exc}"}, @@ -522,6 +556,26 @@ class PromptBrain(BaseBrain): ) return schema, call_tool + async def _emit_trace(self, event: str, **details: Any) -> None: + runtime = self._runtime + if runtime is None: + return + try: + await runtime.queue_frame( + OutputTransportMessageUrgentFrame( + message={ + "type": "workflow-event", + "eventId": f"wfe_{uuid4().hex[:20]}", + "event": event, + "timestamp": time_now_iso8601(), + "sessionId": runtime.session_id, + **details, + } + ) + ) + except Exception as exc: # noqa: BLE001 - trace must not alter execution + logger.warning(f"发送 Prompt 工具轨迹失败，不影响当前调用: {exc}") + def _make_end_call_tool(self, tool, runtime: BrainRuntime): config = (tool.definition or {}).get("config") or {} message_type = str(config.get("message_type") or "none") diff --git a/backend/services/brains/workflow_brain.py b/backend/services/brains/workflow_brain.py index 59920e5..a450eeb 100644 --- a/backend/services/brains/workflow_brain.py +++ b/backend/services/brains/workflow_brain.py @@ -6,7 +6,9 @@ import asyncio from collections.abc import Awaitable from copy import deepcopy from dataclasses import dataclass, replace +from time import monotonic from typing import Any +from uuid import uuid4 from loguru import logger from models import AssistantConfig, RuntimeTool @@ -764,10 +766,46 @@ class WorkflowBrain(BaseBrain): async def handler(args, _flow_manager): transition_id = self._state.transition_id + invocation_id = f"tool_{uuid4().hex[:20]}" + started_at = monotonic() + await self._emit_trace( + "tool_started", + nodeId=node_id, + invocationId=invocation_id, + toolId=tool.id, + toolName=tool.name, + functionName=tool.function_name, + toolType=tool.type, + ) try: result = await self._tools.execute(tool, dict(args or {})) except ToolExecutionError as exc: + await self._emit_trace( + "tool_failed", + nodeId=node_id, + invocationId=invocation_id, + toolId=tool.id, + toolName=tool.name, + functionName=tool.function_name, + toolType=tool.type, + status="error", + durationMs=max(0, round((monotonic() - started_at) * 1000)), + error=str(exc)[:2048], + ) return {"status": "error", "message": str(exc)} + await self._emit_trace( + "tool_completed" if result.get("status") == "ok" else "tool_failed", + nodeId=node_id, + invocationId=invocation_id, + toolId=tool.id, + toolName=tool.name, + functionName=tool.function_name, + toolType=tool.type, + status=str(result.get("status") or "unknown"), + durationMs=max(0, round((monotonic() - started_at) * 1000)), + updatedVariables=list(result.get("updated_variables") or []), + resultKeys=sorted(str(name) for name in result if name != "data"), + ) if ( self._state.current_node_id != node_id or self._state.transition_id != transition_id diff --git a/backend/services/client_tools.py b/backend/services/client_tools.py index 5f6ca83..1fe1cf7 100644 --- a/backend/services/client_tools.py +++ b/backend/services/client_tools.py @@ -7,6 +7,7 @@ from collections import deque from collections.abc import Awaitable, Callable from dataclasses import dataclass from typing import Any, Literal, Protocol +from uuid import uuid4 from loguru import logger from pipecat.frames.frames import ( @@ -17,6 +18,7 @@ from pipecat.frames.frames import ( StopFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.utils.time import time_now_iso8601 class ClientToolError(RuntimeError): @@ -29,6 +31,7 @@ ClientToolResponseWaitMode = Literal["timeout", "session"] @dataclass(frozen=True) class _PendingClientToolCall: future: asyncio.Future[dict[str, Any]] + function_name: str interrupt_on_result: bool = False @@ -80,8 +83,6 @@ class ClientToolBroker(FrameProcessor): response_wait_mode: ClientToolResponseWaitMode = "timeout", interrupt_on_result: bool = False, ) -> dict[str, Any]: - from uuid import uuid4 - if self._closed_message is not None: raise ClientToolError(self._closed_message) @@ -99,9 +100,21 @@ class ClientToolBroker(FrameProcessor): OutputTransportMessageUrgentFrame(message=message) ) except Exception as exc: + await self._emit_trace( + "client_tool_failed", + toolCallId=tool_call_id, + functionName=function_name, + error="客户端工具调用发送失败", + ) raise ClientToolError( f"客户端工具调用发送失败: {function_name}" ) from exc + await self._emit_trace( + "client_tool_completed", + toolCallId=tool_call_id, + functionName=function_name, + status="dispatched", + ) return { "status": "ok", "data": {"dispatched": True}, @@ -111,6 +124,7 @@ class ClientToolBroker(FrameProcessor): future: asyncio.Future[dict[str, Any]] = loop.create_future() self._pending[tool_call_id] = _PendingClientToolCall( future=future, + function_name=function_name, interrupt_on_result=interrupt_on_result, ) try: @@ -121,6 +135,12 @@ class ClientToolBroker(FrameProcessor): return await future return await asyncio.wait_for(future, timeout=timeout_seconds) except TimeoutError as exc: + await self._emit_trace( + "client_tool_failed", + toolCallId=tool_call_id, + functionName=function_name, + error="客户端工具调用超时", + ) raise ClientToolError(f"客户端工具调用超时: {function_name}") from exc except ClientToolError: raise @@ -163,10 +183,28 @@ class ClientToolBroker(FrameProcessor): future = pending.future status = str(message.get("status") or "error") + result_data = message.get("data") + user_action = ( + str(result_data.get("action") or "") + if isinstance(result_data, dict) + else "" + ) + await self._emit_trace( + "client_tool_completed" if status == "ok" else "client_tool_failed", + toolCallId=tool_call_id, + functionName=pending.function_name, + status=status, + **({"userAction": user_action} if user_action else {}), + **( + {"error": str(message.get("message") or "客户端工具执行失败")} + if status != "ok" + else {} + ), + ) if status == "ok": response = { "status": "ok", - "data": message.get("data"), + "data": result_data, } if pending.interrupt_on_result: # Match text input exactly: broadcasting only starts the @@ -206,6 +244,23 @@ class ClientToolBroker(FrameProcessor): } ) + async def _emit_trace(self, event: str, **details: Any) -> None: + """Publish sanitized client interaction metadata for conversation history.""" + try: + await self.push_frame( + OutputTransportMessageUrgentFrame( + message={ + "type": "workflow-event", + "eventId": f"wfe_{uuid4().hex[:20]}", + "event": event, + "timestamp": time_now_iso8601(), + **details, + } + ) + ) + except Exception as exc: # noqa: BLE001 - tracing must not alter the tool call + logger.warning(f"发送客户端工具轨迹失败，不影响工具调用: {exc}") + def on_interruption_processed(self) -> None: """Release one result after the aggregator acknowledges interruption.""" while self._deferred_results: diff --git a/backend/services/conversation_history.py b/backend/services/conversation_history.py index 5787070..e07b6a0 100644 --- a/backend/services/conversation_history.py +++ b/backend/services/conversation_history.py @@ -9,9 +9,10 @@ from copy import deepcopy from datetime import UTC, datetime from uuid import uuid4 -from db.models import ConversationMessage, ConversationSession +from db.models import ConversationArtifact, ConversationMessage, ConversationSession from db.session import SessionLocal from loguru import logger +from services.object_storage import delete_object, put_object, storage_uri MAX_WORKFLOW_TRACE_EVENTS = 1000 @@ -36,6 +37,7 @@ class ConversationRecorder: self._trace_sequence = 0 self._lock = asyncio.Lock() self._seen_events: set[str] = set() + self._pending_artifacts: set[asyncio.Task[None]] = set() @classmethod async def start( @@ -168,6 +170,89 @@ class ConversationRecorder: except Exception as exc: logger.error(f"保存对话文本失败，不影响本次通话: {exc}") + def record_image_later( + self, + data: bytes, + *, + input_id: str, + timestamp: object, + mime_type: str = "image/jpeg", + ) -> None: + """Persist a captured frame without delaying the realtime model turn.""" + + task = asyncio.create_task( + self._record_image( + data, + input_id=input_id, + timestamp=timestamp, + mime_type=mime_type, + ), + name=f"conversation-image:{self.session_id}:{input_id}", + ) + self._pending_artifacts.add(task) + task.add_done_callback(self._pending_artifacts.discard) + + async def _record_image( + self, + data: bytes, + *, + input_id: str, + timestamp: object, + mime_type: str, + ) -> None: + message_id = f"msg_{uuid4().hex[:20]}" + artifact_id = f"artifact_{uuid4().hex[:20]}" + extension = ".jpg" if mime_type == "image/jpeg" else ".bin" + key = f"conversations/{self.session_id}/{artifact_id}{extension}" + try: + await asyncio.to_thread(put_object, key, data, mime_type) + except Exception as exc: + logger.error(f"保存会话图片到对象存储失败，不影响本次通话: {exc}") + return + + async with self._lock: + next_sequence = self._sequence + 1 + try: + async with SessionLocal() as db: + db.add( + ConversationMessage( + id=message_id, + session_id=self.session_id, + sequence=next_sequence, + role="user", + content_type="image", + content="", + occurred_at=_parse_timestamp(timestamp), + extra={ + "input_id": input_id, + "source": "camera_capture", + }, + ) + ) + db.add( + ConversationArtifact( + id=artifact_id, + session_id=self.session_id, + message_id=message_id, + kind="image", + storage_uri=storage_uri(key), + mime_type=mime_type, + size_bytes=len(data), + extra={"input_id": input_id}, + ) + ) + conversation = await db.get(ConversationSession, self.session_id) + if conversation: + conversation.message_count = next_sequence + await db.commit() + self._sequence = next_sequence + except Exception as exc: + logger.error(f"保存会话图片索引失败，不影响本次通话: {exc}") + try: + await asyncio.to_thread(delete_object, key) + except Exception: + logger.warning(f"清理未关联的会话图片失败: {key}") + async def finish(self, *, status: str = "completed") -> None: """Finish the session even if the owning pipeline is being cancelled.""" finish_task = asyncio.create_task( @@ -184,6 +269,9 @@ class ConversationRecorder: raise async def _finish(self, *, status: str) -> None: + pending_artifacts = list(self._pending_artifacts) + if pending_artifacts: + await asyncio.gather(*pending_artifacts, return_exceptions=True) async with self._lock: try: async with SessionLocal() as db: diff --git a/backend/services/knowledge.py b/backend/services/knowledge.py index 147794f..be9d7e4 100644 --- a/backend/services/knowledge.py +++ b/backend/services/knowledge.py @@ -6,9 +6,6 @@ from pathlib import Path import re import uuid -import boto3 -from botocore.config import Config -from botocore.exceptions import ClientError from docx import Document as DocxDocument from openai import AsyncOpenAI from pypdf import PdfReader @@ -17,38 +14,7 @@ from sqlalchemy.ext.asyncio import AsyncSession import settings from db.models import KnowledgeBase, KnowledgeChunk, KnowledgeDocument, ModelResource - - -def _s3_client(): - return boto3.client( - "s3", - endpoint_url=settings.S3_ENDPOINT_URL, - aws_access_key_id=settings.S3_ACCESS_KEY, - aws_secret_access_key=settings.S3_SECRET_KEY, - region_name=settings.S3_REGION, - config=Config(signature_version="s3v4", s3={"addressing_style": "path"}), - ) - - -def _ensure_bucket_and_put(key: str, data: bytes, mime_type: str) -> None: - client = _s3_client() - try: - client.head_bucket(Bucket=settings.S3_BUCKET) - except ClientError as exc: - status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") - if status != 404: - raise - client.create_bucket(Bucket=settings.S3_BUCKET) - client.put_object(Bucket=settings.S3_BUCKET, Key=key, Body=data, ContentType=mime_type) - - -def _delete_object(key: str) -> None: - _s3_client().delete_object(Bucket=settings.S3_BUCKET, Key=key) - - -def _get_object(key: str) -> bytes: - response = _s3_client().get_object(Bucket=settings.S3_BUCKET, Key=key) - return response["Body"].read() +from services.object_storage import delete_object, get_object, put_object def extract_text(filename: str, data: bytes) -> str: @@ -122,7 +88,7 @@ async def create_document( safe_name = Path(name).name extension = ".txt" if source_type == "text" else Path(safe_name).suffix storage_key = f"knowledge/{kb.id}/{document_id}/source{extension}" - await asyncio.to_thread(_ensure_bucket_and_put, storage_key, raw_data, mime_type) + await asyncio.to_thread(put_object, storage_key, raw_data, mime_type) document = KnowledgeDocument( id=document_id, @@ -158,7 +124,7 @@ async def process_document(document_id: str) -> None: await session.commit() try: - data = await asyncio.to_thread(_get_object, document.storage_key) + data = await asyncio.to_thread(get_object, document.storage_key) text = ( data.decode("utf-8", errors="replace") if document.source_type == "text" @@ -245,4 +211,4 @@ async def search( async def delete_storage_object(document: KnowledgeDocument) -> None: if document.storage_key: - await asyncio.to_thread(_delete_object, document.storage_key) + await asyncio.to_thread(delete_object, document.storage_key) diff --git a/backend/services/object_storage.py b/backend/services/object_storage.py new file mode 100644 index 0000000..189c8a7 --- /dev/null +++ b/backend/services/object_storage.py @@ -0,0 +1,60 @@ +"""Small S3-compatible object-storage adapter shared by backend features.""" + +from __future__ import annotations + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +import settings + + +def _client(): + return boto3.client( + "s3", + endpoint_url=settings.S3_ENDPOINT_URL, + aws_access_key_id=settings.S3_ACCESS_KEY, + aws_secret_access_key=settings.S3_SECRET_KEY, + region_name=settings.S3_REGION, + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}), + ) + + +def put_object(key: str, data: bytes, mime_type: str) -> None: + client = _client() + try: + client.head_bucket(Bucket=settings.S3_BUCKET) + except ClientError as exc: + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + if status != 404: + raise + client.create_bucket(Bucket=settings.S3_BUCKET) + client.put_object( + Bucket=settings.S3_BUCKET, + Key=key, + Body=data, + ContentType=mime_type, + ) + + +def get_object(key: str) -> bytes: + response = _client().get_object(Bucket=settings.S3_BUCKET, Key=key) + return response["Body"].read() + + +def delete_object(key: str) -> None: + _client().delete_object(Bucket=settings.S3_BUCKET, Key=key) + + +def storage_uri(key: str) -> str: + return f"s3://{settings.S3_BUCKET}/{key.lstrip('/')}" + + +def key_from_storage_uri(uri: str) -> str: + prefix = f"s3://{settings.S3_BUCKET}/" + if not uri.startswith(prefix): + raise ValueError("不支持的会话附件存储地址") + key = uri[len(prefix) :].strip("/") + if not key: + raise ValueError("会话附件存储地址缺少对象键") + return key diff --git a/backend/services/pipecat/pipeline.py b/backend/services/pipecat/pipeline.py index db60e86..a402c2a 100644 --- a/backend/services/pipecat/pipeline.py +++ b/backend/services/pipecat/pipeline.py @@ -36,6 +36,7 @@ from services.vision import ( config_with_main_llm_as_vision, config_with_vision_resource, image_data_uri, + image_jpeg_bytes, ) from services.workflow_engine import WorkflowEngine @@ -58,6 +59,7 @@ from pipecat.processors.aggregators.llm_response_universal import ( ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.llm_service import FunctionCallParams +from pipecat.utils.time import time_now_iso8601 from services.pipecat.turn_config import ( ConfigurableLLMUserAggregator, create_user_turn_strategies, @@ -763,6 +765,14 @@ async def run_pipeline( except asyncio.TimeoutError as exc: raise ValueError("等待摄像头视频帧超时") from exc + if recorder: + image_bytes = await asyncio.to_thread(image_jpeg_bytes, image_frame) + recorder.record_image_later( + image_bytes, + input_id=value.input_id, + timestamp=time_now_iso8601(), + ) + if native_vision: input_frame = await asyncio.to_thread( _multimodal_user_input_frame, diff --git a/backend/services/pipecat/processors.py b/backend/services/pipecat/processors.py index b9a97a8..88fb90e 100644 --- a/backend/services/pipecat/processors.py +++ b/backend/services/pipecat/processors.py @@ -61,9 +61,10 @@ class UserInput: @property def transcript_text(self) -> str: - if not self.has_camera_frame: - return self.text - return f"{self.text}\n已发送一张图片".strip() + # Images are represented as structured media records. A synthetic + # sentence would be shown as chat text and persisted as if the user had + # said it, while adding no visual information for the model. + return self.text class UserInputError(ValueError): diff --git a/backend/services/vision.py b/backend/services/vision.py index 0bec588..86a166b 100644 --- a/backend/services/vision.py +++ b/backend/services/vision.py @@ -30,8 +30,8 @@ def _require(value: str, label: str) -> str: raise ValueError(f"缺少模型资源配置: {label}") -def image_data_uri(frame: UserImageRawFrame) -> str: - """Encode one Pipecat camera frame for a vision chat-completion request.""" +def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes: + """Encode one Pipecat camera frame as a storage- and model-ready JPEG.""" if not frame.format: raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型") buffer = BytesIO() @@ -40,7 +40,12 @@ def image_data_uri(frame: UserImageRawFrame) -> str: format="JPEG", quality=85, ) - encoded = base64.b64encode(buffer.getvalue()).decode("utf-8") + return buffer.getvalue() + + +def image_data_uri(frame: UserImageRawFrame) -> str: + """Encode one Pipecat camera frame for a vision chat-completion request.""" + encoded = base64.b64encode(image_jpeg_bytes(frame)).decode("utf-8") return f"data:image/jpeg;base64,{encoded}" diff --git a/backend/tests/test_conversation_history.py b/backend/tests/test_conversation_history.py index a60dcde..650f18b 100644 --- a/backend/tests/test_conversation_history.py +++ b/backend/tests/test_conversation_history.py @@ -105,6 +105,54 @@ class ConversationRecorderTest(unittest.IsolatedAsyncioTestCase): self.assertEqual(saved["sessionId"], "conv_test") self.assertEqual(saved["sequence"], 1) + async def test_image_is_persisted_as_message_with_artifact(self): + conversation = SimpleNamespace(message_count=0) + + class FakeSession: + def __init__(self): + self.added = [] + self.committed = False + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return None + + def add(self, value): + self.added.append(value) + + async def get(self, _model, _session_id): + return conversation + + async def commit(self): + self.committed = True + + session = FakeSession() + recorder = ConversationRecorder("conv_test") + with ( + patch("services.conversation_history.SessionLocal", return_value=session), + patch("services.conversation_history.put_object") as put_object, + ): + await recorder._record_image( + b"jpeg-data", + input_id="input_photo", + timestamp="2026-08-05T10:00:00+08:00", + mime_type="image/jpeg", + ) + + self.assertTrue(session.committed) + self.assertEqual(len(session.added), 2) + message, artifact = session.added + self.assertEqual(message.content_type, "image") + self.assertEqual(message.role, "user") + self.assertEqual(message.extra["input_id"], "input_photo") + self.assertEqual(artifact.message_id, message.id) + self.assertEqual(artifact.kind, "image") + self.assertEqual(artifact.size_bytes, len(b"jpeg-data")) + self.assertEqual(conversation.message_count, 1) + put_object.assert_called_once() + if __name__ == "__main__": unittest.main() diff --git a/backend/tests/test_user_input.py b/backend/tests/test_user_input.py index 3fbaac6..681b5e3 100644 --- a/backend/tests/test_user_input.py +++ b/backend/tests/test_user_input.py @@ -32,7 +32,7 @@ class UserInputParserTests(unittest.TestCase): self.assertIsNotNone(value) self.assertEqual(value.text, "帮我看看") self.assertTrue(value.has_camera_frame) - self.assertEqual(value.transcript_text, "帮我看看\n已发送一张图片") + self.assertEqual(value.transcript_text, "帮我看看") def test_rejects_legacy_and_unsupported_image_sources(self): self.assertIsNone(parse_user_input({"type": "user-text", "text": "旧协议"})) @@ -54,6 +54,25 @@ class UserInputParserTests(unittest.TestCase): } ) + def test_image_only_input_has_no_synthetic_chat_text(self): + value = parse_user_input( + { + "type": "user-input", + "schema_version": 1, + "input_id": "input_photo", + "parts": [ + { + "type": "input_image", + "source": {"type": "camera_frame", "frame": "current"}, + } + ], + } + ) + + self.assertIsNotNone(value) + self.assertEqual(value.transcript_text, "") + self.assertEqual(value.prompt_text, "请根据用户刚提交的图片进行回复。") + def test_native_image_uses_the_standard_multimodal_user_turn_path(self): image = UserImageRawFrame( image=bytes([220, 40, 40] * 16 * 16), diff --git a/frontend/src/components/assistant-editor/debug-preview.tsx b/frontend/src/components/assistant-editor/debug-preview.tsx index 25f3a22..0c72f0e 100644 --- a/frontend/src/components/assistant-editor/debug-preview.tsx +++ b/frontend/src/components/assistant-editor/debug-preview.tsx @@ -1154,7 +1154,10 @@ function DebugVisionWorkspace({ } | null>(null); const latestMessage = [...messages] .reverse() - .find((message) => message.content.trim()); + .find( + (message) => + message.content.trim() || (message.attachments?.length ?? 0) > 0, + ); useEffect(() => { const container = containerRef.current; @@ -1201,7 +1204,10 @@ function DebugVisionWorkspace({ {latestMessage?.role === "user" ? "我：" : "助手："} - {latestMessage?.content || "暂无消息，点击返回聊天记录"} + {latestMessage?.content || + (latestMessage?.attachments?.length + ? "发送了一张照片" + : "暂无消息，点击返回聊天记录")} @@ -1449,7 +1455,7 @@ function DebugTranscriptPanel({ 助手{time ? ` · ${time}` : ""}
- {message.content} + {message.content || (message.streaming ? "…" : "")}
) : ( @@ -1460,8 +1466,26 @@ function DebugTranscriptPanel({ 我{time ? ` · ${time}` : ""} -
- {message.content} +
+ {message.attachments?.map((attachment) => ( + // eslint-disable-next-line @next/next/no-img-element + {attachment.alt} + ))} + {message.content && ( +
+ {message.content} +
+ )}
); diff --git a/frontend/src/components/pages/HistoryPage.tsx b/frontend/src/components/pages/HistoryPage.tsx index 16cf7ae..a7d16a2 100644 --- a/frontend/src/components/pages/HistoryPage.tsx +++ b/frontend/src/components/pages/HistoryPage.tsx @@ -1,13 +1,25 @@ "use client"; import { + Activity, + ArrowRight, + Camera, + CheckCircle2, ChevronDown, ChevronUp, + CircleDot, Eye, + GitBranch, + ImageIcon, Loader2, MessageSquareText, + MonitorSmartphone, MoreHorizontal, + Server, Trash2, + Wrench, + XCircle, + type LucideIcon, } from "lucide-react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; @@ -37,9 +49,11 @@ import { } from "@/components/layout/list-page-layout"; import { SearchInput } from "@/components/ui/search-input"; import { + API_BASE, conversationsApi, type Conversation, type ConversationDetail, + type ConversationMessage, } from "@/lib/api"; import { cn } from "@/lib/utils"; @@ -93,6 +107,249 @@ function statusLabel(status: string): string { return "已结束"; } +type TraceEvent = Record; +type WorkflowNode = { label: string; type: string }; +type TraceTone = "default" | "success" | "error" | "user"; + +type TracePresentation = { + title: string; + description: string; + icon: LucideIcon; + tone: TraceTone; +}; + +function recordValue(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function textValue(value: unknown): string { + return typeof value === "string" ? value : ""; +} + +function numberValue(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function workflowNodes(detail: ConversationDetail): Map { + const snapshot = recordValue(detail.extra.workflow?.snapshot); + const nodes = Array.isArray(snapshot.nodes) ? snapshot.nodes : []; + return new Map( + nodes.flatMap((rawNode) => { + const node = recordValue(rawNode); + const id = textValue(node.id); + if (!id) return []; + const data = recordValue(node.data); + const type = textValue(node.type) || "node"; + return [ + [ + id, + { + label: textValue(data.name) || type, + type, + }, + ] as const, + ]; + }), + ); +} + +function nodeLabel(nodes: Map, nodeId: unknown): string { + const id = textValue(nodeId); + const node = nodes.get(id); + return node ? `「${node.label}」` : id ? `「${id}」` : "当前节点"; +} + +function toolLabel(toolType: unknown): string { + if (toolType === "client") return "客户端工具"; + if (toolType === "http") return "HTTP 服务端工具"; + if (toolType === "mcp") return "MCP 服务端工具"; + if (toolType === "system") return "系统工具"; + return "工具"; +} + +function tracePresentation( + event: TraceEvent, + nodes: Map, + startedEvent?: TraceEvent, +): TracePresentation { + const eventName = textValue(event.event); + const node = nodeLabel(nodes, event.nodeId); + const outcome = recordValue(event.outcome); + const duration = numberValue(outcome.durationMs); + const tool = toolLabel(event.toolType ?? startedEvent?.toolType); + + switch (eventName) { + case "node_entered": + return { + title: `进入节点 ${node}`, + description: `节点类型：${textValue(event.nodeType) || nodes.get(textValue(event.nodeId))?.type || "未知"}`, + icon: CircleDot, + tone: "default", + }; + case "node_exited": + return { + title: `离开节点 ${node}`, + description: "当前节点处理完成，准备选择下一步。", + icon: ArrowRight, + tone: "default", + }; + case "edge_selected": + return { + title: `节点转移：${nodeLabel(nodes, event.sourceNodeId)} → ${nodeLabel(nodes, event.targetNodeId)}`, + description: `路由方式：${textValue(event.edgeMode) || "always"}`, + icon: GitBranch, + tone: "default", + }; + case "action_started": + return { + title: `开始调用${tool}`, + description: `${node} · ${textValue(event.toolId) || "未命名工具"}`, + icon: tool === "客户端工具" ? MonitorSmartphone : Server, + tone: "default", + }; + case "action_completed": + return { + title: `${tool}执行成功`, + description: `${node}${duration === null ? "" : ` · ${duration} ms`}`, + icon: CheckCircle2, + tone: "success", + }; + case "action_failed": + case "action_cancelled": + return { + title: `${tool}${eventName === "action_failed" ? "执行失败" : "已取消"}`, + description: textValue(recordValue(outcome.error).message) || node, + icon: XCircle, + tone: "error", + }; + case "tool_started": + return { + title: `开始调用${tool}`, + description: `${node} · ${textValue(event.toolName) || textValue(event.functionName) || textValue(event.toolId) || "未命名工具"}`, + icon: tool === "客户端工具" ? MonitorSmartphone : Server, + tone: "default", + }; + case "tool_completed": + return { + title: `${tool}调用成功`, + description: `${textValue(event.toolName) || textValue(event.functionName) || node}${numberValue(event.durationMs) === null ? "" : ` · ${numberValue(event.durationMs)} ms`}`, + icon: CheckCircle2, + tone: "success", + }; + case "tool_failed": + return { + title: `${tool}调用失败`, + description: textValue(event.error) || textValue(event.toolName) || node, + icon: XCircle, + tone: "error", + }; + case "client_tool_started": + return { + title: + event.functionName === "show_message" + ? "向客户端显示交互消息" + : "向客户端下发工具调用", + description: textValue(event.functionName) || "客户端工具", + icon: MonitorSmartphone, + tone: "default", + }; + case "client_tool_completed": { + const userAction = textValue(event.userAction); + return { + title: userAction === "confirmed" ? "用户已确认" : "客户端交互已完成", + description: userAction + ? `用户操作：${userAction}` + : textValue(event.functionName) || textValue(event.status) || "执行成功", + icon: CheckCircle2, + tone: userAction ? "user" : "success", + }; + } + case "client_tool_failed": + return { + title: "客户端交互失败", + description: textValue(event.error) || textValue(event.functionName), + icon: XCircle, + tone: "error", + }; + case "message_started": + return event.requiresConfirmation + ? { + title: "等待用户确认", + description: `${node}向客户端显示了确认消息。`, + icon: MonitorSmartphone, + tone: "user", + } + : { + title: "开始播放固定消息", + description: node, + icon: Activity, + tone: "default", + }; + case "message_completed": { + const action = textValue(event.action); + return { + title: action === "confirmed" ? "用户已确认" : "消息步骤已完成", + description: `${node}${action ? ` · 操作：${action}` : ""}`, + icon: CheckCircle2, + tone: action ? "user" : "success", + }; + } + case "message_interrupted": + return { + title: "用户输入打断消息", + description: node, + icon: Activity, + tone: "user", + }; + case "message_failed": + return { + title: "消息步骤失败", + description: textValue(event.error) || node, + icon: XCircle, + tone: "error", + }; + case "variables_updated": { + const names = Array.isArray(event.variableNames) + ? event.variableNames.filter((name): name is string => typeof name === "string") + : []; + return { + title: "会话变量已更新", + description: names.length ? names.join("、") : node, + icon: Wrench, + tone: "default", + }; + } + case "variables_snapshot": + return { + title: "记录会话变量快照", + description: node, + icon: Wrench, + tone: "default", + }; + case "workflow_ended": + return { + title: "工作流已结束", + description: `${node} · ${textValue(event.outcome) || "success"}`, + icon: CheckCircle2, + tone: "success", + }; + default: + return { + title: eventName || "运行事件", + description: node, + icon: Activity, + tone: "default", + }; + } +} + +function artifactUrl(path: string): string { + if (/^(https?:|data:|blob:)/.test(path)) return path; + return `${API_BASE}${path.startsWith("/") ? path : `/${path}`}`; +} + export function HistoryPage() { const [rows, setRows] = useState([]); const [total, setTotal] = useState(0); @@ -370,7 +627,7 @@ export function HistoryPage() { <> - + @@ -432,7 +689,7 @@ export function HistoryPage() { {detail ? `${formatDate(detail.startedAt)} · ${channelLabel(detail.channel)} · ${detail.messageCount} 条消息` - : "查看本次会话中最终确认的文本消息。"} + : "查看本次会话的完整对话与运行记录。"} @@ -464,56 +721,7 @@ export function HistoryPage() { -
-
- 对话内容 -
-
- {detail.messages.map((message) => ( -
-
-
- {message.role === "user" ? "用户" : "助手"} ·{" "} - {formatDate(message.occurredAt)} -
-
- {message.content} -
- {message.extra.interrupted && ( -
- 回复在生成中被打断 -
- )} -
-
- ))} - {detail.messages.length === 0 && ( -
- 本次会话没有产生文本消息 -
- )} -
-
+ )} @@ -537,3 +745,237 @@ function Metadata({ label, value }: { label: string; value: string }) { ); } + +type TimelineEntry = + | { + kind: "message"; + timestamp: string; + order: number; + message: ConversationMessage; + } + | { + kind: "trace"; + timestamp: string; + order: number; + event: TraceEvent; + }; + +function timestampOrder(value: string): number { + const parsed = Date.parse(value); + return Number.isNaN(parsed) ? Number.MAX_SAFE_INTEGER : parsed; +} + +function ConversationTimeline({ detail }: { detail: ConversationDetail }) { + const nodes = workflowNodes(detail); + const trace = (detail.extra.workflowTrace ?? []).map(recordValue); + const startedByInvocation = new Map(); + trace.forEach((event) => { + if (event.event === "action_started" || event.event === "tool_started") { + const invocationId = textValue(event.invocationId); + if (invocationId) startedByInvocation.set(invocationId, event); + } + }); + + const entries: TimelineEntry[] = [ + ...detail.messages.map((message) => ({ + kind: "message" as const, + timestamp: message.occurredAt, + order: message.sequence, + message, + })), + ...trace.map((event, index) => ({ + kind: "trace" as const, + timestamp: textValue(event.timestamp), + order: numberValue(event.sequence) ?? index + 1, + event, + })), + ].sort( + (a, b) => + timestampOrder(a.timestamp) - timestampOrder(b.timestamp) || + a.order - b.order, + ); + + const imageCount = detail.messages.reduce( + (count, message) => + count + (message.artifacts ?? []).filter((item) => item.kind === "image").length, + 0, + ); + const toolCount = trace.filter( + (event) => event.event === "action_started" || event.event === "tool_started", + ).length; + const transitionCount = trace.filter((event) => event.event === "edge_selected").length; + + return ( +
+
+
完整时间线
+
+ + + {detail.messages.length} 条对话 + + {imageCount > 0 && ( + + + {imageCount} 张照片 + + )} + {toolCount > 0 && ( + + + {toolCount} 次工具 + + )} + {transitionCount > 0 && ( + + + {transitionCount} 次转移 + + )} +
+
+ +
+ {entries.map((entry, index) => + entry.kind === "message" ? ( + + ) : ( + + ), + )} + {entries.length === 0 && ( +
+ 本次会话没有产生可展示的记录 +
+ )} +
+
+ ); +} + +function TimelineMessage({ message }: { message: ConversationMessage }) { + const isUser = message.role === "user"; + const images = (message.artifacts ?? []).filter((item) => item.kind === "image"); + const isImageMessage = message.contentType === "image" || images.length > 0; + + return ( +
+
+
+ {isImageMessage && } + {isUser ? "用户" : "助手"} · {formatDate(message.occurredAt)} +
+
+ {images.map((image) => { + const url = artifactUrl(image.contentUrl); + return ( + + {/* eslint-disable-next-line @next/next/no-img-element */} + 用户在通话中拍摄的照片 + + ); + })} + {isImageMessage && images.length === 0 && ( +
+ + 图片附件不可用 +
+ )} + {message.content && ( +
{message.content}
+ )} +
+ {(message.extra.source || message.extra.node_id) && ( +
+ {[message.extra.source, message.extra.node_id].filter(Boolean).join(" · ")} +
+ )} + {message.extra.interrupted && ( +
+ 回复在生成中被打断 +
+ )} +
+
+ ); +} + +function TimelineTrace({ + event, + nodes, + startedEvent, +}: { + event: TraceEvent; + nodes: Map; + startedEvent?: TraceEvent; +}) { + const presentation = tracePresentation(event, nodes, startedEvent); + const Icon = presentation.icon; + const iconClass = { + default: "bg-surface-strong text-muted-foreground", + success: "bg-success/10 text-success", + error: "bg-destructive/10 text-destructive", + user: "bg-primary/10 text-primary", + }[presentation.tone]; + + return ( +
+
+ +
+
+
+
+
{presentation.title}
+
+ {presentation.description} +
+
+ +
+
+ + 详细数据 + + 
+            {JSON.stringify(event, null, 2)}
+
+
+
+
+ ); +} diff --git a/frontend/src/components/pages/MobileCallPage.tsx b/frontend/src/components/pages/MobileCallPage.tsx index 4c7c6ad..2b67ca7 100644 --- a/frontend/src/components/pages/MobileCallPage.tsx +++ b/frontend/src/components/pages/MobileCallPage.tsx @@ -91,13 +91,31 @@ function MobileCallTranscript({ messages }: { messages: ChatMessage[] }) {
- {message.content || (message.streaming ? "…" : "")} + {message.attachments?.map((attachment) => ( + // eslint-disable-next-line @next/next/no-img-element + {attachment.alt} + ))} + {message.content && ( +
+ {message.content} +
+ )} + {!message.content && !message.attachments?.length && message.streaming + ? "…" + : null}
); @@ -166,7 +184,10 @@ function MobileCallVisualWorkspace({ } | null>(null); const latestMessage = [...messages] .reverse() - .find((message) => message.content.trim()); + .find( + (message) => + message.content.trim() || (message.attachments?.length ?? 0) > 0, + ); useEffect(() => { const container = containerRef.current; @@ -283,7 +304,10 @@ function MobileCallVisualWorkspace({ {latestMessage?.role === "user" ? "我：" : "助手："} - {latestMessage?.content || "暂无消息，点击返回聊天记录"} + {latestMessage?.content || + (latestMessage?.attachments?.length + ? "发送了一张照片" + : "暂无消息，点击返回聊天记录")} diff --git a/frontend/src/hooks/use-photo-capture-tool.ts b/frontend/src/hooks/use-photo-capture-tool.ts index 7343f20..c4f9476 100644 --- a/frontend/src/hooks/use-photo-capture-tool.ts +++ b/frontend/src/hooks/use-photo-capture-tool.ts @@ -19,8 +19,64 @@ const PHOTO_BUTTON_DEFINITION = { ], } as const; +const MAX_PREVIEW_EDGE = 1280; + +async function capturePreviewImage(stream: MediaStream | null): Promise { + const track = stream?.getVideoTracks()[0]; + if (!stream || !track || track.readyState !== "live") { + throw new Error("当前没有可用的摄像头画面"); + } + + const video = document.createElement("video"); + video.muted = true; + video.playsInline = true; + video.srcObject = stream; + try { + if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) { + await new Promise((resolve, reject) => { + const timeout = window.setTimeout( + () => reject(new Error("等待摄像头预览超时")), + 2_000, + ); + video.addEventListener( + "loadeddata", + () => { + window.clearTimeout(timeout); + resolve(); + }, + { once: true }, + ); + }); + } + await video.play(); + + const sourceWidth = video.videoWidth || track.getSettings().width || 0; + const sourceHeight = video.videoHeight || track.getSettings().height || 0; + if (!sourceWidth || !sourceHeight) { + throw new Error("摄像头画面尺寸不可用"); + } + const scale = Math.min(1, MAX_PREVIEW_EDGE / Math.max(sourceWidth, sourceHeight)); + const canvas = document.createElement("canvas"); + canvas.width = Math.max(1, Math.round(sourceWidth * scale)); + canvas.height = Math.max(1, Math.round(sourceHeight * scale)); + const context = canvas.getContext("2d"); + if (!context) throw new Error("浏览器无法生成照片预览"); + context.drawImage(video, 0, 0, canvas.width, canvas.height); + return canvas.toDataURL("image/jpeg", 0.85); + } finally { + video.pause(); + video.srcObject = null; + } +} + export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) { - const { registerClientTool, sendUserInput, status } = preview; + const { + appendUserImage, + registerClientTool, + sendUserInput, + status, + videoStream, + } = preview; const [visible, setVisible] = useState(false); const [capturing, setCapturing] = useState(false); const [error, setError] = useState(null); @@ -52,7 +108,9 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) { setCapturing(true); setError(null); try { - await sendUserInput( + const timestamp = new Date().toISOString(); + const imageUrl = await capturePreviewImage(videoStream); + const result = await sendUserInput( [ { type: "input_image", @@ -61,6 +119,7 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) { ], { runImmediately: true, interrupt: true }, ); + appendUserImage(result.inputId, imageUrl, timestamp); } catch (captureError) { setError( captureError instanceof Error ? captureError.message : "拍照提交失败", @@ -68,7 +127,7 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) { } finally { setCapturing(false); } - }, [capturing, sendUserInput, status]); + }, [appendUserImage, capturing, sendUserInput, status, videoStream]); return { visible, diff --git a/frontend/src/hooks/use-voice-preview.ts b/frontend/src/hooks/use-voice-preview.ts index 41149e2..41fa1da 100644 --- a/frontend/src/hooks/use-voice-preview.ts +++ b/frontend/src/hooks/use-voice-preview.ts @@ -45,6 +45,14 @@ export type ChatMessage = { sequence: number; turnId?: string; streaming?: boolean; + attachments?: ChatAttachment[]; +}; + +export type ChatAttachment = { + id: string; + type: "image"; + url: string; + alt: string; }; type AppMessage = Record & { type?: string }; @@ -871,6 +879,34 @@ export function useVoicePreview( [], ); + const appendUserImage = useCallback( + (inputId: string, imageUrl: string, timestamp: string) => { + messageSeqRef.current += 1; + const sequence = messageSeqRef.current; + setMessages((previous) => + sortMessages([ + ...previous, + { + id: `user-image-${inputId}`, + role: "user", + content: "", + timestamp, + sequence, + attachments: [ + { + id: `image-${inputId}`, + type: "image", + url: imageUrl, + alt: "用户拍摄的照片", + }, + ], + }, + ]), + ); + }, + [], + ); + const updateSession = useCallback( ({ dynamicVariables, @@ -962,6 +998,7 @@ export function useVoicePreview( supportsOutputSelection, sendText, sendUserInput, + appendUserImage, updateSession, registerClientTool, connect, diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 64f1cb7..3e912d2 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -325,7 +325,24 @@ export type ConversationMessage = { contentType: string; content: string; occurredAt: string; - extra: { interrupted?: boolean; turn_id?: string }; + extra: { + interrupted?: boolean; + turn_id?: string; + source?: string; + node_id?: string; + input_id?: string; + }; + artifacts: ConversationArtifact[]; +}; + +export type ConversationArtifact = { + id: string; + kind: string; + contentUrl: string; + mimeType: string; + sizeBytes: number | null; + durationMs: number | null; + extra: Record; }; export type ConversationDetail = Conversation & {