Enhance MobileCallPage with video functionality and improved view management

- Update CallView type from "camera" to "video" for clarity.
- Introduce MobileVideoSurface component to handle video stream rendering.
- Implement MobileCallVisualWorkspace for managing chat and video views with drag-and-drop functionality for the video panel.
- Refactor state management to streamline view transitions between chat and video modes.
- Enhance network metrics handling in useVoicePreview for better performance monitoring during calls.
This commit is contained in:
Xin Wang
2026-07-13 08:01:39 +08:00
parent a25b149f65
commit 313387778a
2 changed files with 381 additions and 96 deletions

View File

@@ -35,7 +35,7 @@ import {
type ChatMessage,
} from "@/hooks/use-voice-preview";
type CallView = "camera" | "chat";
type CallView = "video" | "chat";
function messageTime(timestamp: string): string {
const date = new Date(timestamp);
@@ -106,6 +106,171 @@ function MobileCallTranscript({ messages }: { messages: ChatMessage[] }) {
);
}
function MobileVideoSurface({
stream,
compact = false,
}: {
stream: MediaStream | null;
compact?: boolean;
}) {
const videoRef = useRef<HTMLVideoElement>(null);
useEffect(() => {
if (videoRef.current) videoRef.current.srcObject = stream;
}, [stream]);
return (
<video
ref={videoRef}
autoPlay
playsInline
muted
className={[
"h-full w-full -scale-x-100",
compact ? "object-cover" : "object-cover lg:object-contain",
stream ? "" : "hidden",
].join(" ")}
/>
);
}
function MobileCallVisualWorkspace({
view,
onViewChange,
messages,
videoStream,
cameraStarting,
cameraError,
}: {
view: CallView;
onViewChange: (view: CallView) => void;
messages: ChatMessage[];
videoStream: MediaStream | null;
cameraStarting: boolean;
cameraError: string | null;
}) {
const containerRef = useRef<HTMLDivElement>(null);
const [position, setPosition] = useState({ x: 16, y: 72 });
const positionedRef = useRef(false);
const dragRef = useRef<{
pointerId: number;
startX: number;
startY: number;
originX: number;
originY: number;
moved: boolean;
} | null>(null);
const latestMessage = [...messages]
.reverse()
.find((message) => message.content.trim());
useEffect(() => {
const container = containerRef.current;
if (!container) return;
const rect = container.getBoundingClientRect();
if (!positionedRef.current) {
positionedRef.current = true;
setPosition({ x: Math.max(16, rect.width - 168), y: 72 });
return;
}
setPosition((current) => ({
x: Math.min(Math.max(16, current.x), Math.max(16, rect.width - 168)),
y: Math.min(Math.max(72, current.y), Math.max(72, rect.height - 200)),
}));
}, [view]);
const moveFloatingVideo = useCallback((event: React.PointerEvent<HTMLButtonElement>) => {
const drag = dragRef.current;
const container = containerRef.current;
if (!drag || !container || drag.pointerId !== event.pointerId) return;
const dx = event.clientX - drag.startX;
const dy = event.clientY - drag.startY;
if (Math.abs(dx) + Math.abs(dy) > 4) drag.moved = true;
const rect = container.getBoundingClientRect();
setPosition({
x: Math.min(Math.max(16, drag.originX + dx), Math.max(16, rect.width - 168)),
y: Math.min(Math.max(72, drag.originY + dy), Math.max(72, rect.height - 200)),
});
}, []);
return (
<div ref={containerRef} className="absolute inset-0 z-[1] overflow-hidden">
{view === "chat" ? (
<>
<MobileCallTranscript messages={messages} />
<button
type="button"
style={{ left: position.x, top: position.y }}
className="absolute z-10 h-24 w-38 touch-none overflow-hidden rounded-2xl border border-white/20 bg-black shadow-2xl ring-1 ring-black/20 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-white/50"
aria-label="拖动摄像头浮窗,点击切换到视频流"
onPointerDown={(event) => {
event.currentTarget.setPointerCapture(event.pointerId);
dragRef.current = {
pointerId: event.pointerId,
startX: event.clientX,
startY: event.clientY,
originX: position.x,
originY: position.y,
moved: false,
};
}}
onPointerMove={moveFloatingVideo}
onPointerUp={(event) => {
const drag = dragRef.current;
event.currentTarget.releasePointerCapture(event.pointerId);
dragRef.current = null;
if (drag && !drag.moved) onViewChange("video");
}}
onPointerCancel={() => {
dragRef.current = null;
}}
>
<MobileVideoSurface stream={videoStream} compact />
{!videoStream && (
<span className="absolute inset-0 flex items-center justify-center text-white/60">
{cameraStarting ? <Loader2 className="size-5 animate-spin" /> : <Video className="size-5" />}
</span>
)}
</button>
</>
) : (
<>
<MobileVideoSurface stream={videoStream} />
{!videoStream && (
<div className="absolute inset-0 flex items-center justify-center px-8 text-center">
<div className="flex max-w-sm flex-col items-center gap-3">
{cameraStarting ? (
<Loader2 size={34} className="animate-spin text-white/70" />
) : (
<Video size={34} className="text-white/60" />
)}
<p className="text-sm leading-6 text-white/75">
{cameraStarting ? "正在开启摄像头…" : cameraError || "等待摄像头画面…"}
</p>
</div>
</div>
)}
<button
type="button"
onClick={() => onViewChange("chat")}
className="absolute left-4 right-4 top-[max(4.5rem,calc(env(safe-area-inset-top)+4.5rem))] z-10 flex h-16 items-center gap-3 overflow-hidden rounded-2xl border border-white/15 bg-[#07101a]/55 px-4 py-2 text-left shadow-xl backdrop-blur-md focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-white/50"
aria-label="切换到聊天记录"
>
<MessageSquareText className="size-[17px] shrink-0 text-white/65" />
<span className="line-clamp-2 min-w-0 flex-1 text-xs leading-5 text-white/90">
<span className="mr-1.5 font-medium text-white/55">
{latestMessage?.role === "user" ? "我:" : "助手:"}
</span>
{latestMessage?.content || "暂无消息,点击返回聊天记录"}
</span>
</button>
</>
)}
<div className="pointer-events-none absolute inset-0 z-[2] bg-gradient-to-b from-black/35 via-transparent to-black/65" />
</div>
);
}
function CallAudioDeviceSelect({
micValue,
micDevices,
@@ -263,15 +428,10 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
deviceId: cameraDeviceId,
selectCamera: changeCamera,
} = camera;
const videoRef = useRef<HTMLVideoElement>(null);
const [view, setView] = useState<CallView>("camera");
const [view, setView] = useState<CallView>("chat");
const connecting = status === "connecting";
const inCall = !callEnded && (connecting || status === "connected");
useEffect(() => {
if (videoRef.current) videoRef.current.srcObject = videoStream;
}, [videoStream]);
const startCall = useCallback(async () => {
await connect({
visionEnabled: true,
@@ -279,7 +439,7 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
}, [connect]);
const endCall = useCallback(() => {
setView("camera");
setView("chat");
disconnect();
}, [disconnect]);
@@ -305,20 +465,16 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
data-testid="mobile-call-viewport"
className="relative isolate h-dvh min-h-80 w-full overflow-hidden bg-[#07101a] text-white lg:h-[calc(100dvh-2rem)] lg:min-h-0 lg:w-auto lg:aspect-[9/16] lg:rounded-[2rem] lg:ring-1 lg:ring-white/15 lg:shadow-2xl"
>
<video
ref={videoRef}
autoPlay
playsInline
muted
className={[
"absolute inset-0 h-full w-full -scale-x-100 object-cover transition-opacity duration-300 lg:object-contain",
videoStream && view === "camera" ? "opacity-100" : "opacity-0",
].join(" ")}
/>
<div className="pointer-events-none absolute inset-0 bg-gradient-to-b from-black/35 via-transparent to-black/65" />
{view === "chat" && inCall && <MobileCallTranscript messages={messages} />}
{inCall && (
<MobileCallVisualWorkspace
view={view}
onViewChange={setView}
messages={messages}
videoStream={videoStream}
cameraStarting={cameraStarting}
cameraError={cameraError}
/>
)}
<NetworkQualityIndicator
quality={networkQuality}
@@ -347,48 +503,13 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
: "通话已结束"}
</div>
{inCall && (
<button
type="button"
onClick={() =>
setView((current) => current === "camera" ? "chat" : "camera")
}
aria-label={view === "camera" ? "显示聊天记录" : "显示摄像头画面"}
title={view === "camera" ? "聊天记录" : "摄像头画面"}
className="absolute right-4 top-[max(1rem,env(safe-area-inset-top))] z-10 flex size-9 items-center justify-center rounded-full border border-white/15 bg-black/40 text-white shadow-lg backdrop-blur-md transition-colors hover:bg-black/60 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-white/40"
>
{view === "camera" ? (
<MessageSquareText className="size-[18px]" />
) : (
<Video className="size-[18px]" />
)}
</button>
)}
{!videoStream && inCall && view === "camera" && (
<div className="absolute inset-0 z-[2] flex items-center justify-center px-8 text-center">
<div className="flex max-w-sm flex-col items-center gap-3">
{cameraStarting ? (
<Loader2 size={34} className="animate-spin text-white/70" />
) : (
<Video size={34} className="text-white/60" />
)}
<p className="text-sm leading-6 text-white/75">
{cameraStarting
? "正在开启摄像头…"
: cameraError || "等待摄像头画面…"}
</p>
</div>
</div>
)}
{!inCall && (
<div className="absolute inset-0 z-[2] flex items-center justify-center px-8">
<div className="flex max-w-sm flex-col items-center gap-4 text-center">
<Button
type="button"
onClick={() => {
setView("camera");
setView("chat");
void startCall();
}}
className="h-12 gap-2 rounded-full bg-white px-6 text-[#07101a] shadow-2xl hover:bg-white/90"