diff --git a/src/components/playground/PhoneSimulator.tsx b/src/components/playground/PhoneSimulator.tsx index b3e3fa6..6c6639c 100644 --- a/src/components/playground/PhoneSimulator.tsx +++ b/src/components/playground/PhoneSimulator.tsx @@ -28,17 +28,76 @@ export function PhoneSimulator({ onConnect, phoneMode = "normal", onCapture }: P const tracks = useTracks(); const voiceAssistant = useVoiceAssistant(); const fileInputRef = useRef(null); + const phoneContainerRef = useRef(null); + const visualizerRef = useRef(null); const [showCameraMenu, setShowCameraMenu] = useState(false); const [cameras, setCameras] = useState([]); + const [processingImage, setProcessingImage] = useState(null); + const [isCapturing, setIsCapturing] = useState(false); + const [processingSource, setProcessingSource] = useState< + "camera" | "upload" | null + >(null); const [currentTime, setCurrentTime] = useState(""); + const [visualizerPosition, setVisualizerPosition] = useState({ + x: 16, + y: 56, + }); + const [isDragging, setIsDragging] = useState(false); + const dragOffset = useRef({ x: 0, y: 0 }); + + const handleDragStart = (e: React.MouseEvent) => { + setIsDragging(true); + dragOffset.current = { + x: e.clientX - visualizerPosition.x, + y: e.clientY - visualizerPosition.y, + }; + }; + + const handleDragMove = (e: MouseEvent) => { + if (!isDragging || !phoneContainerRef.current || !visualizerRef.current) return; + + const containerRect = phoneContainerRef.current.getBoundingClientRect(); + const visualizerRect = visualizerRef.current.getBoundingClientRect(); + + let newX = e.clientX - dragOffset.current.x; + let newY = e.clientY - dragOffset.current.y; + + // Constrain within container + const maxX = containerRect.width - visualizerRect.width; + const maxY = containerRect.height - visualizerRect.height; + const statusBarHeight = 48; // h-12 = 48px + + newX = Math.max(0, Math.min(newX, maxX)); + newY = Math.max(statusBarHeight, Math.min(newY, maxY)); + + setVisualizerPosition({ + x: newX, + y: newY, + }); + }; + + const handleDragEnd = () => { + setIsDragging(false); + }; + + useEffect(() => { + if (isDragging) { + window.addEventListener("mouseup", handleDragEnd); + window.addEventListener("mousemove", handleDragMove); + } + return () => { + window.removeEventListener("mouseup", handleDragEnd); + window.removeEventListener("mousemove", handleDragMove); + }; + }, [isDragging]); + useEffect(() => { if (showCameraMenu) { - Room.getLocalDevices("videoinput").then(setCameras); + Room.getLocalDevices("videoinput").then(setCameras); } }, [showCameraMenu]); - // Close menu when clicking outside useEffect(() => { const handleClickOutside = (event: MouseEvent) => { @@ -54,6 +113,13 @@ export function PhoneSimulator({ onConnect, phoneMode = "normal", onCapture }: P }; }, [showCameraMenu]); + useEffect(() => { + if (voiceAssistant.state === "speaking") { + setProcessingImage(null); + setProcessingSource(null); + } + }, [voiceAssistant.state]); + useEffect(() => { const updateTime = () => { const now = new Date(); @@ -92,22 +158,128 @@ export function PhoneSimulator({ onConnect, phoneMode = "normal", onCapture }: P }; const handleCapture = async () => { - if (!localCameraTrack || !onCapture) return; + if (!localCameraTrack || !onCapture || isCapturing) return; + setIsCapturing(true); + + const trackReference = localCameraTrack as any; + // Handle both TrackReference (from useTracks) and potential direct Track objects + const track = + trackReference.publication?.track?.mediaStreamTrack || + trackReference.mediaStreamTrack; + + if (!track) { + console.error("No media stream track found"); + setIsCapturing(false); + return; + } + + const video = document.createElement("video"); + video.srcObject = new MediaStream([track]); + video.muted = true; + video.playsInline = true; + video.autoplay = true; + // Element needs to be in the DOM for some browsers to play it properly + video.style.position = "absolute"; + video.style.top = "-9999px"; + video.style.left = "-9999px"; + document.body.appendChild(video); + + try { + await video.play(); + + // Wait for video dimensions to be available + if (video.videoWidth === 0 || video.videoHeight === 0) { + await new Promise((resolve) => { + video.onloadedmetadata = () => resolve(); + // Timeout to prevent hanging + setTimeout(resolve, 1000); + }); + } + + const canvas = document.createElement("canvas"); + // Default to video dimensions + let renderWidth = video.videoWidth; + let renderHeight = video.videoHeight; + let sourceX = 0; + let sourceY = 0; + let sourceWidth = video.videoWidth; + let sourceHeight = video.videoHeight; + + // If the video is landscape but we want a portrait crop (like a phone) + // We want an aspect ratio of roughly 9:19.5 (from the container styles) + const targetAspect = 9 / 19.5; + const videoAspect = video.videoWidth / video.videoHeight; + + if (videoAspect > targetAspect) { + // Video is wider than target - crop width + const newWidth = video.videoHeight * targetAspect; + sourceX = (video.videoWidth - newWidth) / 2; + sourceWidth = newWidth; + renderWidth = newWidth; + } else { + // Video is taller than target - crop height (less common for landscape webcam) + const newHeight = video.videoWidth / targetAspect; + sourceY = (video.videoHeight - newHeight) / 2; + sourceHeight = newHeight; + renderHeight = newHeight; + } + + canvas.width = renderWidth; + canvas.height = renderHeight; + const ctx = canvas.getContext("2d"); + + if (ctx) { + // Mirror the image to match the preview + ctx.translate(canvas.width, 0); + ctx.scale(-1, 1); + + // Draw only the cropped portion of the video + ctx.drawImage( + video, + sourceX, + sourceY, + sourceWidth, + sourceHeight, + 0, + 0, + renderWidth, + renderHeight + ); + // Reset transform + ctx.setTransform(1, 0, 0, 1, 0, 0); + + // Use toDataURL for immediate preview feedback + const dataUrl = canvas.toDataURL("image/jpeg"); + setProcessingImage(dataUrl); + setProcessingSource("camera"); + + // Create a new canvas for the final output (unmirrored if needed, but user requested mirrored) + // The user requested to mirror the shuttled photo, which we did above for the canvas. + // So the blob created from this canvas will also be mirrored. - const videoElement = localCameraTrack.attach() as HTMLVideoElement; - const canvas = document.createElement("canvas"); - canvas.width = videoElement.videoWidth; - canvas.height = videoElement.videoHeight; - const ctx = canvas.getContext("2d"); - - if (ctx) { - ctx.drawImage(videoElement, 0, 0); canvas.toBlob((blob) => { - if (blob && onCapture) { - const file = new File([blob], "camera-capture.jpg", { type: "image/jpeg" }); - onCapture(file); - } + if (blob && onCapture) { + const file = new File([blob], "camera-capture.jpg", { + type: "image/jpeg", + }); + onCapture(file); + } + setIsCapturing(false); }, "image/jpeg"); + } else { + setIsCapturing(false); + } + } catch (e) { + console.error("Failed to capture image", e); + setIsCapturing(false); + } finally { + // Cleanup + video.pause(); + video.srcObject = null; + if (document.body.contains(video)) { + document.body.removeChild(video); + } + video.remove(); } }; @@ -131,6 +303,8 @@ export function PhoneSimulator({ onConnect, phoneMode = "normal", onCapture }: P const file = event.target.files?.[0]; if (file && onCapture) { onCapture(file); + setProcessingImage(URL.createObjectURL(file)); + setProcessingSource("upload"); } // Reset input so the same file can be selected again if (event.target) { @@ -182,6 +356,14 @@ export function PhoneSimulator({ onConnect, phoneMode = "normal", onCapture }: P .mirror-video video { transform: scaleX(-1); } + @keyframes scan { + 0% { top: 0%; } + 50% { top: 100%; } + 100% { top: 0%; } + } + .scan-animation { + animation: scan 3s linear infinite; + } `} {/* Status Bar */}
@@ -196,8 +378,28 @@ export function PhoneSimulator({ onConnect, phoneMode = "normal", onCapture }: P
{/* Main Content */} -
+
{videoContent} + {processingImage && ( +
+ {/* eslint-disable-next-line @next/next/no-img-element */} + Processing +
+
+
+
+ 正在分析... +
+
+ )} {/* Capture Guide Lines */} - {roomState === ConnectionState.Connected && phoneMode === "capture" && ( + {roomState === ConnectionState.Connected && phoneMode === "capture" && !processingImage && (
{/* Thirds Grid */}
@@ -222,93 +424,104 @@ export function PhoneSimulator({ onConnect, phoneMode = "normal", onCapture }: P
)} - {/* Agent Audio Visualizer (Top Left) */} + {/* Agent Audio Visualizer (Draggable) */} {roomState === ConnectionState.Connected && voiceAssistant.audioTrack && ( -
-
- -
+
+
+
+
)}
{/* Call Controls Overlay */} {roomState === ConnectionState.Connected && ( -
- {phoneMode === "capture" ? ( -
-
- -
-
- -
-
- - {showCameraMenu && ( -
- {cameras.length === 0 ? ( -
No cameras found
- ) : ( - cameras.map((device) => ( - - )) - )} -
- )} -
-
- ) : ( - <> - +
+
+ +
+
+ + {showCameraMenu && ( +
+ {cameras.length === 0 ? ( +
+ No cameras found +
) : ( - + cameras.map((device) => ( + + )) )} - - - - - )} +
+ )} +
+
+ ) : ( +
+ + + +
+ )}
)}
diff --git a/src/components/playground/icons.tsx b/src/components/playground/icons.tsx index a17c15f..c7720ae 100644 --- a/src/components/playground/icons.tsx +++ b/src/components/playground/icons.tsx @@ -184,7 +184,7 @@ export const SwitchCameraIcon = ({ className }: { className?: string }) => ( strokeLinejoin="round" className={className} > - - + + );