diff --git a/src/components/playground/PhoneSimulator.tsx b/src/components/playground/PhoneSimulator.tsx index 3fa6856..b3e3fa6 100644 --- a/src/components/playground/PhoneSimulator.tsx +++ b/src/components/playground/PhoneSimulator.tsx @@ -10,24 +10,50 @@ import { useVoiceAssistant, VideoTrack, } from "@livekit/components-react"; -import { ConnectionState, Track, LocalParticipant } from "livekit-client"; -import { useEffect, useMemo, useState } from "react"; -import { BatteryIcon, MicIcon, MicOffIcon, PhoneIcon, PhoneOffIcon, WifiIcon } from "./icons"; +import { ConnectionState, Track, LocalParticipant, Room } from "livekit-client"; +import { useEffect, useMemo, useState, useRef } from "react"; +import { BatteryIcon, ImageIcon, MicIcon, MicOffIcon, PhoneIcon, PhoneOffIcon, WifiIcon, SwitchCameraIcon } from "./icons"; export interface PhoneSimulatorProps { onConnect: () => void; + phoneMode?: "normal" | "capture"; + onCapture?: (image: File) => void; } -export function PhoneSimulator({ onConnect }: PhoneSimulatorProps) { +export function PhoneSimulator({ onConnect, phoneMode = "normal", onCapture }: PhoneSimulatorProps) { const { config } = useConfig(); const room = useRoomContext(); const roomState = useConnectionState(); const { localParticipant } = useLocalParticipant(); const tracks = useTracks(); const voiceAssistant = useVoiceAssistant(); + const fileInputRef = useRef(null); + const [showCameraMenu, setShowCameraMenu] = useState(false); + const [cameras, setCameras] = useState([]); const [currentTime, setCurrentTime] = useState(""); + useEffect(() => { + if (showCameraMenu) { + Room.getLocalDevices("videoinput").then(setCameras); + } + }, [showCameraMenu]); + + // Close menu when clicking outside + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if (showCameraMenu) { + setShowCameraMenu(false); + } + }; + if (showCameraMenu) { + document.addEventListener("click", handleClickOutside); + } + return () => { + document.removeEventListener("click", handleClickOutside); + }; + }, [showCameraMenu]); + useEffect(() => { const updateTime = () => { const now = new Date(); @@ -65,6 +91,53 @@ export function PhoneSimulator({ onConnect }: PhoneSimulatorProps) { room.disconnect(); }; + const handleCapture = async () => { + if (!localCameraTrack || !onCapture) return; + + const videoElement = localCameraTrack.attach() as HTMLVideoElement; + const canvas = document.createElement("canvas"); + canvas.width = videoElement.videoWidth; + canvas.height = videoElement.videoHeight; + const ctx = canvas.getContext("2d"); + + if (ctx) { + ctx.drawImage(videoElement, 0, 0); + canvas.toBlob((blob) => { + if (blob && onCapture) { + const file = new File([blob], "camera-capture.jpg", { type: "image/jpeg" }); + onCapture(file); + } + }, "image/jpeg"); + } + }; + + const handleUpload = () => { + if (fileInputRef.current) { + fileInputRef.current.click(); + } + }; + + const handleSwitchCamera = async (e: React.MouseEvent) => { + e.stopPropagation(); // Prevent immediate close + setShowCameraMenu(!showCameraMenu); + }; + + const handleSelectCamera = async (deviceId: string) => { + await room.switchActiveDevice("videoinput", deviceId); + setShowCameraMenu(false); + }; + + const handleFileChange = (event: React.ChangeEvent) => { + const file = event.target.files?.[0]; + if (file && onCapture) { + onCapture(file); + } + // Reset input so the same file can be selected again + if (event.target) { + event.target.value = ""; + } + }; + const videoContent = useMemo(() => { if (roomState === ConnectionState.Disconnected) { return ( @@ -126,6 +199,29 @@ export function PhoneSimulator({ onConnect }: PhoneSimulatorProps) {
{videoContent} + + + {/* Capture Guide Lines */} + {roomState === ConnectionState.Connected && phoneMode === "capture" && ( +
+ {/* Thirds Grid */} +
+
+
+
+ + {/* Center Focus Indicator */} +
+
+
+ )} + {/* Agent Audio Visualizer (Top Left) */} {roomState === ConnectionState.Connected && voiceAssistant.audioTrack && (
@@ -144,27 +240,75 @@ export function PhoneSimulator({ onConnect }: PhoneSimulatorProps) { {/* Call Controls Overlay */} {roomState === ConnectionState.Connected && (
- +
+
+ +
+
+ + {showCameraMenu && ( +
+ {cameras.length === 0 ? ( +
No cameras found
+ ) : ( + cameras.map((device) => ( + + )) + )} +
+ )} +
+
) : ( - + <> + + + + )} - - -
)} diff --git a/src/components/playground/Playground.tsx b/src/components/playground/Playground.tsx index beefafd..b05d4e3 100644 --- a/src/components/playground/Playground.tsx +++ b/src/components/playground/Playground.tsx @@ -63,6 +63,7 @@ export default function Playground({ const roomState = useConnectionState(); const tracks = useTracks(); const room = useRoomContext(); + const [phoneMode, setPhoneMode] = useState<"normal" | "capture">("normal"); const [rpcMethod, setRpcMethod] = useState(""); const [rpcPayload, setRpcPayload] = useState(""); @@ -100,8 +101,30 @@ export default function Playground({ } } ); + + localParticipant.registerRpcMethod( + 'enterImageCaptureMode', + async () => { + setPhoneMode("capture"); + return JSON.stringify({ success: true }); + } + ); + + localParticipant.registerRpcMethod( + 'exitImageCaptureMode', + async () => { + setPhoneMode("normal"); + return JSON.stringify({ success: true }); + } + ); }, [localParticipant, roomState]); + useEffect(() => { + if (roomState === ConnectionState.Connected) { + setPhoneMode("normal"); + } + }, [roomState]); + useEffect(() => { if (!localParticipant || roomState !== ConnectionState.Connected) { return; @@ -570,7 +593,15 @@ export default function Playground({ className="w-full h-full grow" childrenClassName="justify-center" > - onConnect(true)} /> + onConnect(true)} + phoneMode={phoneMode} + onCapture={(content: File) => { + if (localParticipant) { + localParticipant.sendFile(content, { topic: "image" }); + } + }} + /> ), }); @@ -633,7 +664,15 @@ export default function Playground({ className="w-full h-full grow" childrenClassName="justify-center" > - onConnect(true)} /> + onConnect(true)} + phoneMode={phoneMode} + onCapture={(content: File) => { + if (localParticipant) { + localParticipant.sendFile(content, { topic: "image" }); + } + }} + /> diff --git a/src/components/playground/icons.tsx b/src/components/playground/icons.tsx index 5e9120b..a17c15f 100644 --- a/src/components/playground/icons.tsx +++ b/src/components/playground/icons.tsx @@ -151,3 +151,40 @@ export const PhoneIcon = ({ className }: { className?: string }) => ( ); + +export const ImageIcon = ({ className }: { className?: string }) => ( + + + + + +); + +export const SwitchCameraIcon = ({ className }: { className?: string }) => ( + + + + +);