add image upload button

2025-12-05 14:46:00 +08:00 · 2025-12-05 14:46:00 +08:00 · 3180923f75
commit 3180923f75
parent 9b9ffec064
3 changed files with 245 additions and 25 deletions
--- a/src/components/playground/PhoneSimulator.tsx
+++ b/src/components/playground/PhoneSimulator.tsx
@ -10,24 +10,50 @@ import {
  useVoiceAssistant,
  VideoTrack,
 } from "@livekit/components-react";
-import { ConnectionState, Track, LocalParticipant } from "livekit-client";
-import { useEffect, useMemo, useState } from "react";
-import { BatteryIcon, MicIcon, MicOffIcon, PhoneIcon, PhoneOffIcon, WifiIcon } from "./icons";
+import { ConnectionState, Track, LocalParticipant, Room } from "livekit-client";
+import { useEffect, useMemo, useState, useRef } from "react";
+import { BatteryIcon, ImageIcon, MicIcon, MicOffIcon, PhoneIcon, PhoneOffIcon, WifiIcon, SwitchCameraIcon } from "./icons";

 export interface PhoneSimulatorProps {
    onConnect: () => void;
+    phoneMode?: "normal" | "capture";
+    onCapture?: (image: File) => void;
 }

-export function PhoneSimulator({ onConnect }: PhoneSimulatorProps) {
+export function PhoneSimulator({ onConnect, phoneMode = "normal", onCapture }: PhoneSimulatorProps) {
  const { config } = useConfig();
  const room = useRoomContext();
  const roomState = useConnectionState();
  const { localParticipant } = useLocalParticipant();
  const tracks = useTracks();
  const voiceAssistant = useVoiceAssistant();
+  const fileInputRef = useRef<HTMLInputElement>(null);
+  const [showCameraMenu, setShowCameraMenu] = useState(false);
+  const [cameras, setCameras] = useState<MediaDeviceInfo[]>([]);

  const [currentTime, setCurrentTime] = useState("");

+  useEffect(() => {
+    if (showCameraMenu) {
+        Room.getLocalDevices("videoinput").then(setCameras);
+    }
+  }, [showCameraMenu]);
+
+  // Close menu when clicking outside
+  useEffect(() => {
+    const handleClickOutside = (event: MouseEvent) => {
+      if (showCameraMenu) {
+        setShowCameraMenu(false);
+      }
+    };
+    if (showCameraMenu) {
+        document.addEventListener("click", handleClickOutside);
+    }
+    return () => {
+        document.removeEventListener("click", handleClickOutside);
+    };
+  }, [showCameraMenu]);
+
  useEffect(() => {
    const updateTime = () => {
      const now = new Date();
@ -65,6 +91,53 @@ export function PhoneSimulator({ onConnect }: PhoneSimulatorProps) {
    room.disconnect();
  };

+  const handleCapture = async () => {
+    if (!localCameraTrack || !onCapture) return;
+
+    const videoElement = localCameraTrack.attach() as HTMLVideoElement;
+    const canvas = document.createElement("canvas");
+    canvas.width = videoElement.videoWidth;
+    canvas.height = videoElement.videoHeight;
+    const ctx = canvas.getContext("2d");
+    
+    if (ctx) {
+        ctx.drawImage(videoElement, 0, 0);
+        canvas.toBlob((blob) => {
+            if (blob && onCapture) {
+                const file = new File([blob], "camera-capture.jpg", { type: "image/jpeg" });
+                onCapture(file);
+            }
+        }, "image/jpeg");
+    }
+  };
+
+  const handleUpload = () => {
+    if (fileInputRef.current) {
+      fileInputRef.current.click();
+    }
+  };
+
+  const handleSwitchCamera = async (e: React.MouseEvent) => {
+    e.stopPropagation(); // Prevent immediate close
+    setShowCameraMenu(!showCameraMenu);
+  };
+
+  const handleSelectCamera = async (deviceId: string) => {
+    await room.switchActiveDevice("videoinput", deviceId);
+    setShowCameraMenu(false);
+  };
+
+  const handleFileChange = (event: React.ChangeEvent<HTMLInputElement>) => {
+    const file = event.target.files?.[0];
+    if (file && onCapture) {
+      onCapture(file);
+    }
+    // Reset input so the same file can be selected again
+    if (event.target) {
+      event.target.value = "";
+    }
+  };
+
  const videoContent = useMemo(() => {
    if (roomState === ConnectionState.Disconnected) {
      return (
@ -126,6 +199,29 @@ export function PhoneSimulator({ onConnect }: PhoneSimulatorProps) {
      <div className="flex-grow relative bg-gray-950 w-full h-full">
        {videoContent}
        
+        <input 
+            type="file" 
+            ref={fileInputRef} 
+            onChange={handleFileChange} 
+            accept="image/*" 
+            className="hidden" 
+        />
+
+        {/* Capture Guide Lines */}
+        {roomState === ConnectionState.Connected && phoneMode === "capture" && (
+            <div className="absolute inset-0 pointer-events-none z-10">
+                {/* Thirds Grid */}
+                <div className="absolute top-1/3 left-0 w-full h-[1px] bg-white/20"></div>
+                <div className="absolute top-2/3 left-0 w-full h-[1px] bg-white/20"></div>
+                <div className="absolute left-1/3 top-0 w-[1px] h-full bg-white/20"></div>
+                <div className="absolute left-2/3 top-0 w-[1px] h-full bg-white/20"></div>
+                
+                {/* Center Focus Indicator */}
+                <div className="absolute top-1/2 left-1/2 -translate-x-1/2 -translate-y-1/2 w-16 h-16 border border-white/50 rounded-sm"></div>
+                <div className="absolute top-1/2 left-1/2 -translate-x-1/2 -translate-y-1/2 w-1 h-1 bg-white/50 rounded-full"></div>
+            </div>
+        )}
+        
        {/* Agent Audio Visualizer (Top Left) */}
        {roomState === ConnectionState.Connected && voiceAssistant.audioTrack && (
            <div className="absolute top-14 left-4 z-20 p-2 bg-black/40 backdrop-blur-md rounded-lg border border-white/10 shadow-lg">
@ -144,27 +240,75 @@ export function PhoneSimulator({ onConnect }: PhoneSimulatorProps) {
      {/* Call Controls Overlay */}
      {roomState === ConnectionState.Connected && (
        <div className="absolute bottom-8 left-0 w-full px-8 flex items-center justify-between z-20">
-            <button
-            className={`p-4 rounded-full backdrop-blur-md transition-colors ${
-                !isMicEnabled 
-                ? "bg-white text-black" 
-                : "bg-gray-600/50 text-white hover:bg-gray-600/70"
-            }`}
-            onClick={handleMicToggle}
-            >
-            {isMicEnabled ? (
-                <MicIcon className="w-6 h-6" />
+            {phoneMode === "capture" ? (
+                <div className="w-full grid grid-cols-3 items-center px-8">
+                    <div className="flex justify-start">
+                        <button
+                            className="p-4 rounded-full bg-gray-800/50 text-white hover:bg-gray-800/70 transition-colors shrink-0"
+                            onClick={handleUpload}
+                        >
+                            <ImageIcon className="w-6 h-6" />
+                        </button>
+                    </div>
+                    <div className="flex justify-center">
+                        <button
+                            className="w-20 h-20 rounded-full border-4 border-white p-1 hover:scale-105 transition-transform shrink-0 aspect-square"
+                            onClick={handleCapture}
+                        >
+                            <div className="w-full h-full bg-white rounded-full"></div>
+                        </button>
+                    </div>
+                    <div className="flex justify-end relative">
+                        <button
+                            className="p-4 rounded-full bg-gray-800/50 text-white hover:bg-gray-800/70 transition-colors shrink-0"
+                            onClick={handleSwitchCamera}
+                        >
+                            <SwitchCameraIcon className="w-6 h-6" />
+                        </button>
+                        {showCameraMenu && (
+                            <div className="absolute bottom-16 right-0 bg-gray-900 border border-gray-800 rounded-lg shadow-xl py-2 w-48 z-50">
+                                {cameras.length === 0 ? (
+                                    <div className="px-4 py-2 text-gray-500 text-sm">No cameras found</div>
+                                ) : (
+                                    cameras.map((device) => (
+                                        <button
+                                            key={device.deviceId}
+                                            onClick={() => handleSelectCamera(device.deviceId)}
+                                            className="w-full text-left px-4 py-2 text-sm text-white hover:bg-gray-800 transition-colors truncate"
+                                        >
+                                            {device.label || `Camera ${cameras.indexOf(device) + 1}`}
+                                        </button>
+                                    ))
+                                )}
+                            </div>
+                        )}
+                    </div>
+                </div>
            ) : (
-                <MicOffIcon className="w-6 h-6" />
+                <>
+                    <button
+                    className={`p-4 rounded-full backdrop-blur-md transition-colors ${
+                        !isMicEnabled 
+                        ? "bg-white text-black" 
+                        : "bg-gray-600/50 text-white hover:bg-gray-600/70"
+                    }`}
+                    onClick={handleMicToggle}
+                    >
+                    {isMicEnabled ? (
+                        <MicIcon className="w-6 h-6" />
+                    ) : (
+                        <MicOffIcon className="w-6 h-6" />
+                    )}
+                    </button>
+                    
+                    <button
+                    className="p-4 rounded-full bg-red-500 text-white hover:bg-red-600 transition-colors"
+                    onClick={handleDisconnect}
+                    >
+                    <PhoneOffIcon className="w-6 h-6" />
+                    </button>
+                </>
            )}
-            </button>
-            
-            <button
-            className="p-4 rounded-full bg-red-500 text-white hover:bg-red-600 transition-colors"
-            onClick={handleDisconnect}
-            >
-            <PhoneOffIcon className="w-6 h-6" />
-            </button>
        </div>
      )}
    </div>
--- a/src/components/playground/Playground.tsx
+++ b/src/components/playground/Playground.tsx
@ -63,6 +63,7 @@ export default function Playground({
  const roomState = useConnectionState();
  const tracks = useTracks();
  const room = useRoomContext();
+  const [phoneMode, setPhoneMode] = useState<"normal" | "capture">("normal");

  const [rpcMethod, setRpcMethod] = useState("");
  const [rpcPayload, setRpcPayload] = useState("");
@ -100,8 +101,30 @@ export default function Playground({
        }
      }
    );
+
+    localParticipant.registerRpcMethod(
+      'enterImageCaptureMode',
+      async () => {
+        setPhoneMode("capture");
+        return JSON.stringify({ success: true });
+      }
+    );
+
+    localParticipant.registerRpcMethod(
+      'exitImageCaptureMode',
+      async () => {
+        setPhoneMode("normal");
+        return JSON.stringify({ success: true });
+      }
+    );
  }, [localParticipant, roomState]);

+  useEffect(() => {
+    if (roomState === ConnectionState.Connected) {
+      setPhoneMode("normal");
+    }
+  }, [roomState]);
+
  useEffect(() => {
    if (!localParticipant || roomState !== ConnectionState.Connected) {
      return;
@ -570,7 +593,15 @@ export default function Playground({
          className="w-full h-full grow"
          childrenClassName="justify-center"
        >
-          <PhoneSimulator onConnect={() => onConnect(true)} />
+          <PhoneSimulator 
+            onConnect={() => onConnect(true)} 
+            phoneMode={phoneMode} 
+            onCapture={(content: File) => {
+              if (localParticipant) {
+                localParticipant.sendFile(content, { topic: "image" });
+              }
+            }}
+          />
        </PlaygroundTile>
      ),
    });
@ -633,7 +664,15 @@ export default function Playground({
            className="w-full h-full grow"
            childrenClassName="justify-center"
          >
-            <PhoneSimulator onConnect={() => onConnect(true)} />
+            <PhoneSimulator 
+            onConnect={() => onConnect(true)} 
+            phoneMode={phoneMode} 
+            onCapture={(content: File) => {
+              if (localParticipant) {
+                localParticipant.sendFile(content, { topic: "image" });
+              }
+            }}
+          />
          </PlaygroundTile>
        </div>

--- a/src/components/playground/icons.tsx
+++ b/src/components/playground/icons.tsx
@ -151,3 +151,40 @@ export const PhoneIcon = ({ className }: { className?: string }) => (
    <path d="M22 16.92v3a2 2 0 0 1-2.18 2 19.79 19.79 0 0 1-8.63-3.07 19.5 19.5 0 0 1-6-6 19.79 19.79 0 0 1-3.07-8.67A2 2 0 0 1 4.11 2h3a2 2 0 0 1 2 1.72 12.84 12.84 0 0 0 .7 2.81 2 2 0 0 1-.45 2.11L8.09 9.91a16 16 0 0 0 6 6l1.27-1.27a2 2 0 0 1 2.11-.45 12.84 12.84 0 0 0 2.81.7A2 2 0 0 1 22 16.92z"></path>
  </svg>
 );
+
+export const ImageIcon = ({ className }: { className?: string }) => (
+  <svg
+    xmlns="http://www.w3.org/2000/svg"
+    width="24"
+    height="24"
+    viewBox="0 0 24 24"
+    fill="none"
+    stroke="currentColor"
+    strokeWidth="2"
+    strokeLinecap="round"
+    strokeLinejoin="round"
+    className={className}
+  >
+    <rect x="3" y="3" width="18" height="18" rx="2" ry="2"></rect>
+    <circle cx="8.5" cy="8.5" r="1.5"></circle>
+    <polyline points="21 15 16 10 5 21"></polyline>
+  </svg>
+);
+
+export const SwitchCameraIcon = ({ className }: { className?: string }) => (
+  <svg
+    xmlns="http://www.w3.org/2000/svg"
+    width="24"
+    height="24"
+    viewBox="0 0 24 24"
+    fill="none"
+    stroke="currentColor"
+    strokeWidth="2"
+    strokeLinecap="round"
+    strokeLinejoin="round"
+    className={className}
+  >
+    <path d="M20 4v5h-.58c-.89-3.1-3.85-5.23-7.17-5.23-4.26 0-7.74 3.4-7.74 7.6 0 .65.09 1.29.26 1.91l1.69-1.69c-.14-.22-.2-.46-.2-.72 0-3.2 2.63-5.81 5.87-5.81 2.67 0 4.96 1.75 5.6 4.23h-1.73l2.5 2.5 2.5-2.5h-1z" />
+    <path d="M4 20v-5h.58c.89 3.1 3.85 5.23 7.17 5.23 4.26 0 7.74-3.4 7.74-7.6 0-.65-.09-1.29-.26-1.91l-1.69 1.69c.14.22.2.46.2.72 0 3.2-2.63 5.81-5.87 5.81-2.67 0-4.96-1.75-5.6-4.23h1.73l-2.5-2.5-2.5 2.5h1z" />
+  </svg>
+);