From 944deec4826f75617008174c3c6a781a6c56bf7e Mon Sep 17 00:00:00 2001
From: Xin Wang <wangxin19930411@163.com>
Date: Wed, 10 Dec 2025 16:48:27 +0800
Subject: [PATCH] add instruction window

---
 src/components/playground/PhoneSimulator.tsx | 12 +++-
 src/components/playground/Playground.tsx     | 69 ++++++++++++++++++--
 src/hooks/useConfig.tsx                      |  3 +
 src/hooks/useConnection.tsx                  | 21 +++---
 4 files changed, 87 insertions(+), 18 deletions(-)

diff --git a/src/components/playground/PhoneSimulator.tsx b/src/components/playground/PhoneSimulator.tsx
index e66bba8..3969f7c 100644
--- a/src/components/playground/PhoneSimulator.tsx
+++ b/src/components/playground/PhoneSimulator.tsx
@@ -20,9 +20,10 @@ export interface PhoneSimulatorProps {
     onDisconnect: () => void;
     phoneMode?: "normal" | "capture";
     onCapture?: (image: File) => void;
+    capturePrompt?: string;
 }
 
-export function PhoneSimulator({ onConnect, onDisconnect, phoneMode = "normal", onCapture }: PhoneSimulatorProps) {
+export function PhoneSimulator({ onConnect, onDisconnect, phoneMode = "normal", onCapture, capturePrompt }: PhoneSimulatorProps) {
   const { config, setUserSettings } = useConfig();
   const { setToastMessage } = useToast();
   const room = useRoomContext();
@@ -517,6 +518,15 @@ export function PhoneSimulator({ onConnect, onDisconnect, phoneMode = "normal",
                 {/* Center Focus Indicator */}
                 <div className="absolute top-1/2 left-1/2 -translate-x-1/2 -translate-y-1/2 w-16 h-16 border border-white/50 rounded-sm"></div>
                 <div className="absolute top-1/2 left-1/2 -translate-x-1/2 -translate-y-1/2 w-1 h-1 bg-white/50 rounded-full"></div>
+
+                {/* Prompt Display */}
+                {capturePrompt && (
+                   <div className="absolute top-32 left-0 w-full px-6 text-center z-20">
+                      <div className="inline-block bg-black/60 backdrop-blur-md text-white px-4 py-3 rounded-2xl text-sm font-medium shadow-lg border border-white/10 max-w-full break-words">
+                        {capturePrompt}
+                      </div>
+                   </div>
+                )}
             </div>
         )}
         
diff --git a/src/components/playground/Playground.tsx b/src/components/playground/Playground.tsx
index d230fb1..0cfa4b4 100644
--- a/src/components/playground/Playground.tsx
+++ b/src/components/playground/Playground.tsx
@@ -66,6 +66,7 @@ export default function Playground({
   const tracks = useTracks();
   const room = useRoomContext();
   const [phoneMode, setPhoneMode] = useState<"normal" | "capture">("normal");
+  const [capturePrompt, setCapturePrompt] = useState<string>("");
 
   const [rpcMethod, setRpcMethod] = useState("");
   const [rpcPayload, setRpcPayload] = useState("");
@@ -106,7 +107,17 @@ export default function Playground({
 
     localParticipant.registerRpcMethod(
       'enterImageCaptureMode',
-      async () => {
+      async (data: RpcInvocationData) => {
+        if (data.payload) {
+            try {
+                const payload = JSON.parse(data.payload);
+                if (payload.prompt) {
+                    setCapturePrompt(payload.prompt);
+                }
+            } catch (e) {
+                console.error("Failed to parse enterImageCaptureMode payload", e);
+            }
+        }
         setPhoneMode("capture");
         return JSON.stringify({ success: true });
       }
@@ -116,6 +127,7 @@ export default function Playground({
       'exitImageCaptureMode',
       async () => {
         setPhoneMode("normal");
+        setCapturePrompt("");
         return JSON.stringify({ success: true });
       }
     );
@@ -315,6 +327,24 @@ export default function Playground({
     voiceAssistant.agent,
   ]);
 
+  const instructionsContent = (
+    <ConfigurationPanelItem title="Instructions">
+      <textarea
+        className="w-full bg-gray-950 text-white text-sm p-3 rounded-md border border-gray-800 focus:border-gray-600 focus:outline-none transition-colors resize-none disabled:opacity-50 disabled:cursor-not-allowed"
+        style={{ minHeight: "80px" }}
+        rows={3}
+        placeholder="Enter system instructions for the agent..."
+        value={config.settings.instructions}
+        onChange={(e) => {
+          const newSettings = { ...config.settings };
+          newSettings.instructions = e.target.value;
+          setUserSettings(newSettings);
+        }}
+        disabled={roomState !== ConnectionState.Disconnected}
+      />
+    </ConfigurationPanelItem>
+  );
+
   const handleRpcCall = useCallback(async () => {
     if (!voiceAssistant.agent || !room) {
       throw new Error("No agent or room available");
@@ -599,6 +629,7 @@ export default function Playground({
             onConnect={() => onConnect(true)} 
             onDisconnect={() => onConnect(false)}
             phoneMode={phoneMode} 
+            capturePrompt={capturePrompt}
             onCapture={async (content: File) => {
               if (localParticipant) {
                 await localParticipant.sendFile(content, { topic: "image" });
@@ -616,6 +647,19 @@ export default function Playground({
       title: "Chat",
       content: chatTileContent,
     });
+    mobileTabs.push({
+      title: "Instructions",
+      content: (
+        <PlaygroundTile
+          padding={false}
+          backgroundColor="gray-950"
+          className="h-full w-full grow items-start overflow-y-auto"
+          childrenClassName="h-full grow items-start"
+        >
+          {instructionsContent}
+        </PlaygroundTile>
+      ),
+    });
   }
 
   mobileTabs.push({
@@ -672,6 +716,7 @@ export default function Playground({
             onConnect={() => onConnect(true)} 
             onDisconnect={() => onConnect(false)}
             phoneMode={phoneMode} 
+            capturePrompt={capturePrompt}
             onCapture={async (content: File) => {
               if (localParticipant) {
                 await localParticipant.sendFile(content, { topic: "image" });
@@ -683,12 +728,22 @@ export default function Playground({
         </div>
 
         {config.settings.chat && (
-          <PlaygroundTile
-            title="Chat"
-            className="h-full grow basis-1/4 hidden lg:flex"
-          >
-            {chatTileContent}
-          </PlaygroundTile>
+          <div className="flex flex-col h-full grow basis-1/4 hidden lg:flex gap-4">
+            <PlaygroundTile
+              padding={false}
+              backgroundColor="gray-950"
+              className="h-auto w-full flex-none min-h-0"
+              childrenClassName="items-start"
+            >
+              {instructionsContent}
+            </PlaygroundTile>
+            <PlaygroundTile
+              title="Chat"
+              className="w-full grow min-h-0"
+            >
+              {chatTileContent}
+            </PlaygroundTile>
+          </div>
         )}
         <PlaygroundTile
           padding={false}
diff --git a/src/hooks/useConfig.tsx b/src/hooks/useConfig.tsx
index 35ebdfb..4dad264 100644
--- a/src/hooks/useConfig.tsx
+++ b/src/hooks/useConfig.tsx
@@ -40,6 +40,7 @@ export type UserSettings = {
   participant_id: string;
   participant_name: string;
   agent_name?: string;
+  instructions?: string;
   metadata?: string;
   attributes?: AttributeItem[];
 };
@@ -67,6 +68,7 @@ const defaultConfig: AppConfig = {
     room_name: "",
     participant_id: "",
     participant_name: "",
+    instructions: "",
     metadata: "",
     attributes: [],
   },
@@ -140,6 +142,7 @@ export const ConfigProvider = ({ children }: { children: React.ReactNode }) => {
       room_name: "",
       participant_id: "",
       participant_name: "",
+      instructions: "",
     } as UserSettings;
   }, [appConfig]);
 
diff --git a/src/hooks/useConnection.tsx b/src/hooks/useConnection.tsx
index 0222270..f155382 100644
--- a/src/hooks/useConnection.tsx
+++ b/src/hooks/useConnection.tsx
@@ -72,19 +72,19 @@ export const ConnectionProvider = ({
         if (config.settings.metadata) {
           body.metadata = config.settings.metadata;
         }
+        const attributes: Record<string, string> = {};
+        if (config.settings.instructions) {
+          attributes.instructions = config.settings.instructions;
+        }
         const attributesArray = Array.isArray(config.settings.attributes)
           ? config.settings.attributes
           : [];
-        if (attributesArray?.length) {
-          const attributes = attributesArray.reduce(
-            (acc, attr) => {
-              if (attr.key) {
-                acc[attr.key] = attr.value;
-              }
-              return acc;
-            },
-            {} as Record<string, string>,
-          );
+        attributesArray.forEach((attr) => {
+          if (attr.key) {
+            attributes[attr.key] = attr.value;
+          }
+        });
+        if (Object.keys(attributes).length > 0) {
           body.attributes = attributes;
         }
         const { accessToken } = await fetch(`/api/token`, {
@@ -111,6 +111,7 @@ export const ConnectionProvider = ({
       config.settings.participant_id,
       config.settings.metadata,
       config.settings.attributes,
+      config.settings.instructions,
       generateToken,
       setToastMessage,
     ],