diff --git a/src/components/playground/PhoneSimulator.tsx b/src/components/playground/PhoneSimulator.tsx
index e66bba8..3969f7c 100644
--- a/src/components/playground/PhoneSimulator.tsx
+++ b/src/components/playground/PhoneSimulator.tsx
@@ -20,9 +20,10 @@ export interface PhoneSimulatorProps {
onDisconnect: () => void;
phoneMode?: "normal" | "capture";
onCapture?: (image: File) => void;
+ capturePrompt?: string;
}
-export function PhoneSimulator({ onConnect, onDisconnect, phoneMode = "normal", onCapture }: PhoneSimulatorProps) {
+export function PhoneSimulator({ onConnect, onDisconnect, phoneMode = "normal", onCapture, capturePrompt }: PhoneSimulatorProps) {
const { config, setUserSettings } = useConfig();
const { setToastMessage } = useToast();
const room = useRoomContext();
@@ -517,6 +518,15 @@ export function PhoneSimulator({ onConnect, onDisconnect, phoneMode = "normal",
{/* Center Focus Indicator */}
+
+ {/* Prompt Display */}
+ {capturePrompt && (
+
+ )}
)}
diff --git a/src/components/playground/Playground.tsx b/src/components/playground/Playground.tsx
index d230fb1..0cfa4b4 100644
--- a/src/components/playground/Playground.tsx
+++ b/src/components/playground/Playground.tsx
@@ -66,6 +66,7 @@ export default function Playground({
const tracks = useTracks();
const room = useRoomContext();
const [phoneMode, setPhoneMode] = useState<"normal" | "capture">("normal");
+ const [capturePrompt, setCapturePrompt] = useState("");
const [rpcMethod, setRpcMethod] = useState("");
const [rpcPayload, setRpcPayload] = useState("");
@@ -106,7 +107,17 @@ export default function Playground({
localParticipant.registerRpcMethod(
'enterImageCaptureMode',
- async () => {
+ async (data: RpcInvocationData) => {
+ if (data.payload) {
+ try {
+ const payload = JSON.parse(data.payload);
+ if (payload.prompt) {
+ setCapturePrompt(payload.prompt);
+ }
+ } catch (e) {
+ console.error("Failed to parse enterImageCaptureMode payload", e);
+ }
+ }
setPhoneMode("capture");
return JSON.stringify({ success: true });
}
@@ -116,6 +127,7 @@ export default function Playground({
'exitImageCaptureMode',
async () => {
setPhoneMode("normal");
+ setCapturePrompt("");
return JSON.stringify({ success: true });
}
);
@@ -315,6 +327,24 @@ export default function Playground({
voiceAssistant.agent,
]);
+ const instructionsContent = (
+
+
+ );
+
const handleRpcCall = useCallback(async () => {
if (!voiceAssistant.agent || !room) {
throw new Error("No agent or room available");
@@ -599,6 +629,7 @@ export default function Playground({
onConnect={() => onConnect(true)}
onDisconnect={() => onConnect(false)}
phoneMode={phoneMode}
+ capturePrompt={capturePrompt}
onCapture={async (content: File) => {
if (localParticipant) {
await localParticipant.sendFile(content, { topic: "image" });
@@ -616,6 +647,19 @@ export default function Playground({
title: "Chat",
content: chatTileContent,
});
+ mobileTabs.push({
+ title: "Instructions",
+ content: (
+
+ {instructionsContent}
+
+ ),
+ });
}
mobileTabs.push({
@@ -672,6 +716,7 @@ export default function Playground({
onConnect={() => onConnect(true)}
onDisconnect={() => onConnect(false)}
phoneMode={phoneMode}
+ capturePrompt={capturePrompt}
onCapture={async (content: File) => {
if (localParticipant) {
await localParticipant.sendFile(content, { topic: "image" });
@@ -683,12 +728,22 @@ export default function Playground({
{config.settings.chat && (
-
- {chatTileContent}
-
+
+
+ {instructionsContent}
+
+
+ {chatTileContent}
+
+
)}
{
room_name: "",
participant_id: "",
participant_name: "",
+ instructions: "",
} as UserSettings;
}, [appConfig]);
diff --git a/src/hooks/useConnection.tsx b/src/hooks/useConnection.tsx
index 0222270..f155382 100644
--- a/src/hooks/useConnection.tsx
+++ b/src/hooks/useConnection.tsx
@@ -72,19 +72,19 @@ export const ConnectionProvider = ({
if (config.settings.metadata) {
body.metadata = config.settings.metadata;
}
+ const attributes: Record = {};
+ if (config.settings.instructions) {
+ attributes.instructions = config.settings.instructions;
+ }
const attributesArray = Array.isArray(config.settings.attributes)
? config.settings.attributes
: [];
- if (attributesArray?.length) {
- const attributes = attributesArray.reduce(
- (acc, attr) => {
- if (attr.key) {
- acc[attr.key] = attr.value;
- }
- return acc;
- },
- {} as Record,
- );
+ attributesArray.forEach((attr) => {
+ if (attr.key) {
+ attributes[attr.key] = attr.value;
+ }
+ });
+ if (Object.keys(attributes).length > 0) {
body.attributes = attributes;
}
const { accessToken } = await fetch(`/api/token`, {
@@ -111,6 +111,7 @@ export const ConnectionProvider = ({
config.settings.participant_id,
config.settings.metadata,
config.settings.attributes,
+ config.settings.instructions,
generateToken,
setToastMessage,
],