Add support for image input in model resources and enhance related configurations

- Introduce a new `support_image_input` field in model resources, allowing models to indicate support for image input.
- Update the backend models, schemas, and database seed scripts to accommodate the new field.
- Enhance the AssistantConfig and related routes to handle image input capabilities, ensuring proper validation and error handling.
- Modify the frontend components to include toggles for enabling visual understanding and filtering models based on image input support.
- Implement necessary adjustments in the voice preview and pipeline to integrate video stream handling alongside audio functionalities.

frontend/src/components/pages/AssistantPage.tsx

@@ -426,10 +426,35 @@ export function AssistantPage(props: AssistantPageProps) {
   // 按资源类型生成 {value:id, label:name} 选项
   const credOptions = (type: ModelResource["capability"]) =>
     modelResources
-      .filter((c) => c.capability === type)
+      .filter((c) => {
+        if (c.capability !== type) return false;
+        if (type === "LLM" && visionEnabled) return c.supportImageInput;
+        return true;
+      })
       .map((c) => ({ value: c.id, label: c.name }));
   const kbOptions = knowledgeBases.map((k) => ({ value: k.id, label: k.name }));
 
+  function modelSupportsImageInput(resourceId: string): boolean {
+    return Boolean(
+      modelResources.find((resource) => resource.id === resourceId)
+        ?.supportImageInput,
+    );
+  }
+
+  function handleVisionEnabledChange(enabled: boolean) {
+    setVisionEnabled(enabled);
+    if (enabled && form.model && !modelSupportsImageInput(form.model)) {
+      updateForm("model", "");
+    }
+    if (
+      enabled &&
+      openCodeForm.model &&
+      !modelSupportsImageInput(openCodeForm.model)
+    ) {
+      updateOpenCodeForm("model", "");
+    }
+  }
+
   function startCreate() {
     router.push("/assistants/new");
   }
@@ -1369,7 +1394,7 @@ export function AssistantPage(props: AssistantPageProps) {
             >
               <ToggleRow
                 title="允许用户打断"
-                description="用户说话时，助手可以停止当前播报并重新理解用户输入"
+                hint="用户说话时，助手可以停止当前播报并重新理解用户输入。"
                 checked={difyForm.enableInterrupt}
                 onChange={(checked) =>
                   updateDifyForm("enableInterrupt", checked)
@@ -1483,7 +1508,7 @@ export function AssistantPage(props: AssistantPageProps) {
             >
               <ToggleRow
                 title="允许用户打断"
-                description="用户说话时，助手可以停止当前播报并重新理解用户输入"
+                hint="用户说话时，助手可以停止当前播报并重新理解用户输入。"
                 checked={fastGptForm.enableInterrupt}
                 onChange={(checked) =>
                   updateFastGptForm("enableInterrupt", checked)
@@ -1572,6 +1597,12 @@ export function AssistantPage(props: AssistantPageProps) {
               title="模型与语音配置"
               description="配置 OpenCode 使用的大语言模型、语音识别与语音合成资源。"
             >
+              <ToggleRow
+                title="视觉理解"
+                hint="开启后，大语言模型只能选择支持图片输入的资源；开始对话时会允许助手按需理解当前视频画面。"
+                checked={visionEnabled}
+                onChange={handleVisionEnabledChange}
+              />
               <ResourceSelectField
                 label="大语言模型"
                 value={openCodeForm.model}
@@ -1602,7 +1633,7 @@ export function AssistantPage(props: AssistantPageProps) {
             >
               <ToggleRow
                 title="允许用户打断"
-                description="用户说话时，助手可以停止当前播报并重新理解用户输入"
+                hint="用户说话时，助手可以停止当前播报并重新理解用户输入。"
                 checked={openCodeForm.enableInterrupt}
                 onChange={(checked) =>
                   updateOpenCodeForm("enableInterrupt", checked)
@@ -1724,15 +1755,6 @@ export function AssistantPage(props: AssistantPageProps) {
                 </div>
               </div>
             </div>
-            <div className="mt-4 border-t border-hairline pt-4">
-              <ToggleRow
-                icon={<Eye size={16} />}
-                title="视觉理解"
-                hint="开启后，右侧调试面板可切换到视频流预览，并可同时选择麦克风与摄像头。"
-                checked={visionEnabled}
-                onChange={setVisionEnabled}
-              />
-            </div>
           </SectionCard>
 
           <SectionCard
@@ -1754,6 +1776,12 @@ export function AssistantPage(props: AssistantPageProps) {
               title="模型配置"
               description="从「模型资源」中选择大语言模型、语音识别与语音合成"
             >
+              <ToggleRow
+                title="视觉理解"
+                hint="开启后，大语言模型只能选择支持图片输入的资源；开始对话时会允许助手按需理解当前视频画面。"
+                checked={visionEnabled}
+                onChange={handleVisionEnabledChange}
+              />
               <ResourceSelectField
                 label="大语言模型"
                 value={form.model}
@@ -1824,7 +1852,7 @@ export function AssistantPage(props: AssistantPageProps) {
           >
             <ToggleRow
               title="允许用户打断"
-              description="用户说话时，助手可以停止当前播报并重新理解用户输入"
+              hint="用户说话时，助手可以停止当前播报并重新理解用户输入。"
               checked={form.enableInterrupt}
               onChange={(checked) => updateForm("enableInterrupt", checked)}
             />
@@ -1839,7 +1867,7 @@ export function AssistantPage(props: AssistantPageProps) {
 
 type VizStyle = "aura" | "nebula" | "bars" | "wave";
 
-// 调试面板顶部主视图:聊天记录 / 视频流(仅视觉理解开启时可选)
+// 调试面板顶部主视图:聊天记录 / 视频流
 type DebugView = "chat" | "video";
 type DebugInputMode = "mic" | "text";
 
@@ -1919,28 +1947,27 @@ function DebugDrawer({
   const [showTranscript, setShowTranscript] = useState(false);
   const [vizStyle, setVizStyle] = useState<VizStyle>("aura");
   const [view, setView] = useState<DebugView>("chat");
-  const effectiveView: DebugView = vision ? view : "chat";
   const stopCamera = camera.stop;
   const startCamera = camera.start;
-  const cameraShouldRun =
-    vision &&
-    (preview.status === "connecting" || preview.status === "connected");
+  const recording =
+    preview.status === "connecting" || preview.status === "connected";
+  const shouldKeepCamera = vision && recording;
 
   useEffect(() => {
-    if (!vision) {
-      // eslint-disable-next-line react-hooks/set-state-in-effect
-      setView("chat");
-      stopCamera();
-    }
-  }, [vision, stopCamera]);
-
-  useEffect(() => {
-    if (cameraShouldRun) {
+    if (shouldKeepCamera) {
       void startCamera();
     } else {
       stopCamera();
     }
-  }, [cameraShouldRun, camera.deviceId, startCamera, stopCamera]);
+  }, [shouldKeepCamera, startCamera, stopCamera]);
+
+  const selectCamera = useCallback(
+    async (deviceId: string) => {
+      const nextStream = await camera.selectCamera(deviceId);
+      await preview.replaceVideoStream(nextStream);
+    },
+    [camera, preview],
+  );
 
   const containerClass = asSheet
     ? "flex h-full min-w-0 flex-1 flex-col overflow-hidden"
@@ -1959,7 +1986,7 @@ function DebugDrawer({
           />
         </div>
         <div className="flex shrink-0 items-center gap-2">
-          {SHOW_VOICE_VIZ && effectiveView === "chat" && (
+          {SHOW_VOICE_VIZ && view === "chat" && (
             <>
               {!showTranscript && (
                 <SegmentedIconGroup label="可视化样式">
@@ -1995,33 +2022,29 @@ function DebugDrawer({
           )}
           <SegmentedIconGroup label="预览视图">
             <SegmentedIconButton
-              selected={effectiveView === "chat"}
+              selected={view === "chat"}
               label="聊天记录"
               onClick={() => setView("chat")}
             >
               <MessageSquareText size={14} />
             </SegmentedIconButton>
-            {vision && (
-              <SegmentedIconButton
-                selected={effectiveView === "video"}
-                label="视频流"
-                onClick={() => setView("video")}
-              >
-                <Video size={14} />
-              </SegmentedIconButton>
-            )}
+            <SegmentedIconButton
+              selected={view === "video"}
+              label="视频流"
+              onClick={() => setView("video")}
+            >
+              <Video size={14} />
+            </SegmentedIconButton>
           </SegmentedIconGroup>
         </div>
       </div>
-      {vision && (
-        <div className="shrink-0 border-b border-hairline px-5 py-2.5">
-          <div className="flex h-10 min-w-0 items-center rounded-[1.4rem] border border-hairline-strong bg-background px-2">
-            <CameraDeviceField camera={camera} />
-          </div>
+      <div className="shrink-0 border-b border-hairline px-5 py-2.5">
+        <div className="flex h-10 min-w-0 items-center rounded-[1.4rem] border border-hairline-strong bg-background px-2">
+          <CameraDeviceField camera={camera} onSelect={selectCamera} />
         </div>
-      )}
+      </div>
       <DebugVoicePanel
-        view={effectiveView}
+        view={view}
         showTranscript={showTranscript}
         vizStyle={vizStyle}
         assistantId={assistantId}
@@ -2100,7 +2123,13 @@ function MicrophoneDeviceField({ preview }: { preview: VoicePreview }) {
   );
 }
 
-function CameraDeviceField({ camera }: { camera: CameraPreview }) {
+function CameraDeviceField({
+  camera,
+  onSelect,
+}: {
+  camera: CameraPreview;
+  onSelect?: (deviceId: string) => void | Promise<void>;
+}) {
   return (
     <DeviceSelectField
       icon={<Video size={15} className="shrink-0 text-muted-soft" />}
@@ -2109,7 +2138,7 @@ function CameraDeviceField({ camera }: { camera: CameraPreview }) {
       fallbackLabel="摄像头"
       value={camera.deviceId}
       devices={camera.devices}
-      onSelect={(deviceId) => void camera.selectCamera(deviceId)}
+      onSelect={(deviceId) => void (onSelect ?? camera.selectCamera)(deviceId)}
     />
   );
 }
@@ -2179,9 +2208,8 @@ function DebugVoicePanel({
   const inChatView = view === "chat" && (!SHOW_VOICE_VIZ || showTranscript);
   const idleOrFailed = status === "idle" || status === "failed";
   const showIdleHub =
-    messages.length === 0 &&
     idleOrFailed &&
-    (inChatView || (vision && view === "video"));
+    (view === "video" || (messages.length === 0 && inChatView));
 
   function handleSendText() {
     if (sendText(textDraft)) {
@@ -2189,26 +2217,36 @@ function DebugVoicePanel({
     }
   }
 
+  const startConversation = useCallback(async () => {
+    const videoStream = vision ? await camera.start() : null;
+    await connect({
+      visionEnabled: vision,
+      videoStream,
+    });
+  }, [camera, connect, vision]);
+
   return (
     <div className="flex min-h-0 flex-1 flex-col">
       {/* 后端 TTS 音频经 WebRTC 媒体流过来,挂这里播放 */}
       <audio ref={audioRef} autoPlay playsInline className="hidden" />
-      {view === "video" && showIdleHub ? (
-        <DebugIdleHub
-          status={status}
-          error={error}
-          assistantId={assistantId}
-          connect={connect}
-        />
-      ) : view === "video" ? (
-        <DebugVideoPanel camera={camera} />
+      {view === "video" ? (
+        showIdleHub ? (
+          <DebugIdleHub
+            status={status}
+            error={error}
+            assistantId={assistantId}
+            connect={startConversation}
+          />
+        ) : (
+          <DebugVideoPanel camera={camera} />
+        )
       ) : !SHOW_VOICE_VIZ || showTranscript ? (
         showIdleHub ? (
           <DebugIdleHub
             status={status}
             error={error}
             assistantId={assistantId}
-            connect={connect}
+            connect={startConversation}
           />
         ) : (
           <DebugTranscriptPanel messages={messages} recording={recording} />
@@ -2283,7 +2321,7 @@ function DebugVoicePanel({
               if (recording) {
                 disconnect();
               } else {
-                void connect();
+                void startConversation();
               }
             }}
             className={[
@@ -2372,7 +2410,7 @@ function DebugVoicePanel({
                 if (recording) {
                   disconnect();
                 } else {
-                  void connect();
+                  void startConversation();
                 }
               }}
               className={[

frontend/src/components/pages/ComponentsModelsPage.tsx

@@ -5,6 +5,7 @@ import {
   ChevronDown,
   ChevronUp,
   Copy,
+  Eye,
   Loader2,
   MoreHorizontal,
   Pencil,
@@ -37,6 +38,8 @@ import { FilterPills } from "@/components/ui/filter-pills";
 import { SearchInput } from "@/components/ui/search-input";
 import { ListToolbar } from "@/components/ui/list-toolbar";
 import { Input } from "@/components/ui/input";
+import { Switch } from "@/components/ui/switch";
+import { Textarea } from "@/components/ui/textarea";
 import {
   Select,
   SelectContent,
@@ -69,6 +72,7 @@ type ResourceDraft = {
   interfaceType: string;
   values: Record<string, unknown>;
   secrets: Record<string, unknown>;
+  supportImageInput: boolean;
 };
 
 const emptyDraft: ResourceDraft = {
@@ -77,6 +81,7 @@ const emptyDraft: ResourceDraft = {
   interfaceType: "",
   values: {},
   secrets: {},
+  supportImageInput: false,
 };
 
 function defaults(definition: InterfaceDefinition): Record<string, unknown> {
@@ -115,6 +120,44 @@ function fieldValue(draft: ResourceDraft, field: InterfaceField): unknown {
     : draft.values[field.key];
 }
 
+function extraBodyText(values: Record<string, unknown>): string {
+  const value = values.extraBody;
+  if (!value || typeof value !== "object" || Array.isArray(value)) return "";
+  return JSON.stringify(value, null, 2);
+}
+
+function parseExtraBody(text: string):
+  | { ok: true; value: Record<string, unknown> | null }
+  | { ok: false; error: string } {
+  const trimmed = text.trim();
+  if (!trimmed) return { ok: true, value: null };
+  try {
+    const parsed = JSON.parse(trimmed) as unknown;
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      return { ok: false, error: "Extra Body 必须是 JSON 对象" };
+    }
+    return { ok: true, value: parsed as Record<string, unknown> };
+  } catch {
+    return { ok: false, error: "Extra Body 不是合法 JSON" };
+  }
+}
+
+function valuesWithExtraBody(
+  values: Record<string, unknown>,
+  extraBody: Record<string, unknown> | null,
+): Record<string, unknown> {
+  const next = { ...values };
+  delete next.extraBody;
+  if (extraBody) next.extraBody = extraBody;
+  return next;
+}
+
+function isSelectableDefinition(definition: InterfaceDefinition): boolean {
+  return (
+    definition.capability !== "LLM" || definition.interfaceType === "openai-llm"
+  );
+}
+
 export function ComponentsModelsPage() {
   const [resources, setResources] = useState<ModelResource[]>([]);
   const [definitions, setDefinitions] = useState<InterfaceDefinition[]>([]);
@@ -128,6 +171,7 @@ export function ComponentsModelsPage() {
   const [dialogOpen, setDialogOpen] = useState(false);
   const [editingId, setEditingId] = useState<string | null>(null);
   const [draft, setDraft] = useState<ResourceDraft>(emptyDraft);
+  const [extraBodyDraft, setExtraBodyDraft] = useState("");
   const [saving, setSaving] = useState(false);
   const [saveError, setSaveError] = useState("");
   const [testing, setTesting] = useState(false);
@@ -161,7 +205,8 @@ export function ComponentsModelsPage() {
     void load();
   }, [load]);
 
-  const availableDefinitions = definitions.filter(
+  const selectableDefinitions = definitions.filter(isSelectableDefinition);
+  const availableDefinitions = selectableDefinitions.filter(
     (definition) => definition.capability === draft.capability,
   );
   const selectedDefinition = definitions.find(
@@ -239,11 +284,12 @@ export function ComponentsModelsPage() {
       values: definition ? defaults(definition) : {},
       secrets: {},
     }));
+    setExtraBodyDraft("");
     setTestResult(null);
   }
 
   function selectCapability(capability: ModelType) {
-    const first = definitions.find(
+    const first = selectableDefinitions.find(
       (definition) => definition.capability === capability,
     );
     setDraft((previous) => ({
@@ -252,12 +298,15 @@ export function ComponentsModelsPage() {
       interfaceType: first?.interfaceType ?? "",
       values: first ? defaults(first) : {},
       secrets: {},
+      supportImageInput:
+        capability === "LLM" ? previous.supportImageInput : false,
     }));
+    setExtraBodyDraft("");
     setTestResult(null);
   }
 
   function openCreate() {
-    const first = definitions.find(
+    const first = selectableDefinitions.find(
       (definition) => definition.capability === "LLM",
     );
     setEditingId(null);
@@ -265,7 +314,9 @@ export function ComponentsModelsPage() {
       ...emptyDraft,
       interfaceType: first?.interfaceType ?? "",
       values: first ? defaults(first) : {},
+      supportImageInput: false,
     });
+    setExtraBodyDraft("");
     setSaveError("");
     setTestResult(null);
     setDialogOpen(true);
@@ -279,7 +330,9 @@ export function ComponentsModelsPage() {
       interfaceType: resource.interfaceType,
       values: resource.values,
       secrets: resource.secrets,
+      supportImageInput: resource.supportImageInput,
     });
+    setExtraBodyDraft(extraBodyText(resource.values));
     setSaveError("");
     setTestResult(null);
     setDialogOpen(true);
@@ -294,9 +347,12 @@ export function ComponentsModelsPage() {
     setTestResult(null);
   }
 
+  const parsedExtraBody = parseExtraBody(extraBodyDraft);
+  const extraBodyError = parsedExtraBody.ok ? "" : parsedExtraBody.error;
   const canSave = Boolean(
     draft.name.trim() &&
       selectedDefinition &&
+      !extraBodyError &&
       selectedDefinition.fieldSchema.fields.every((field) => {
         if (!field.required) return true;
         const value = fieldValue(draft, field);
@@ -305,11 +361,17 @@ export function ComponentsModelsPage() {
   );
 
   function payload() {
+    const extraBody = parsedExtraBody.ok ? parsedExtraBody.value : null;
     return {
       name: draft.name.trim(),
       interfaceType: draft.interfaceType,
-      values: draft.values,
+      values:
+        draft.capability === "LLM"
+          ? valuesWithExtraBody(draft.values, extraBody)
+          : draft.values,
       secrets: draft.secrets,
+      supportImageInput:
+        draft.capability === "LLM" ? draft.supportImageInput : false,
       enabled: editingId
         ? resources.find((resource) => resource.id === editingId)?.enabled ?? true
         : true,
@@ -457,8 +519,19 @@ export function ComponentsModelsPage() {
               width: "md:w-[360px]",
               cell: (resource) => (
                 <>
-                  <div className="truncate font-medium text-foreground">
-                    {resource.name}
+                  <div className="flex min-w-0 items-center gap-2">
+                    <span className="truncate font-medium text-foreground">
+                      {resource.name}
+                    </span>
+                    {resource.supportImageInput && (
+                      <span
+                        className="flex h-5 w-5 shrink-0 items-center justify-center rounded-full bg-surface-strong text-muted-foreground"
+                        title="支持图片输入"
+                        aria-label="支持图片输入"
+                      >
+                        <Eye size={12} />
+                      </span>
+                    )}
                   </div>
                   <div className="mt-1 truncate text-xs text-muted-soft">
                     {String(
@@ -652,6 +725,20 @@ export function ComponentsModelsPage() {
                     </Select>
                   </Field>
                 </div>
+                {draft.capability === "LLM" && (
+                  <div className="flex items-center justify-between rounded-xl border border-hairline p-4">
+                    <span className="text-sm font-medium">支持图片输入</span>
+                    <Switch
+                      checked={draft.supportImageInput}
+                      onCheckedChange={(checked) =>
+                        setDraft((previous) => ({
+                          ...previous,
+                          supportImageInput: checked,
+                        }))
+                      }
+                    />
+                  </div>
+                )}
               </FieldSection>
 
               <FieldSection title="连接与鉴权信息" scrollable fill>
@@ -667,6 +754,25 @@ export function ComponentsModelsPage() {
             </div>
 
             <FieldSection title="可选项" scrollable tall>
+              {draft.capability === "LLM" && (
+                <Field label="Extra Body">
+                  <Textarea
+                    rows={7}
+                    value={extraBodyDraft}
+                    onChange={(event) => {
+                      setExtraBodyDraft(event.target.value);
+                      setTestResult(null);
+                    }}
+                    placeholder={'例如：{\n  "thinking": { "type": "disabled" }\n}'}
+                    className="font-mono text-xs leading-5"
+                  />
+                  {extraBodyError && (
+                    <div className="text-xs text-destructive">
+                      {extraBodyError}
+                    </div>
+                  )}
+                </Field>
+              )}
               {optionalFields.length > 0 ? (
                 optionalFields.map((field) => (
                   <DynamicField
@@ -676,7 +782,7 @@ export function ComponentsModelsPage() {
                     onChange={(value) => updateField(field, value)}
                   />
                 ))
-              ) : (
+              ) : draft.capability === "LLM" ? null : (
                 <div className="flex h-full items-center justify-center text-sm text-muted-foreground">
                   当前接口没有可选项
                 </div>