Add vision model support and related configurations in Assistant
- Introduce new fields in AssistantConfig, schemas, and database models to support vision capabilities, including `vision_enabled` and `vision_model_resource_id`. - Enhance validation logic in routes to ensure proper handling of vision models and their requirements. - Update the AssistantPage and related frontend components to include options for enabling vision understanding and selecting appropriate vision models. - Modify database seed scripts to include vision-related data for assistants, ensuring consistent setup. - Refactor related functions to integrate vision model handling in the audio-visual processing pipeline.
This commit is contained in:
Xin Wang
@@ -125,6 +125,8 @@ type AssistantForm = {
|
||||
voice: string;
|
||||
knowledgeBase: string;
|
||||
enableInterrupt: boolean;
|
||||
visionEnabled: boolean;
|
||||
visionModelResourceId: string;
|
||||
};
|
||||
|
||||
type FastGptForm = {
|
||||
@@ -155,6 +157,8 @@ type OpenCodeForm = {
|
||||
asr: string;
|
||||
voice: string;
|
||||
enableInterrupt: boolean;
|
||||
visionEnabled: boolean;
|
||||
visionModelResourceId: string;
|
||||
};
|
||||
|
||||
type AssistantType = "提示词" | "工作流" | "Dify" | "FastGPT" | "OpenCode";
|
||||
@@ -214,6 +218,8 @@ function blankPromptForm(name: string): AssistantForm {
|
||||
voice: "",
|
||||
knowledgeBase: "",
|
||||
enableInterrupt: true,
|
||||
visionEnabled: false,
|
||||
visionModelResourceId: "",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -250,6 +256,8 @@ function blankOpenCodeForm(name: string): OpenCodeForm {
|
||||
asr: "",
|
||||
voice: "",
|
||||
enableInterrupt: true,
|
||||
visionEnabled: false,
|
||||
visionModelResourceId: "",
|
||||
};
|
||||
}
|
||||
function formatTimestamp(iso?: string | null): string {
|
||||
@@ -331,8 +339,6 @@ export function AssistantPage(props: AssistantPageProps) {
|
||||
const editingId = props.mode === "edit" ? props.assistantId : null;
|
||||
|
||||
const [form, setForm] = useState<AssistantForm>(() => blankPromptForm(""));
|
||||
// 视觉理解(暂不持久化,不进 savedSnapshot,避免影响「未保存改动」判定)
|
||||
const [visionEnabled, setVisionEnabled] = useState(false);
|
||||
const [fastGptForm, setFastGptForm] = useState<FastGptForm>(() =>
|
||||
blankFastGptForm(""),
|
||||
);
|
||||
@@ -426,32 +432,39 @@ export function AssistantPage(props: AssistantPageProps) {
|
||||
// 按资源类型生成 {value:id, label:name} 选项
|
||||
const credOptions = (type: ModelResource["capability"]) =>
|
||||
modelResources
|
||||
.filter((c) => {
|
||||
if (c.capability !== type) return false;
|
||||
if (type === "LLM" && visionEnabled) return c.supportImageInput;
|
||||
return true;
|
||||
})
|
||||
.filter((c) => c.capability === type)
|
||||
.map((c) => ({ value: c.id, label: c.name }));
|
||||
const visionModelOptionsFor = (currentModelId: string) => modelResources
|
||||
.filter(
|
||||
(c) =>
|
||||
c.capability === "LLM" &&
|
||||
c.supportImageInput &&
|
||||
c.id !== currentModelId,
|
||||
)
|
||||
.map((c) => ({ value: c.id, label: c.name }));
|
||||
const kbOptions = knowledgeBases.map((k) => ({ value: k.id, label: k.name }));
|
||||
|
||||
function modelSupportsImageInput(resourceId: string): boolean {
|
||||
return Boolean(
|
||||
modelResources.find((resource) => resource.id === resourceId)
|
||||
?.supportImageInput,
|
||||
);
|
||||
function handlePromptVisionEnabledChange(enabled: boolean) {
|
||||
updateForm("visionEnabled", enabled);
|
||||
if (!enabled) updateForm("visionModelResourceId", "");
|
||||
}
|
||||
|
||||
function handleVisionEnabledChange(enabled: boolean) {
|
||||
setVisionEnabled(enabled);
|
||||
if (enabled && form.model && !modelSupportsImageInput(form.model)) {
|
||||
updateForm("model", "");
|
||||
function handlePromptModelChange(value: string) {
|
||||
updateForm("model", value);
|
||||
if (form.visionModelResourceId === value) {
|
||||
updateForm("visionModelResourceId", "");
|
||||
}
|
||||
if (
|
||||
enabled &&
|
||||
openCodeForm.model &&
|
||||
!modelSupportsImageInput(openCodeForm.model)
|
||||
) {
|
||||
updateOpenCodeForm("model", "");
|
||||
}
|
||||
|
||||
function handleOpenCodeVisionEnabledChange(enabled: boolean) {
|
||||
updateOpenCodeForm("visionEnabled", enabled);
|
||||
if (!enabled) updateOpenCodeForm("visionModelResourceId", "");
|
||||
}
|
||||
|
||||
function handleOpenCodeModelChange(value: string) {
|
||||
updateOpenCodeForm("model", value);
|
||||
if (openCodeForm.visionModelResourceId === value) {
|
||||
updateOpenCodeForm("visionModelResourceId", "");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -473,6 +486,8 @@ export function AssistantPage(props: AssistantPageProps) {
|
||||
voice: a.modelResourceIds.TTS ?? "",
|
||||
knowledgeBase: a.knowledgeBaseId ?? "",
|
||||
enableInterrupt: a.enableInterrupt,
|
||||
visionEnabled: a.visionEnabled,
|
||||
visionModelResourceId: a.visionModelResourceId ?? "",
|
||||
};
|
||||
setForm(next);
|
||||
return next;
|
||||
@@ -533,6 +548,8 @@ export function AssistantPage(props: AssistantPageProps) {
|
||||
runtimeMode: "pipeline",
|
||||
greeting: "",
|
||||
enableInterrupt: true,
|
||||
visionEnabled: false,
|
||||
visionModelResourceId: null,
|
||||
modelResourceIds: {},
|
||||
knowledgeBaseId: null,
|
||||
prompt: "",
|
||||
@@ -591,6 +608,8 @@ export function AssistantPage(props: AssistantPageProps) {
|
||||
runtimeMode: form.runtimeMode,
|
||||
greeting: form.greeting,
|
||||
enableInterrupt: form.enableInterrupt,
|
||||
visionEnabled: form.visionEnabled,
|
||||
visionModelResourceId: form.visionModelResourceId || null,
|
||||
modelResourceIds: {
|
||||
...(form.model ? { LLM: form.model } : {}),
|
||||
...(form.asr ? { ASR: form.asr } : {}),
|
||||
@@ -679,6 +698,8 @@ export function AssistantPage(props: AssistantPageProps) {
|
||||
asr: a.modelResourceIds.ASR ?? "",
|
||||
voice: a.modelResourceIds.TTS ?? "",
|
||||
enableInterrupt: a.enableInterrupt,
|
||||
visionEnabled: a.visionEnabled,
|
||||
visionModelResourceId: a.visionModelResourceId ?? "",
|
||||
};
|
||||
setOpenCodeForm(next);
|
||||
return next;
|
||||
@@ -748,6 +769,8 @@ export function AssistantPage(props: AssistantPageProps) {
|
||||
name: openCodeForm.name.trim(),
|
||||
type: "opencode",
|
||||
enableInterrupt: openCodeForm.enableInterrupt,
|
||||
visionEnabled: openCodeForm.visionEnabled,
|
||||
visionModelResourceId: openCodeForm.visionModelResourceId || null,
|
||||
modelResourceIds: {
|
||||
...(openCodeForm.model ? { LLM: openCodeForm.model } : {}),
|
||||
...(openCodeForm.asr ? { ASR: openCodeForm.asr } : {}),
|
||||
@@ -1518,7 +1541,11 @@ export function AssistantPage(props: AssistantPageProps) {
|
||||
</SectionCard>
|
||||
</div>
|
||||
|
||||
<DebugDrawer assistantId={editingId} hasUnsavedChanges={dirty} />
|
||||
<DebugDrawer
|
||||
assistantId={editingId}
|
||||
hasUnsavedChanges={dirty}
|
||||
vision={openCodeForm.visionEnabled}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
@@ -1600,14 +1627,25 @@ export function AssistantPage(props: AssistantPageProps) {
|
||||
>
|
||||
<ToggleRow
|
||||
title="视觉理解"
|
||||
hint="开启后，大语言模型只能选择支持图片输入的资源；开始对话时会允许助手按需理解当前视频画面。"
|
||||
checked={visionEnabled}
|
||||
onChange={handleVisionEnabledChange}
|
||||
hint="开启后，开始对话时会允许助手按需理解当前视频画面。视觉模型选「模型自己」时，大语言模型本身必须支持图片输入。"
|
||||
checked={openCodeForm.visionEnabled}
|
||||
onChange={handleOpenCodeVisionEnabledChange}
|
||||
/>
|
||||
{openCodeForm.visionEnabled && (
|
||||
<ResourceSelectField
|
||||
label="视觉模型"
|
||||
value={openCodeForm.visionModelResourceId}
|
||||
onChange={(value) =>
|
||||
updateOpenCodeForm("visionModelResourceId", value)
|
||||
}
|
||||
options={visionModelOptionsFor(openCodeForm.model)}
|
||||
noneLabel="模型自己"
|
||||
/>
|
||||
)}
|
||||
<ResourceSelectField
|
||||
label="大语言模型"
|
||||
value={openCodeForm.model}
|
||||
onChange={(value) => updateOpenCodeForm("model", value)}
|
||||
onChange={handleOpenCodeModelChange}
|
||||
options={credOptions("LLM")}
|
||||
noneLabel="无"
|
||||
/>
|
||||
@@ -1779,14 +1817,25 @@ export function AssistantPage(props: AssistantPageProps) {
|
||||
>
|
||||
<ToggleRow
|
||||
title="视觉理解"
|
||||
hint="开启后，大语言模型只能选择支持图片输入的资源；开始对话时会允许助手按需理解当前视频画面。"
|
||||
checked={visionEnabled}
|
||||
onChange={handleVisionEnabledChange}
|
||||
hint="开启后，开始对话时会允许助手按需理解当前视频画面。视觉模型选「模型自己」时，大语言模型本身必须支持图片输入。"
|
||||
checked={form.visionEnabled}
|
||||
onChange={handlePromptVisionEnabledChange}
|
||||
/>
|
||||
{form.visionEnabled && (
|
||||
<ResourceSelectField
|
||||
label="视觉模型"
|
||||
value={form.visionModelResourceId}
|
||||
onChange={(value) =>
|
||||
updateForm("visionModelResourceId", value)
|
||||
}
|
||||
options={visionModelOptionsFor(form.model)}
|
||||
noneLabel="模型自己"
|
||||
/>
|
||||
)}
|
||||
<ResourceSelectField
|
||||
label="大语言模型"
|
||||
value={form.model}
|
||||
onChange={(value) => updateForm("model", value)}
|
||||
onChange={handlePromptModelChange}
|
||||
options={credOptions("LLM")}
|
||||
noneLabel="无"
|
||||
/>
|
||||
@@ -1863,7 +1912,7 @@ export function AssistantPage(props: AssistantPageProps) {
|
||||
<DebugDrawer
|
||||
assistantId={editingId}
|
||||
hasUnsavedChanges={dirty}
|
||||
vision={visionEnabled}
|
||||
vision={form.visionEnabled}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user