wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add prompt opening behavior modes

Browse Source
This commit is contained in:
Xin Wang
2026-08-04 13:29:21 +08:00
parent 78bba090a2
commit 5bf5987fe4
11 changed files with 235 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
backend/schemas.py
View File

@@ -29,6 +29,7 @@ ToolExecutionMode = Literal["immediate", "async"]
ClientToolResponseWaitMode = Literal["timeout", "session"]
DynamicVariableType = Literal["string", "number", "boolean"]
PromptEntryMode = Literal["wait_user", "generate"]
PromptOpeningMode = Literal["interruptible", "playback", "confirmation"]
# 外部应用类型:其 config.apiKey 是该助手私有密钥,读时打码 / 写时哨兵
EXTERNAL_TYPES = {"dify", "fastgpt", "opencode"}
@@ -107,11 +108,11 @@ class OpeningMessageConfig(CamelModel):
title: str = Field(default="重要提示", min_length=1, max_length=120)
message: str = Field(min_length=1, max_length=2000)
confirm_label: str = Field(default="确认", min_length=1, max_length=40)
skip_speech_on_confirm: bool = True
class StartupConfig(CamelModel):
execution_mode: Literal["sequential"] = "sequential"
opening_mode: PromptOpeningMode | None = None
entry_mode: PromptEntryMode | None = None
actions: list[StartupAction] = Field(default_factory=list, max_length=5)
opening_message: OpeningMessageConfig | None = None
@@ -121,11 +122,22 @@ class StartupConfig(CamelModel):
ids = [action.id for action in self.actions]
if len(ids) != len(set(ids)):
raise ValueError("启动 Action 的 id 不能重复")
legacy_confirmation = (
self.opening_mode is None and self.opening_message is not None
)
if self.opening_mode is None:
self.opening_mode = (
"confirmation" if legacy_confirmation else "interruptible"
)
if self.opening_mode == "confirmation" and self.opening_message is None:
raise ValueError("确认后继续必须配置开场确认弹窗")
if self.opening_mode != "confirmation":
self.opening_message = None
if self.entry_mode is None:
# Preserve the behavior introduced before entryMode became an
# explicit setting. Newly saved clients always send the field.
self.entry_mode = (
"generate" if self.opening_message is not None else "wait_user"
"generate" if legacy_confirmation else "wait_user"
)
return self
@@ -191,6 +203,7 @@ class AssistantUpsert(CamelModel):
if self.runtime_mode == "realtime" and (
self.startup.actions
or self.startup.opening_message is not None
or self.startup.opening_mode != "interruptible"
or self.startup.entry_mode != "wait_user"
):
raise ValueError("Prompt Realtime 模式暂不支持启动阶段")

5
backend/services/brains/base.py
View File

@@ -132,6 +132,9 @@ class BaseBrain:
async def on_greeting_finished(self) -> None:
"""Continue startup after the shared greeting has actually played."""
async def on_interruption_processed(self) -> None:
"""Observe that an output interruption reached assistant aggregation."""
def prepare_greeting_context(
self,
greeting: str,
@@ -214,6 +217,8 @@ class Brain(Protocol):
async def on_greeting_finished(self) -> None: ...
async def on_interruption_processed(self) -> None: ...
def prepare_greeting_context(
self,
greeting: str,

54
backend/services/brains/prompt_brain.py
View File

@@ -81,7 +81,7 @@ class PromptBrain(BaseBrain):
async def greeting(self, cfg: AssistantConfig) -> str:
# The built-in opening Message owns the greeting so speech and the
# client dialog can start as one atomic stage.
if self._opening_message() is not None:
if self._opening_mode() == "confirmation":
return ""
return self._render_greeting(cfg)
@@ -155,8 +155,19 @@ class PromptBrain(BaseBrain):
async def on_connected(self, *, greeting_pending: bool = False) -> None:
self._greeting_pending = greeting_pending
should_block_input = (
self._has_opening_stage()
or (
greeting_pending
and self._opening_mode() == "playback"
)
or (
not greeting_pending
and self._entry_mode() == "generate"
)
)
if (
(self._has_opening_stage() or self._entry_mode() == "generate")
should_block_input
and self._runtime is not None
and self._runtime.set_input_enabled is not None
):
@@ -168,6 +179,16 @@ class PromptBrain(BaseBrain):
self._greeting_pending = False
await self._enter_prompt_if_ready()
async def on_interruption_processed(self) -> None:
if (
self._greeting_pending
and self._opening_mode() == "interruptible"
):
# The user's input now owns the first Agent turn. Do not also queue
# an empty-context entry reply when the interrupted greeting stops.
self._entry_dispatched = True
logger.debug("Prompt 可打断开场白已被用户输入接管")
async def on_client_ready(self) -> None:
if self._output is not None:
await self._output.mark_client_ready()
@@ -180,7 +201,11 @@ class PromptBrain(BaseBrain):
runtime = self._runtime
if runtime is None:
raise RuntimeError("PromptBrain 尚未初始化")
opening_message = self._opening_message()
opening_message = (
self._opening_message()
if self._opening_mode() == "confirmation"
else None
)
opening_actions = self._startup_actions("opening")
speech = (
self._render_greeting(self._cfg).strip()
@@ -238,6 +263,19 @@ class PromptBrain(BaseBrain):
value = startup.get("opening_message", startup.get("openingMessage"))
return value if isinstance(value, dict) else None
def _opening_mode(self) -> str:
startup = self._cfg.startup if isinstance(self._cfg.startup, dict) else {}
value = str(startup.get("opening_mode", startup.get("openingMode")) or "")
if value in {"interruptible", "playback", "confirmation"}:
return value
# Before openingMode existed, an opening message meant confirmation;
# ordinary Prompt greetings were interruptible.
return (
"confirmation"
if self._opening_message() is not None
else "interruptible"
)
def _entry_mode(self) -> str:
startup = self._cfg.startup if isinstance(self._cfg.startup, dict) else {}
value = str(startup.get("entry_mode", startup.get("entryMode")) or "")
@@ -245,7 +283,7 @@ class PromptBrain(BaseBrain):
return value
# Raw runtime configs saved before entryMode existed generated a first
# reply after their opening confirmation. Keep that legacy behavior.
return "generate" if self._opening_message() is not None else "wait_user"
return "generate" if self._opening_mode() == "confirmation" else "wait_user"
async def _enter_prompt_if_ready(self) -> None:
if (
@@ -269,7 +307,7 @@ class PromptBrain(BaseBrain):
self._opening_input_blocked = False
def _has_opening_stage(self) -> bool:
return self._opening_message() is not None or bool(
return self._opening_mode() == "confirmation" or bool(
self._startup_actions("opening")
)
@@ -322,12 +360,6 @@ class PromptBrain(BaseBrain):
).strip(),
),
completion_policy=MESSAGE_CONFIRMATION,
skip_speech_on_confirmation=bool(
config.get(
"skip_speech_on_confirm",
config.get("skipSpeechOnConfirm", True),
)
),
)
async def _publish_opening_outcome(

17
backend/services/message_stage.py
View File

@@ -38,7 +38,6 @@ class MessageStageSpec:
speech: str = ""
display: MessageDisplaySpec | None = None
completion_policy: MessageCompletionPolicy = MESSAGE_PLAYBACK
skip_speech_on_confirmation: bool = True
@dataclass(frozen=True)
@@ -55,7 +54,7 @@ class MessageStageRunner:
"""Run one atomic user-visible message stage.
Speech is queued before the client message is dispatched. A confirmation
may interrupt the remaining audio or keep playback as the final gate. A
stage completes when the user confirms, even if audio is still playing. A
speech-only stage completes at the real transport playback boundary.
"""
@@ -118,14 +117,11 @@ class MessageStageRunner:
return result
action = result.action
# A confirmation may either interrupt the remaining speech or
# acknowledge the dialog while keeping playback as the final gate.
# Confirmation is the gate. It deliberately does not wait for the
# audio completion future, so the user can continue immediately.
if (
playback_completion is not None
and (
not require_confirmation
or not spec.skip_speech_on_confirmation
)
and not require_confirmation
):
await playback_completion
@@ -191,10 +187,7 @@ class MessageStageRunner:
response_wait_mode=(
"session" if require_confirmation else "timeout"
),
interrupt_on_result=(
require_confirmation
and spec.skip_speech_on_confirmation
),
interrupt_on_result=require_confirmation,
)
except ClientToolError as exc:
return MessageStageResult(

3
backend/services/pipecat/pipeline_events.py
View File

@@ -161,6 +161,9 @@ def bind_cascade_pipeline_events(
@assistant_aggregator.event_handler("on_interruption_processed")
async def on_interruption_processed(_aggregator):
brain_handler = getattr(brain, "on_interruption_processed", None)
if callable(brain_handler):
await brain_handler()
if client_tools is not None:
client_tools.on_interruption_processed()
if not pending_user_inputs:

86
backend/tests/test_brains.py
View File

@@ -366,6 +366,7 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
tools=[tool],
startup={
"execution_mode": "sequential",
"opening_mode": "confirmation",
"entry_mode": "generate",
"opening_message": {
"title": "重要提示",
@@ -485,6 +486,7 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
type="prompt",
greeting="请先确认",
startup={
"opening_mode": "confirmation",
"entry_mode": "wait_user",
"opening_message": {
"title": "重要提示",
@@ -527,7 +529,10 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
cfg = AssistantConfig(
type="prompt",
greeting="固定开场白",
startup={"entry_mode": "generate"},
startup={
"opening_mode": "playback",
"entry_mode": "generate",
},
)
brain = build_brain(cfg)
queued = []
@@ -561,45 +566,22 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
1,
)
async def test_opening_confirmation_can_keep_speech_playing_before_entry(self):
async def test_interruptible_opening_user_input_suppresses_entry_reply(self):
cfg = AssistantConfig(
type="prompt",
greeting="必须完整播放的开场白",
greeting="可以打断的开场白",
startup={
"opening_mode": "interruptible",
"entry_mode": "generate",
"opening_message": {
"title": "重要提示",
"message": "请确认已阅读。",
"skip_speech_on_confirm": False,
},
},
)
brain = build_brain(cfg)
queued = []
input_states = []
message_displayed = asyncio.Event()
class TrackedCallEnd(FakeCallEnd):
def __init__(self):
super().__init__()
self.playback_completion = None
def track_speech(self):
self.tracked_speeches += 1
self.playback_completion = asyncio.get_running_loop().create_future()
return self.playback_completion
class FakeClientTools:
async def call(self, *_args, **options):
self.options = options
message_displayed.set()
return {"status": "ok", "data": {"action": "confirmed"}}
async def queue_frame(frame):
queued.append(frame)
call_end = TrackedCallEnd()
client_tools = FakeClientTools()
await brain.setup(
cfg,
BrainRuntime(
@@ -608,24 +590,52 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
queue_frame=queue_frame,
set_system_prompt=lambda _prompt: None,
set_tools=lambda _tools: None,
call_end=call_end,
client_tools=client_tools,
call_end=FakeCallEnd(),
set_input_enabled=input_states.append,
),
)
await brain.on_connected(greeting_pending=False)
opening_task = asyncio.create_task(brain.on_client_ready())
await message_displayed.wait()
await asyncio.sleep(0)
await brain.on_connected(greeting_pending=True)
self.assertEqual(input_states, [])
await brain.on_interruption_processed()
await brain.on_greeting_finished()
self.assertFalse(client_tools.options["interrupt_on_result"])
self.assertFalse(opening_task.done())
self.assertFalse(any(isinstance(frame, LLMRunFrame) for frame in queued))
call_end.playback_completion.set_result(None)
await opening_task
self.assertEqual(input_states, [False, True])
async def test_interruptible_opening_natural_finish_runs_entry_behavior(self):
cfg = AssistantConfig(
type="prompt",
greeting="可以打断的开场白",
startup={
"opening_mode": "interruptible",
"entry_mode": "generate",
},
)
brain = build_brain(cfg)
queued = []
input_states = []
async def queue_frame(frame):
queued.append(frame)
await brain.setup(
cfg,
BrainRuntime(
context=LLMContext(messages=[]),
llm=FakeLLM(),
queue_frame=queue_frame,
set_system_prompt=lambda _prompt: None,
set_tools=lambda _tools: None,
call_end=FakeCallEnd(),
set_input_enabled=input_states.append,
),
)
await brain.on_connected(greeting_pending=True)
await brain.on_greeting_finished()
self.assertEqual(input_states, [])
self.assertEqual(
sum(isinstance(frame, LLMRunFrame) for frame in queued),
1,

5
backend/tests/test_pipeline_events.py
View File

@@ -52,6 +52,7 @@ class _Brain:
self.prepared_greeting = ""
self.greeting_pending = False
self.greeting_finished = 0
self.interruptions_processed = 0
def prepare_greeting_context(self, greeting, _context):
self.prepared_greeting = greeting
@@ -62,6 +63,9 @@ class _Brain:
async def on_greeting_finished(self):
self.greeting_finished += 1
async def on_interruption_processed(self):
self.interruptions_processed += 1
async def on_client_ready(self):
for content, timestamp in (
("Message 节点播报", "2026-07-14T10:00:00.200+00:00"),
@@ -111,6 +115,7 @@ class PipelineEventTest(unittest.IsolatedAsyncioTestCase):
)
self.assertEqual(acknowledgements, [True])
self.assertEqual(brain.interruptions_processed, 1)
async def test_greeting_keeps_playback_timestamp_until_client_ready(self):
transport = _EventSource()

2
backend/tests/test_startup_actions.py
View File

@@ -109,7 +109,7 @@ class StartupActionValidationTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(body.startup.actions, [])
self.assertEqual(body.startup.opening_message.confirm_label, "确认")
self.assertTrue(body.startup.opening_message.skip_speech_on_confirm)
self.assertEqual(body.startup.opening_mode, "confirmation")
self.assertEqual(body.startup.entry_mode, "generate")
async def test_startup_tool_does_not_need_conversation_binding(self):

138
frontend/src/components/assistant-editor/prompt-editor.tsx
View File

@@ -26,7 +26,6 @@ import {
ResourceSelectField,
RuntimeModeSelector,
TextAreaField,
ToggleRow,
ToolPicker,
} from "@/components/assistant-editor/editor-controls";
import type { AssistantForm } from "@/components/assistant-editor/types";
@@ -35,9 +34,52 @@ import { VisionConfigSection } from "@/components/editor/vision-config-section";
import { TurnConfigEditor } from "@/components/turn-config-editor";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import type { DynamicVariableDefinition, Tool } from "@/lib/api";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import type {
DynamicVariableDefinition,
StartupConfig,
Tool,
} from "@/lib/api";
type ResourceOption = { value: string; label: string };
type PromptOpeningMode = StartupConfig["openingMode"];
type PromptEntryMode = StartupConfig["entryMode"];
const OPENING_MODE_OPTIONS: Array<{
value: PromptOpeningMode;
label: string;
description: string;
}> = [
{
value: "interruptible",
label: "可打断",
description: "用户说话、发文字或图片时立即结束开场白，并保留这次输入。",
},
{
value: "playback",
label: "播完继续",
description: "开场白播放期间关闭输入，实际播放完成后进入 Agent。",
},
{
value: "confirmation",
label: "需要弹窗确认继续",
description: "关闭对话输入并显示弹窗，用户确认后立即进入 Agent。",
},
];
const ENTRY_MODE_OPTIONS: Array<{
value: PromptEntryMode;
label: string;
}> = [
{ value: "wait_user", label: "等待下一轮用户输入" },
{ value: "generate", label: "进入后立即回复" },
];
const promptSections = [
{ id: "conversation", label: "对话内容" },
@@ -94,6 +136,10 @@ export function PromptEditor({
handlePromptModelChange,
}: PromptEditorProps) {
const openingMessage = form.startup.openingMessage;
const openingMode = form.startup.openingMode;
const openingModeDescription = OPENING_MODE_OPTIONS.find(
(option) => option.value === openingMode,
)?.description;
const scrollContainerRef = useRef<HTMLDivElement>(null);
const sectionRefs = useRef<Record<PromptSectionId, HTMLElement | null>>({
conversation: null,
@@ -192,15 +238,15 @@ export function PromptEditor({
});
}
function setOpeningMessage(enabled: boolean) {
function setOpeningMode(nextMode: PromptOpeningMode) {
updateForm("startup", {
...form.startup,
openingMessage: enabled
openingMode: nextMode,
openingMessage: nextMode === "confirmation"
? {
title: "重要提示",
message: "请确认已阅读以上信息。",
confirmLabel: "确认",
skipSpeechOnConfirm: true,
title: openingMessage?.title ?? "重要提示",
message: openingMessage?.message ?? "请确认已阅读以上信息。",
confirmLabel: openingMessage?.confirmLabel ?? "确认",
}
: null,
});
@@ -322,14 +368,33 @@ export function PromptEditor({
/>
{form.runtimeMode === "pipeline" && (
<div className="mt-4 space-y-3 border-t border-hairline pt-4">
<ToggleRow
title="开场确认弹窗"
description="与开场白同时显示，用户确认后按下方策略进入对话。"
checked={Boolean(openingMessage)}
onChange={setOpeningMessage}
/>
<div>
<div className="mb-1.5 text-sm font-medium text-foreground">
开场白继续方式
</div>
<Select
value={openingMode}
onValueChange={(value: PromptOpeningMode) =>
setOpeningMode(value)
}
>
<SelectTrigger className="w-full border-hairline-strong bg-background text-foreground">
<SelectValue />
</SelectTrigger>
<SelectContent>
{OPENING_MODE_OPTIONS.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
</SelectItem>
))}
</SelectContent>
</Select>
<p className="mt-1.5 text-xs leading-5 text-muted-foreground">
{openingModeDescription}
</p>
</div>
{openingMessage && (
{openingMode === "confirmation" && openingMessage && (
<div className="space-y-3 rounded-xl border border-hairline bg-canvas-soft p-4">
<label className="block">
<span className="mb-1.5 block text-sm font-medium text-foreground">
@@ -368,33 +433,39 @@ export function PromptEditor({
className="border-hairline-strong bg-background"
/>
</label>
<ToggleRow
title="确认后跳过剩余开场白"
description="开启后立即停止尚未播完的开场白；关闭后等待播报完成再进入对话。"
checked={openingMessage.skipSpeechOnConfirm}
onChange={(skipSpeechOnConfirm) =>
updateOpeningMessage({ skipSpeechOnConfirm })
}
/>
<p className="text-xs leading-5 text-muted-foreground">
弹窗不可跳过；失败时将结束本次通话。这是平台内置开场阶段，
不会作为工具暴露给模型。
确认前关闭语音、文字和图片输入；确认后立即停止剩余开场白并进入 Agent。
</p>
</div>
)}
<div className="border-t border-hairline pt-3">
<ToggleRow
title="进入后立即回复"
description="开启后由助手主动生成首句；关闭后等待用户下一轮输入。"
checked={form.startup.entryMode === "generate"}
onChange={(generate) =>
<div className="mb-1.5 text-sm font-medium text-foreground">
Agent 进入行为
</div>
<Select
value={form.startup.entryMode}
onValueChange={(entryMode: PromptEntryMode) =>
updateForm("startup", {
...form.startup,
entryMode: generate ? "generate" : "wait_user",
entryMode,
})
}
/>
>
<SelectTrigger className="w-full border-hairline-strong bg-background text-foreground">
<SelectValue />
</SelectTrigger>
<SelectContent>
{ENTRY_MODE_OPTIONS.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
</SelectItem>
))}
</SelectContent>
</Select>
<p className="mt-1.5 text-xs leading-5 text-muted-foreground">
该设置仅控制开场阶段正常完成后的 Agent 首轮行为。
</p>
</div>
</div>
)}
@@ -423,10 +494,13 @@ export function PromptEditor({
if (
runtimeMode === "realtime" &&
(form.startup.actions.length ||
form.startup.openingMode !== "interruptible" ||
form.startup.entryMode !== "wait_user" ||
form.startup.openingMessage)
) {
updateForm("startup", {
executionMode: "sequential",
openingMode: "interruptible",
entryMode: "wait_user",
actions: [],
openingMessage: null,

3
frontend/src/components/pages/AssistantPage.tsx
View File

@@ -179,6 +179,7 @@ function blankPromptForm(name: string): AssistantForm {
turnConfig: defaultTurnConfig(),
startup: {
executionMode: "sequential",
openingMode: "interruptible",
entryMode: "wait_user",
actions: [],
openingMessage: null,
@@ -473,6 +474,7 @@ export function AssistantPage(props: AssistantPageProps) {
turnConfig: normalizeTurnConfig(a.turnConfig),
startup: {
executionMode: "sequential",
openingMode: a.startup?.openingMode ?? "interruptible",
entryMode: a.startup?.entryMode ?? "wait_user",
actions: a.startup?.actions ?? [],
openingMessage: a.startup?.openingMessage ?? null,
@@ -548,6 +550,7 @@ export function AssistantPage(props: AssistantPageProps) {
turnConfig: defaultTurnConfig(),
startup: {
executionMode: "sequential",
openingMode: "interruptible",
entryMode: "wait_user",
actions: [],
openingMessage: null,

2
frontend/src/lib/api.ts
View File

@@ -206,13 +206,13 @@ export type StartupAction = {
export type StartupConfig = {
executionMode: "sequential";
openingMode: "interruptible" | "playback" | "confirmation";
entryMode: "wait_user" | "generate";
actions: StartupAction[];
openingMessage: {
title: string;
message: string;
confirmLabel: string;
skipSpeechOnConfirm: boolean;
} | null;
};