wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add prompt opening behavior modes

Browse Source
This commit is contained in:
Xin Wang
2026-08-04 13:29:21 +08:00
parent 78bba090a2
commit 5bf5987fe4
11 changed files with 235 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
backend/schemas.py
View File

@@ -29,6 +29,7 @@ ToolExecutionMode = Literal["immediate", "async"]
ClientToolResponseWaitMode = Literal["timeout", "session"]
DynamicVariableType = Literal["string", "number", "boolean"]
PromptEntryMode = Literal["wait_user", "generate"]
PromptOpeningMode = Literal["interruptible", "playback", "confirmation"]
# 外部应用类型:其 config.apiKey 是该助手私有密钥,读时打码 / 写时哨兵
EXTERNAL_TYPES = {"dify", "fastgpt", "opencode"}
@@ -107,11 +108,11 @@ class OpeningMessageConfig(CamelModel):
title: str = Field(default="重要提示", min_length=1, max_length=120)
message: str = Field(min_length=1, max_length=2000)
confirm_label: str = Field(default="确认", min_length=1, max_length=40)
skip_speech_on_confirm: bool = True
class StartupConfig(CamelModel):
execution_mode: Literal["sequential"] = "sequential"
opening_mode: PromptOpeningMode | None = None
entry_mode: PromptEntryMode | None = None
actions: list[StartupAction] = Field(default_factory=list, max_length=5)
opening_message: OpeningMessageConfig | None = None
@@ -121,11 +122,22 @@ class StartupConfig(CamelModel):
ids = [action.id for action in self.actions]
if len(ids) != len(set(ids)):
raise ValueError("启动 Action 的 id 不能重复")
legacy_confirmation = (
self.opening_mode is None and self.opening_message is not None
)
if self.opening_mode is None:
self.opening_mode = (
"confirmation" if legacy_confirmation else "interruptible"
)
if self.opening_mode == "confirmation" and self.opening_message is None:
raise ValueError("确认后继续必须配置开场确认弹窗")
if self.opening_mode != "confirmation":
self.opening_message = None
if self.entry_mode is None:
# Preserve the behavior introduced before entryMode became an
# explicit setting. Newly saved clients always send the field.
self.entry_mode = (
"generate" if self.opening_message is not None else "wait_user"
"generate" if legacy_confirmation else "wait_user"
)
return self
@@ -191,6 +203,7 @@ class AssistantUpsert(CamelModel):
if self.runtime_mode == "realtime" and (
self.startup.actions
or self.startup.opening_message is not None
or self.startup.opening_mode != "interruptible"
or self.startup.entry_mode != "wait_user"
):
raise ValueError("Prompt Realtime 模式暂不支持启动阶段")

5
backend/services/brains/base.py
View File

@@ -132,6 +132,9 @@ class BaseBrain:
async def on_greeting_finished(self) -> None:
"""Continue startup after the shared greeting has actually played."""
async def on_interruption_processed(self) -> None:
"""Observe that an output interruption reached assistant aggregation."""
def prepare_greeting_context(
self,
greeting: str,
@@ -214,6 +217,8 @@ class Brain(Protocol):
async def on_greeting_finished(self) -> None: ...
async def on_interruption_processed(self) -> None: ...
def prepare_greeting_context(
self,
greeting: str,

54
backend/services/brains/prompt_brain.py
View File

@@ -81,7 +81,7 @@ class PromptBrain(BaseBrain):
async def greeting(self, cfg: AssistantConfig) -> str:
# The built-in opening Message owns the greeting so speech and the
# client dialog can start as one atomic stage.
if self._opening_message() is not None:
if self._opening_mode() == "confirmation":
return ""
return self._render_greeting(cfg)
@@ -155,8 +155,19 @@ class PromptBrain(BaseBrain):
async def on_connected(self, *, greeting_pending: bool = False) -> None:
self._greeting_pending = greeting_pending
should_block_input = (
self._has_opening_stage()
or (
greeting_pending
and self._opening_mode() == "playback"
)
or (
not greeting_pending
and self._entry_mode() == "generate"
)
)
if (
(self._has_opening_stage() or self._entry_mode() == "generate")
should_block_input
and self._runtime is not None
and self._runtime.set_input_enabled is not None
):
@@ -168,6 +179,16 @@ class PromptBrain(BaseBrain):
self._greeting_pending = False
await self._enter_prompt_if_ready()
async def on_interruption_processed(self) -> None:
if (
self._greeting_pending
and self._opening_mode() == "interruptible"
):
# The user's input now owns the first Agent turn. Do not also queue
# an empty-context entry reply when the interrupted greeting stops.
self._entry_dispatched = True
logger.debug("Prompt 可打断开场白已被用户输入接管")
async def on_client_ready(self) -> None:
if self._output is not None:
await self._output.mark_client_ready()
@@ -180,7 +201,11 @@ class PromptBrain(BaseBrain):
runtime = self._runtime
if runtime is None:
raise RuntimeError("PromptBrain 尚未初始化")
opening_message = self._opening_message()
opening_message = (
self._opening_message()
if self._opening_mode() == "confirmation"
else None
)
opening_actions = self._startup_actions("opening")
speech = (
self._render_greeting(self._cfg).strip()
@@ -238,6 +263,19 @@ class PromptBrain(BaseBrain):
value = startup.get("opening_message", startup.get("openingMessage"))
return value if isinstance(value, dict) else None
def _opening_mode(self) -> str:
startup = self._cfg.startup if isinstance(self._cfg.startup, dict) else {}
value = str(startup.get("opening_mode", startup.get("openingMode")) or "")
if value in {"interruptible", "playback", "confirmation"}:
return value
# Before openingMode existed, an opening message meant confirmation;
# ordinary Prompt greetings were interruptible.
return (
"confirmation"
if self._opening_message() is not None
else "interruptible"
)
def _entry_mode(self) -> str:
startup = self._cfg.startup if isinstance(self._cfg.startup, dict) else {}
value = str(startup.get("entry_mode", startup.get("entryMode")) or "")
@@ -245,7 +283,7 @@ class PromptBrain(BaseBrain):
return value
# Raw runtime configs saved before entryMode existed generated a first
# reply after their opening confirmation. Keep that legacy behavior.
return "generate" if self._opening_message() is not None else "wait_user"
return "generate" if self._opening_mode() == "confirmation" else "wait_user"
async def _enter_prompt_if_ready(self) -> None:
if (
@@ -269,7 +307,7 @@ class PromptBrain(BaseBrain):
self._opening_input_blocked = False
def _has_opening_stage(self) -> bool:
return self._opening_message() is not None or bool(
return self._opening_mode() == "confirmation" or bool(
self._startup_actions("opening")
)
@@ -322,12 +360,6 @@ class PromptBrain(BaseBrain):
).strip(),
),
completion_policy=MESSAGE_CONFIRMATION,
skip_speech_on_confirmation=bool(
config.get(
"skip_speech_on_confirm",
config.get("skipSpeechOnConfirm", True),
)
),
)
async def _publish_opening_outcome(

17
backend/services/message_stage.py
View File

@@ -38,7 +38,6 @@ class MessageStageSpec:
speech: str = ""
display: MessageDisplaySpec | None = None
completion_policy: MessageCompletionPolicy = MESSAGE_PLAYBACK
skip_speech_on_confirmation: bool = True
@dataclass(frozen=True)
@@ -55,7 +54,7 @@ class MessageStageRunner:
"""Run one atomic user-visible message stage.
Speech is queued before the client message is dispatched. A confirmation
may interrupt the remaining audio or keep playback as the final gate. A
stage completes when the user confirms, even if audio is still playing. A
speech-only stage completes at the real transport playback boundary.
"""
@@ -118,14 +117,11 @@ class MessageStageRunner:
return result
action = result.action
# A confirmation may either interrupt the remaining speech or
# acknowledge the dialog while keeping playback as the final gate.
# Confirmation is the gate. It deliberately does not wait for the
# audio completion future, so the user can continue immediately.
if (
playback_completion is not None
and (
not require_confirmation
or not spec.skip_speech_on_confirmation
)
and not require_confirmation
):
await playback_completion
@@ -191,10 +187,7 @@ class MessageStageRunner:
response_wait_mode=(
"session" if require_confirmation else "timeout"
),
interrupt_on_result=(
require_confirmation
and spec.skip_speech_on_confirmation
),
interrupt_on_result=require_confirmation,
)
except ClientToolError as exc:
return MessageStageResult(

3
backend/services/pipecat/pipeline_events.py
View File

@@ -161,6 +161,9 @@ def bind_cascade_pipeline_events(
@assistant_aggregator.event_handler("on_interruption_processed")
async def on_interruption_processed(_aggregator):
brain_handler = getattr(brain, "on_interruption_processed", None)
if callable(brain_handler):
await brain_handler()
if client_tools is not None:
client_tools.on_interruption_processed()
if not pending_user_inputs:

86
backend/tests/test_brains.py
View File

@@ -366,6 +366,7 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
tools=[tool],
startup={
"execution_mode": "sequential",
"opening_mode": "confirmation",
"entry_mode": "generate",
"opening_message": {
"title": "重要提示",
@@ -485,6 +486,7 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
type="prompt",
greeting="请先确认",
startup={
"opening_mode": "confirmation",
"entry_mode": "wait_user",
"opening_message": {
"title": "重要提示",
@@ -527,7 +529,10 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
cfg = AssistantConfig(
type="prompt",
greeting="固定开场白",
startup={"entry_mode": "generate"},
startup={
"opening_mode": "playback",
"entry_mode": "generate",
},
)
brain = build_brain(cfg)
queued = []
@@ -561,45 +566,22 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
1,
)
async def test_opening_confirmation_can_keep_speech_playing_before_entry(self):
async def test_interruptible_opening_user_input_suppresses_entry_reply(self):
cfg = AssistantConfig(
type="prompt",
greeting="必须完整播放的开场白",
greeting="可以打断的开场白",
startup={
"opening_mode": "interruptible",
"entry_mode": "generate",
"opening_message": {
"title": "重要提示",
"message": "请确认已阅读。",
"skip_speech_on_confirm": False,
},
},
)
brain = build_brain(cfg)
queued = []
input_states = []
message_displayed = asyncio.Event()
class TrackedCallEnd(FakeCallEnd):
def __init__(self):
super().__init__()
self.playback_completion = None
def track_speech(self):
self.tracked_speeches += 1
self.playback_completion = asyncio.get_running_loop().create_future()
return self.playback_completion
class FakeClientTools:
async def call(self, *_args, **options):
self.options = options
message_displayed.set()
return {"status": "ok", "data": {"action": "confirmed"}}
async def queue_frame(frame):
queued.append(frame)
call_end = TrackedCallEnd()
client_tools = FakeClientTools()
await brain.setup(
cfg,
BrainRuntime(
@@ -608,24 +590,52 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
queue_frame=queue_frame,
set_system_prompt=lambda _prompt: None,
set_tools=lambda _tools: None,
call_end=call_end,
client_tools=client_tools,
call_end=FakeCallEnd(),
set_input_enabled=input_states.append,
),
)
await brain.on_connected(greeting_pending=False)
opening_task = asyncio.create_task(brain.on_client_ready())
await message_displayed.wait()
await asyncio.sleep(0)
await brain.on_connected(greeting_pending=True)
self.assertEqual(input_states, [])
await brain.on_interruption_processed()
await brain.on_greeting_finished()
self.assertFalse(client_tools.options["interrupt_on_result"])
self.assertFalse(opening_task.done())
self.assertFalse(any(isinstance(frame, LLMRunFrame) for frame in queued))
call_end.playback_completion.set_result(None)
await opening_task
self.assertEqual(input_states, [False, True])
async def test_interruptible_opening_natural_finish_runs_entry_behavior(self):
cfg = AssistantConfig(
type="prompt",
greeting="可以打断的开场白",
startup={
"opening_mode": "interruptible",
"entry_mode": "generate",
},
)
brain = build_brain(cfg)
queued = []
input_states = []
async def queue_frame(frame):
queued.append(frame)
await brain.setup(
cfg,
BrainRuntime(
context=LLMContext(messages=[]),
llm=FakeLLM(),
queue_frame=queue_frame,
set_system_prompt=lambda _prompt: None,
set_tools=lambda _tools: None,
call_end=FakeCallEnd(),
set_input_enabled=input_states.append,
),
)
await brain.on_connected(greeting_pending=True)
await brain.on_greeting_finished()
self.assertEqual(input_states, [])
self.assertEqual(
sum(isinstance(frame, LLMRunFrame) for frame in queued),
1,

5
backend/tests/test_pipeline_events.py
View File

@@ -52,6 +52,7 @@ class _Brain:
self.prepared_greeting = ""
self.greeting_pending = False
self.greeting_finished = 0
self.interruptions_processed = 0
def prepare_greeting_context(self, greeting, _context):
self.prepared_greeting = greeting
@@ -62,6 +63,9 @@ class _Brain:
async def on_greeting_finished(self):
self.greeting_finished += 1
async def on_interruption_processed(self):
self.interruptions_processed += 1
async def on_client_ready(self):
for content, timestamp in (
("Message 节点播报", "2026-07-14T10:00:00.200+00:00"),
@@ -111,6 +115,7 @@ class PipelineEventTest(unittest.IsolatedAsyncioTestCase):
)
self.assertEqual(acknowledgements, [True])
self.assertEqual(brain.interruptions_processed, 1)
async def test_greeting_keeps_playback_timestamp_until_client_ready(self):
transport = _EventSource()

2
backend/tests/test_startup_actions.py
View File

@@ -109,7 +109,7 @@ class StartupActionValidationTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(body.startup.actions, [])
self.assertEqual(body.startup.opening_message.confirm_label, "确认")
self.assertTrue(body.startup.opening_message.skip_speech_on_confirm)
self.assertEqual(body.startup.opening_mode, "confirmation")
self.assertEqual(body.startup.entry_mode, "generate")
async def test_startup_tool_does_not_need_conversation_binding(self):