162a3d8bec
- Update WorkflowBrain to handle greeting playback more effectively, ensuring that the initial greeting completes before transitioning to the first node. - Introduce new methods for managing greeting states and conditions, enhancing the interaction flow for user turns. - Refactor WorkflowLLMRouter to improve routing logic and ensure proper handling of conditional paths. - Enhance tests to verify the correct behavior of greeting management and routing under various scenarios, including waiting for audio playback to finish. - Update frontend components to reflect changes in edge handling and improve user experience in workflow configurations.
241 lines
8.6 KiB
Python
241 lines
8.6 KiB
Python
"""Event registration for cascade and realtime conversation pipelines."""
|
|
|
|
from loguru import logger
|
|
|
|
from pipecat.frames.frames import (
|
|
BotStartedSpeakingFrame,
|
|
BotStoppedSpeakingFrame,
|
|
EndFrame,
|
|
LLMMessagesAppendFrame,
|
|
OutputTransportMessageUrgentFrame,
|
|
TTSSpeakFrame,
|
|
)
|
|
from pipecat.runner.utils import (
|
|
get_transport_client_id,
|
|
maybe_capture_participant_camera,
|
|
)
|
|
from pipecat.utils.time import time_now_iso8601
|
|
|
|
|
|
def bind_cascade_pipeline_events(
|
|
*,
|
|
transport,
|
|
worker,
|
|
brain,
|
|
context,
|
|
text_input,
|
|
user_aggregator,
|
|
assistant_aggregator,
|
|
greeting: str,
|
|
vision_enabled: bool,
|
|
vision_state: dict[str, str | None],
|
|
) -> None:
|
|
"""Connect processors to transport events without owning pipeline assembly."""
|
|
|
|
pending_text_inputs: list[str] = []
|
|
greeting_transcript_sent = False
|
|
greeting_timestamp = ""
|
|
greeting_playback_pending = False
|
|
greeting_playback_started = False
|
|
|
|
# FlowManager already observes downstream frames for its own actions. Add
|
|
# to that filter instead of replacing it, then use the real transport
|
|
# playback boundary to release Workflow startup.
|
|
worker.add_reached_downstream_filter(
|
|
(BotStartedSpeakingFrame, BotStoppedSpeakingFrame)
|
|
)
|
|
|
|
@worker.event_handler("on_frame_reached_downstream")
|
|
async def on_frame_reached_downstream(_worker, frame):
|
|
nonlocal greeting_playback_pending, greeting_playback_started
|
|
if not greeting_playback_pending:
|
|
return
|
|
if isinstance(frame, BotStartedSpeakingFrame):
|
|
greeting_playback_started = True
|
|
return
|
|
if isinstance(frame, BotStoppedSpeakingFrame) and greeting_playback_started:
|
|
greeting_playback_pending = False
|
|
greeting_playback_started = False
|
|
await brain.on_greeting_finished()
|
|
|
|
async def queue_transcript(role: str, content: str, timestamp: str) -> None:
|
|
if not content:
|
|
return
|
|
await worker.queue_frame(
|
|
OutputTransportMessageUrgentFrame(
|
|
message={
|
|
"type": "transcript",
|
|
"role": role,
|
|
"content": content,
|
|
"timestamp": timestamp,
|
|
}
|
|
)
|
|
)
|
|
|
|
async def append_user_text_to_context(text: str, *, run_llm: bool) -> None:
|
|
await worker.queue_frame(
|
|
LLMMessagesAppendFrame(
|
|
messages=[{"role": "user", "content": text}],
|
|
run_llm=run_llm,
|
|
)
|
|
)
|
|
|
|
@user_aggregator.event_handler("on_user_turn_stopped")
|
|
async def on_user_turn_stopped(_aggregator, _strategy, message):
|
|
await queue_transcript("user", message.content, message.timestamp)
|
|
|
|
@assistant_aggregator.event_handler("on_assistant_text_start")
|
|
async def on_assistant_text_start(_aggregator, turn_id, timestamp):
|
|
await brain.on_assistant_text_start(turn_id)
|
|
await worker.queue_frame(
|
|
OutputTransportMessageUrgentFrame(
|
|
message={
|
|
"type": "assistant-text-start",
|
|
"turn_id": turn_id,
|
|
"timestamp": timestamp,
|
|
}
|
|
)
|
|
)
|
|
|
|
@assistant_aggregator.event_handler("on_assistant_text_delta")
|
|
async def on_assistant_text_delta(_aggregator, turn_id, delta):
|
|
await worker.queue_frame(
|
|
OutputTransportMessageUrgentFrame(
|
|
message={
|
|
"type": "assistant-text-delta",
|
|
"turn_id": turn_id,
|
|
"delta": delta,
|
|
}
|
|
)
|
|
)
|
|
|
|
@assistant_aggregator.event_handler("on_assistant_text_end")
|
|
async def on_assistant_text_end(_aggregator, turn_id, content, interrupted):
|
|
await worker.queue_frame(
|
|
OutputTransportMessageUrgentFrame(
|
|
message={
|
|
"type": "assistant-text-end",
|
|
"turn_id": turn_id,
|
|
"content": content,
|
|
"interrupted": interrupted,
|
|
}
|
|
)
|
|
)
|
|
await brain.on_assistant_text_end(turn_id, content, interrupted)
|
|
|
|
@text_input.event_handler("on_text_input")
|
|
async def on_text_input(_processor, text):
|
|
pending_text_inputs.append(text)
|
|
# The transcript must be queued before the interruption is broadcast.
|
|
await queue_transcript("user", text, time_now_iso8601())
|
|
|
|
@assistant_aggregator.event_handler("on_interruption_processed")
|
|
async def on_interruption_processed(_aggregator):
|
|
if not pending_text_inputs:
|
|
return
|
|
text = pending_text_inputs.pop(0)
|
|
await append_user_text_to_context(text, run_llm=True)
|
|
|
|
@text_input.event_handler("on_text_append")
|
|
async def on_text_append(_processor, text):
|
|
brain.record_user_message(text)
|
|
await queue_transcript("user", text, time_now_iso8601())
|
|
await append_user_text_to_context(text, run_llm=False)
|
|
|
|
@text_input.event_handler("on_client_ready")
|
|
async def on_client_ready(_processor):
|
|
nonlocal greeting_transcript_sent
|
|
if greeting and not greeting_transcript_sent:
|
|
greeting_transcript_sent = True
|
|
await queue_transcript(
|
|
"assistant",
|
|
greeting,
|
|
greeting_timestamp or time_now_iso8601(),
|
|
)
|
|
await brain.on_client_ready()
|
|
|
|
@transport.event_handler("on_client_connected")
|
|
async def on_client_connected(_transport, _client):
|
|
nonlocal greeting_timestamp, greeting_playback_pending
|
|
if vision_enabled:
|
|
try:
|
|
vision_state["client_id"] = get_transport_client_id(
|
|
_transport,
|
|
_client,
|
|
)
|
|
await maybe_capture_participant_camera(_transport, _client)
|
|
logger.info(
|
|
f"视觉理解已接入视频客户端: {vision_state['client_id']}"
|
|
)
|
|
except Exception as exc: # noqa: BLE001 - media availability is optional
|
|
logger.warning(f"视觉理解摄像头捕获初始化失败: {exc}")
|
|
has_greeting = bool(greeting.strip())
|
|
if has_greeting:
|
|
# Preserve the actual playback order. The transcript is delivered
|
|
# later on client-ready, but the preview sorts by this timestamp.
|
|
greeting_timestamp = greeting_timestamp or time_now_iso8601()
|
|
if brain.spec.owns_context:
|
|
brain.prepare_greeting_context(greeting, context)
|
|
greeting_playback_pending = True
|
|
|
|
# Initialize the Workflow before the greeting is queued so a very
|
|
# short TTS response cannot finish before the brain arms its startup
|
|
# gate. Other brain types simply ignore greeting_pending.
|
|
await brain.on_connected(greeting_pending=has_greeting)
|
|
|
|
if has_greeting:
|
|
await worker.queue_frame(
|
|
TTSSpeakFrame(greeting, append_to_context=False)
|
|
)
|
|
|
|
@transport.event_handler("on_client_disconnected")
|
|
async def on_client_disconnected(_transport, _client):
|
|
logger.info("对端断开,结束管线")
|
|
await worker.queue_frame(EndFrame())
|
|
|
|
|
|
def bind_realtime_pipeline_events(
|
|
*,
|
|
transport,
|
|
worker,
|
|
realtime,
|
|
text_input,
|
|
greeting: str,
|
|
) -> None:
|
|
"""Connect text and lifecycle events for a realtime model pipeline."""
|
|
|
|
async def queue_transcript(role: str, content: str) -> None:
|
|
if not content:
|
|
return
|
|
await worker.queue_frame(
|
|
OutputTransportMessageUrgentFrame(
|
|
message={
|
|
"type": "transcript",
|
|
"role": role,
|
|
"content": content,
|
|
"timestamp": time_now_iso8601(),
|
|
}
|
|
)
|
|
)
|
|
|
|
@text_input.event_handler("on_text_input")
|
|
async def on_text_input(_processor, text):
|
|
await queue_transcript("user", text)
|
|
await realtime.interrupt()
|
|
await realtime.send_text(text, run_immediately=True)
|
|
|
|
@text_input.event_handler("on_text_append")
|
|
async def on_text_append(_processor, text):
|
|
await queue_transcript("user", text)
|
|
await realtime.send_text(text, run_immediately=False)
|
|
|
|
@transport.event_handler("on_client_connected")
|
|
async def on_client_connected(_transport, _client):
|
|
if greeting:
|
|
await realtime.speak(greeting)
|
|
|
|
@transport.event_handler("on_client_disconnected")
|
|
async def on_client_disconnected(_transport, _client):
|
|
logger.info("Realtime 对端断开,结束管线")
|
|
await worker.queue_frame(EndFrame())
|