ai-video-fullstack/backend/services/pipecat/pipeline_events.py

"""Event registration for cascade and realtime conversation pipelines."""

from loguru import logger

from pipecat.frames.frames import (
    EndFrame,
    LLMMessagesAppendFrame,
    OutputTransportMessageUrgentFrame,
    TTSSpeakFrame,
)
from pipecat.runner.utils import (
    get_transport_client_id,
    maybe_capture_participant_camera,
)
from pipecat.utils.time import time_now_iso8601


def bind_cascade_pipeline_events(
    *,
    transport,
    worker,
    brain,
    context,
    text_input,
    user_aggregator,
    assistant_aggregator,
    greeting: str,
    vision_enabled: bool,
    vision_state: dict[str, str | None],
) -> None:
    """Connect processors to transport events without owning pipeline assembly."""

    pending_text_inputs: list[str] = []
    greeting_transcript_sent = False
    greeting_timestamp = ""

    async def queue_transcript(role: str, content: str, timestamp: str) -> None:
        if not content:
            return
        await worker.queue_frame(
            OutputTransportMessageUrgentFrame(
                message={
                    "type": "transcript",
                    "role": role,
                    "content": content,
                    "timestamp": timestamp,
                }
            )
        )

    async def append_user_text_to_context(text: str, *, run_llm: bool) -> None:
        await worker.queue_frame(
            LLMMessagesAppendFrame(
                messages=[{"role": "user", "content": text}],
                run_llm=run_llm,
            )
        )

    @user_aggregator.event_handler("on_user_turn_stopped")
    async def on_user_turn_stopped(_aggregator, _strategy, message):
        await queue_transcript("user", message.content, message.timestamp)

    @assistant_aggregator.event_handler("on_assistant_text_start")
    async def on_assistant_text_start(_aggregator, turn_id, timestamp):
        await brain.on_assistant_text_start(turn_id)
        await worker.queue_frame(
            OutputTransportMessageUrgentFrame(
                message={
                    "type": "assistant-text-start",
                    "turn_id": turn_id,
                    "timestamp": timestamp,
                }
            )
        )

    @assistant_aggregator.event_handler("on_assistant_text_delta")
    async def on_assistant_text_delta(_aggregator, turn_id, delta):
        await worker.queue_frame(
            OutputTransportMessageUrgentFrame(
                message={
                    "type": "assistant-text-delta",
                    "turn_id": turn_id,
                    "delta": delta,
                }
            )
        )

    @assistant_aggregator.event_handler("on_assistant_text_end")
    async def on_assistant_text_end(_aggregator, turn_id, content, interrupted):
        await worker.queue_frame(
            OutputTransportMessageUrgentFrame(
                message={
                    "type": "assistant-text-end",
                    "turn_id": turn_id,
                    "content": content,
                    "interrupted": interrupted,
                }
            )
        )
        await brain.on_assistant_text_end(turn_id, content, interrupted)

    @text_input.event_handler("on_text_input")
    async def on_text_input(_processor, text):
        pending_text_inputs.append(text)
        # The transcript must be queued before the interruption is broadcast.
        await queue_transcript("user", text, time_now_iso8601())

    @assistant_aggregator.event_handler("on_interruption_processed")
    async def on_interruption_processed(_aggregator):
        if not pending_text_inputs:
            return
        text = pending_text_inputs.pop(0)
        await append_user_text_to_context(text, run_llm=True)

    @text_input.event_handler("on_text_append")
    async def on_text_append(_processor, text):
        brain.record_user_message(text)
        await queue_transcript("user", text, time_now_iso8601())
        await append_user_text_to_context(text, run_llm=False)

    @text_input.event_handler("on_client_ready")
    async def on_client_ready(_processor):
        nonlocal greeting_transcript_sent
        if greeting and not greeting_transcript_sent:
            greeting_transcript_sent = True
            await queue_transcript(
                "assistant",
                greeting,
                greeting_timestamp or time_now_iso8601(),
            )
        await brain.on_client_ready()

    @transport.event_handler("on_client_connected")
    async def on_client_connected(_transport, _client):
        nonlocal greeting_timestamp
        if vision_enabled:
            try:
                vision_state["client_id"] = get_transport_client_id(
                    _transport,
                    _client,
                )
                await maybe_capture_participant_camera(_transport, _client)
                logger.info(
                    f"视觉理解已接入视频客户端: {vision_state['client_id']}"
                )
            except Exception as exc:  # noqa: BLE001 - media availability is optional
                logger.warning(f"视觉理解摄像头捕获初始化失败: {exc}")
        if greeting:
            # Preserve the actual playback order. The transcript is delivered
            # later on client-ready, but the preview sorts by this timestamp.
            greeting_timestamp = greeting_timestamp or time_now_iso8601()
            if brain.spec.owns_context:
                brain.prepare_greeting_context(greeting, context)
            await worker.queue_frame(
                TTSSpeakFrame(greeting, append_to_context=False)
            )
        await brain.on_connected()

    @transport.event_handler("on_client_disconnected")
    async def on_client_disconnected(_transport, _client):
        logger.info("对端断开,结束管线")
        await worker.queue_frame(EndFrame())


def bind_realtime_pipeline_events(
    *,
    transport,
    worker,
    realtime,
    text_input,
    greeting: str,
) -> None:
    """Connect text and lifecycle events for a realtime model pipeline."""

    async def queue_transcript(role: str, content: str) -> None:
        if not content:
            return
        await worker.queue_frame(
            OutputTransportMessageUrgentFrame(
                message={
                    "type": "transcript",
                    "role": role,
                    "content": content,
                    "timestamp": time_now_iso8601(),
                }
            )
        )

    @text_input.event_handler("on_text_input")
    async def on_text_input(_processor, text):
        await queue_transcript("user", text)
        await realtime.interrupt()
        await realtime.send_text(text, run_immediately=True)

    @text_input.event_handler("on_text_append")
    async def on_text_append(_processor, text):
        await queue_transcript("user", text)
        await realtime.send_text(text, run_immediately=False)

    @transport.event_handler("on_client_connected")
    async def on_client_connected(_transport, _client):
        if greeting:
            await realtime.speak(greeting)

    @transport.event_handler("on_client_disconnected")
    async def on_client_disconnected(_transport, _client):
        logger.info("Realtime 对端断开,结束管线")
        await worker.queue_frame(EndFrame())