Merge pull request #3433 from pipecat-ai/mb/port-realtime-examples-transcript-events
Update examples to use transcription events from context aggregators
This commit is contained in:
@@ -15,14 +15,17 @@ from loguru import logger
|
|||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import LLMRunFrame, LLMSetToolsFrame, TranscriptionMessage
|
from pipecat.frames.frames import LLMRunFrame, LLMSetToolsFrame
|
||||||
from pipecat.observers.loggers.transcription_log_observer import TranscriptionLogObserver
|
from pipecat.observers.loggers.transcription_log_observer import TranscriptionLogObserver
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.llm_context import LLMContext
|
from pipecat.processors.aggregators.llm_context import LLMContext
|
||||||
from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair
|
from pipecat.processors.aggregators.llm_response_universal import (
|
||||||
from pipecat.processors.transcript_processor import TranscriptProcessor
|
AssistantTurnStoppedMessage,
|
||||||
|
LLMContextAggregatorPair,
|
||||||
|
UserTurnStoppedMessage,
|
||||||
|
)
|
||||||
from pipecat.runner.types import RunnerArguments
|
from pipecat.runner.types import RunnerArguments
|
||||||
from pipecat.runner.utils import create_transport
|
from pipecat.runner.utils import create_transport
|
||||||
from pipecat.services.llm_service import FunctionCallParams
|
from pipecat.services.llm_service import FunctionCallParams
|
||||||
@@ -177,8 +180,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re
|
|||||||
llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation)
|
llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation)
|
||||||
llm.register_function("get_news", get_news)
|
llm.register_function("get_news", get_news)
|
||||||
|
|
||||||
transcript = TranscriptProcessor()
|
|
||||||
|
|
||||||
# Create a standard OpenAI LLM context object using the normal messages format. The
|
# Create a standard OpenAI LLM context object using the normal messages format. The
|
||||||
# OpenAIRealtimeLLMService will convert this internally to messages that the
|
# OpenAIRealtimeLLMService will convert this internally to messages that the
|
||||||
# openai WebSocket API can understand.
|
# openai WebSocket API can understand.
|
||||||
@@ -189,15 +190,16 @@ Remember, your responses should be short. Just one or two sentences, usually. Re
|
|||||||
|
|
||||||
context_aggregator = LLMContextAggregatorPair(context)
|
context_aggregator = LLMContextAggregatorPair(context)
|
||||||
|
|
||||||
|
user_aggregator = context_aggregator.user()
|
||||||
|
assistant_aggregator = context_aggregator.assistant()
|
||||||
|
|
||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
context_aggregator.user(),
|
user_aggregator,
|
||||||
transcript.user(), # LLM pushes TranscriptionFrames upstream
|
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
transport.output(), # Transport bot output
|
transport.output(), # Transport bot output
|
||||||
transcript.assistant(), # After the transcript output, to time with the audio output
|
assistant_aggregator,
|
||||||
context_aggregator.assistant(),
|
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -238,14 +240,18 @@ Remember, your responses should be short. Just one or two sentences, usually. Re
|
|||||||
logger.info(f"Client disconnected")
|
logger.info(f"Client disconnected")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
# Register event handler for transcript updates
|
# Log transcript updates
|
||||||
@transcript.event_handler("on_transcript_update")
|
@user_aggregator.event_handler("on_user_turn_stopped")
|
||||||
async def on_transcript_update(processor, frame):
|
async def on_user_turn_stopped(aggregator, strategy, message: UserTurnStoppedMessage):
|
||||||
for msg in frame.messages:
|
timestamp = f"[{message.timestamp}] " if message.timestamp else ""
|
||||||
if isinstance(msg, TranscriptionMessage):
|
line = f"{timestamp}user: {message.content}"
|
||||||
timestamp = f"[{msg.timestamp}] " if msg.timestamp else ""
|
logger.info(f"Transcript: {line}")
|
||||||
line = f"{timestamp}{msg.role}: {msg.content}"
|
|
||||||
logger.info(f"Transcript: {line}")
|
@assistant_aggregator.event_handler("on_assistant_turn_stopped")
|
||||||
|
async def on_assistant_turn_stopped(aggregator, message: AssistantTurnStoppedMessage):
|
||||||
|
timestamp = f"[{message.timestamp}] " if message.timestamp else ""
|
||||||
|
line = f"{timestamp}assistant: {message.content}"
|
||||||
|
logger.info(f"Transcript: {line}")
|
||||||
|
|
||||||
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
||||||
|
|
||||||
|
|||||||
@@ -14,12 +14,11 @@ from loguru import logger
|
|||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import LLMRunFrame, TranscriptionMessage
|
from pipecat.frames.frames import LLMRunFrame
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.processors.transcript_processor import TranscriptProcessor
|
|
||||||
from pipecat.runner.types import RunnerArguments
|
from pipecat.runner.types import RunnerArguments
|
||||||
from pipecat.runner.utils import create_transport
|
from pipecat.runner.utils import create_transport
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
@@ -157,8 +156,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re
|
|||||||
llm.register_function("get_current_weather", fetch_weather_from_api)
|
llm.register_function("get_current_weather", fetch_weather_from_api)
|
||||||
llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation)
|
llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation)
|
||||||
|
|
||||||
transcript = TranscriptProcessor()
|
|
||||||
|
|
||||||
# Create a standard OpenAI LLM context object using the normal messages format. The
|
# Create a standard OpenAI LLM context object using the normal messages format. The
|
||||||
# OpenAIRealtimeBetaLLMService will convert this internally to messages that the
|
# OpenAIRealtimeBetaLLMService will convert this internally to messages that the
|
||||||
# openai WebSocket API can understand.
|
# openai WebSocket API can understand.
|
||||||
@@ -175,9 +172,7 @@ Remember, your responses should be short. Just one or two sentences, usually. Re
|
|||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
transcript.user(), # Placed after the LLM, as LLM pushes TranscriptionFrames downstream
|
|
||||||
transport.output(), # Transport bot output
|
transport.output(), # Transport bot output
|
||||||
transcript.assistant(), # After the transcript output, to time with the audio output
|
|
||||||
context_aggregator.assistant(),
|
context_aggregator.assistant(),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
@@ -202,15 +197,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re
|
|||||||
logger.info(f"Client disconnected")
|
logger.info(f"Client disconnected")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
# Register event handler for transcript updates
|
|
||||||
@transcript.event_handler("on_transcript_update")
|
|
||||||
async def on_transcript_update(processor, frame):
|
|
||||||
for msg in frame.messages:
|
|
||||||
if isinstance(msg, TranscriptionMessage):
|
|
||||||
timestamp = f"[{msg.timestamp}] " if msg.timestamp else ""
|
|
||||||
line = f"{timestamp}{msg.role}: {msg.content}"
|
|
||||||
logger.info(f"Transcript: {line}")
|
|
||||||
|
|
||||||
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|||||||
@@ -14,13 +14,12 @@ from loguru import logger
|
|||||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import LLMRunFrame, TranscriptionMessage
|
from pipecat.frames.frames import LLMRunFrame
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.llm_context import LLMContext
|
from pipecat.processors.aggregators.llm_context import LLMContext
|
||||||
from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair
|
from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair
|
||||||
from pipecat.processors.transcript_processor import TranscriptProcessor
|
|
||||||
from pipecat.runner.types import RunnerArguments
|
from pipecat.runner.types import RunnerArguments
|
||||||
from pipecat.runner.utils import create_transport
|
from pipecat.runner.utils import create_transport
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
@@ -164,8 +163,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re
|
|||||||
llm.register_function("get_current_weather", fetch_weather_from_api)
|
llm.register_function("get_current_weather", fetch_weather_from_api)
|
||||||
llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation)
|
llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation)
|
||||||
|
|
||||||
transcript = TranscriptProcessor()
|
|
||||||
|
|
||||||
# Create a standard OpenAI LLM context object using the normal messages format. The
|
# Create a standard OpenAI LLM context object using the normal messages format. The
|
||||||
# OpenAIRealtimeLLMService will convert this internally to messages that the
|
# OpenAIRealtimeLLMService will convert this internally to messages that the
|
||||||
# openai WebSocket API can understand.
|
# openai WebSocket API can understand.
|
||||||
@@ -180,11 +177,9 @@ Remember, your responses should be short. Just one or two sentences, usually. Re
|
|||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
context_aggregator.user(),
|
context_aggregator.user(),
|
||||||
transcript.user(), # LLM pushes TranscriptionFrames upstream
|
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
transport.output(), # Transport bot output
|
transport.output(), # Transport bot output
|
||||||
transcript.assistant(), # After the transcript output, to time with the audio output
|
|
||||||
context_aggregator.assistant(),
|
context_aggregator.assistant(),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
@@ -209,15 +204,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re
|
|||||||
logger.info(f"Client disconnected")
|
logger.info(f"Client disconnected")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
# Register event handler for transcript updates
|
|
||||||
@transcript.event_handler("on_transcript_update")
|
|
||||||
async def on_transcript_update(processor, frame):
|
|
||||||
for msg in frame.messages:
|
|
||||||
if isinstance(msg, TranscriptionMessage):
|
|
||||||
timestamp = f"[{msg.timestamp}] " if msg.timestamp else ""
|
|
||||||
line = f"{timestamp}{msg.role}: {msg.content}"
|
|
||||||
logger.info(f"Transcript: {line}")
|
|
||||||
|
|
||||||
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|||||||
@@ -1,209 +0,0 @@
|
|||||||
#
|
|
||||||
# Copyright (c) 2024-2026, Daily
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
|
||||||
#
|
|
||||||
|
|
||||||
import os
|
|
||||||
from typing import List, Optional
|
|
||||||
|
|
||||||
from dotenv import load_dotenv
|
|
||||||
from loguru import logger
|
|
||||||
|
|
||||||
from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
|
|
||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
|
||||||
from pipecat.audio.vad.vad_analyzer import VADParams
|
|
||||||
from pipecat.frames.frames import LLMRunFrame, TranscriptionMessage, TranscriptionUpdateFrame
|
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
|
||||||
from pipecat.processors.aggregators.llm_context import LLMContext
|
|
||||||
from pipecat.processors.aggregators.llm_response_universal import (
|
|
||||||
LLMContextAggregatorPair,
|
|
||||||
LLMUserAggregatorParams,
|
|
||||||
)
|
|
||||||
from pipecat.processors.transcript_processor import TranscriptProcessor
|
|
||||||
from pipecat.runner.types import RunnerArguments
|
|
||||||
from pipecat.runner.utils import create_transport
|
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
|
||||||
from pipecat.services.deepgram.stt import DeepgramSTTService
|
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
|
||||||
from pipecat.transports.base_transport import BaseTransport, TransportParams
|
|
||||||
from pipecat.transports.daily.transport import DailyParams
|
|
||||||
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
|
|
||||||
from pipecat.turns.user_stop import TurnAnalyzerUserTurnStopStrategy
|
|
||||||
from pipecat.turns.user_turn_strategies import UserTurnStrategies
|
|
||||||
|
|
||||||
load_dotenv(override=True)
|
|
||||||
|
|
||||||
|
|
||||||
class TranscriptHandler:
|
|
||||||
"""Handles real-time transcript processing and output.
|
|
||||||
|
|
||||||
Maintains a list of conversation messages and outputs them either to a log
|
|
||||||
or to a file as they are received. Each message includes its timestamp and role.
|
|
||||||
|
|
||||||
Attributes:
|
|
||||||
messages: List of all processed transcript messages
|
|
||||||
output_file: Optional path to file where transcript is saved. If None, outputs to log only.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, output_file: Optional[str] = None):
|
|
||||||
"""Initialize handler with optional file output.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
output_file: Path to output file. If None, outputs to log only.
|
|
||||||
"""
|
|
||||||
self.messages: List[TranscriptionMessage] = []
|
|
||||||
self.output_file: Optional[str] = output_file
|
|
||||||
logger.debug(
|
|
||||||
f"TranscriptHandler initialized {'with output_file=' + output_file if output_file else 'with log output only'}"
|
|
||||||
)
|
|
||||||
|
|
||||||
async def save_message(self, message: TranscriptionMessage):
|
|
||||||
"""Save a single transcript message.
|
|
||||||
|
|
||||||
Outputs the message to the log and optionally to a file.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
message: The message to save
|
|
||||||
"""
|
|
||||||
timestamp = f"[{message.timestamp}] " if message.timestamp else ""
|
|
||||||
line = f"{timestamp}{message.role}: {message.content}"
|
|
||||||
|
|
||||||
# Always log the message
|
|
||||||
logger.info(f"Transcript: {line}")
|
|
||||||
|
|
||||||
# Optionally write to file
|
|
||||||
if self.output_file:
|
|
||||||
try:
|
|
||||||
with open(self.output_file, "a", encoding="utf-8") as f:
|
|
||||||
f.write(line + "\n")
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error saving transcript message to file: {e}")
|
|
||||||
|
|
||||||
async def on_transcript_update(
|
|
||||||
self, processor: TranscriptProcessor, frame: TranscriptionUpdateFrame
|
|
||||||
):
|
|
||||||
"""Handle new transcript messages.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
processor: The TranscriptProcessor that emitted the update
|
|
||||||
frame: TranscriptionUpdateFrame containing new messages
|
|
||||||
"""
|
|
||||||
logger.debug(f"Received transcript update with {len(frame.messages)} new messages")
|
|
||||||
|
|
||||||
for msg in frame.messages:
|
|
||||||
self.messages.append(msg)
|
|
||||||
await self.save_message(msg)
|
|
||||||
|
|
||||||
|
|
||||||
# We store functions so objects (e.g. SileroVADAnalyzer) don't get
|
|
||||||
# instantiated. The function will be called when the desired transport gets
|
|
||||||
# selected.
|
|
||||||
transport_params = {
|
|
||||||
"daily": lambda: DailyParams(
|
|
||||||
audio_in_enabled=True,
|
|
||||||
audio_out_enabled=True,
|
|
||||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
|
|
||||||
),
|
|
||||||
"twilio": lambda: FastAPIWebsocketParams(
|
|
||||||
audio_in_enabled=True,
|
|
||||||
audio_out_enabled=True,
|
|
||||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
|
|
||||||
),
|
|
||||||
"webrtc": lambda: TransportParams(
|
|
||||||
audio_in_enabled=True,
|
|
||||||
audio_out_enabled=True,
|
|
||||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
|
||||||
logger.info(f"Starting bot")
|
|
||||||
|
|
||||||
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
|
|
||||||
|
|
||||||
tts = CartesiaTTSService(
|
|
||||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
|
||||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
|
||||||
)
|
|
||||||
|
|
||||||
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
|
|
||||||
|
|
||||||
messages = [
|
|
||||||
{
|
|
||||||
"role": "system",
|
|
||||||
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative, helpful, and brief way. Say hello.",
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
context = LLMContext(messages)
|
|
||||||
context_aggregator = LLMContextAggregatorPair(
|
|
||||||
context,
|
|
||||||
user_params=LLMUserAggregatorParams(
|
|
||||||
user_turn_strategies=UserTurnStrategies(
|
|
||||||
stop=[TurnAnalyzerUserTurnStopStrategy(turn_analyzer=LocalSmartTurnAnalyzerV3())]
|
|
||||||
),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
# Create transcript processor and handler
|
|
||||||
transcript = TranscriptProcessor()
|
|
||||||
transcript_handler = TranscriptHandler() # Output to log only
|
|
||||||
# transcript_handler = TranscriptHandler(output_file="transcript.txt") # Output to file and log
|
|
||||||
|
|
||||||
pipeline = Pipeline(
|
|
||||||
[
|
|
||||||
transport.input(), # Transport user input
|
|
||||||
stt, # STT
|
|
||||||
transcript.user(), # User transcripts
|
|
||||||
context_aggregator.user(), # User responses
|
|
||||||
llm, # LLM
|
|
||||||
tts, # TTS
|
|
||||||
transport.output(), # Transport bot output
|
|
||||||
transcript.assistant(), # Assistant transcripts
|
|
||||||
context_aggregator.assistant(), # Assistant spoken responses
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
task = PipelineTask(
|
|
||||||
pipeline,
|
|
||||||
params=PipelineParams(
|
|
||||||
enable_metrics=True,
|
|
||||||
enable_usage_metrics=True,
|
|
||||||
),
|
|
||||||
idle_timeout_secs=runner_args.pipeline_idle_timeout_secs,
|
|
||||||
)
|
|
||||||
|
|
||||||
@transport.event_handler("on_client_connected")
|
|
||||||
async def on_client_connected(transport, client):
|
|
||||||
logger.info(f"Client connected")
|
|
||||||
# Start conversation - empty prompt to let LLM follow system instructions
|
|
||||||
await task.queue_frames([LLMRunFrame()])
|
|
||||||
|
|
||||||
# Register event handler for transcript updates
|
|
||||||
@transcript.event_handler("on_transcript_update")
|
|
||||||
async def on_transcript_update(processor, frame):
|
|
||||||
await transcript_handler.on_transcript_update(processor, frame)
|
|
||||||
|
|
||||||
@transport.event_handler("on_client_disconnected")
|
|
||||||
async def on_client_disconnected(transport, client):
|
|
||||||
logger.info(f"Client disconnected")
|
|
||||||
await task.cancel()
|
|
||||||
|
|
||||||
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
|
||||||
await runner.run(task)
|
|
||||||
|
|
||||||
|
|
||||||
async def bot(runner_args: RunnerArguments):
|
|
||||||
"""Main bot entry point compatible with Pipecat Cloud."""
|
|
||||||
transport = await create_transport(runner_args, transport_params)
|
|
||||||
await run_bot(transport, runner_args)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
from pipecat.runner.run import main
|
|
||||||
|
|
||||||
main()
|
|
||||||
@@ -21,7 +21,11 @@ from pipecat.pipeline.pipeline import Pipeline
|
|||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.llm_context import LLMContext
|
from pipecat.processors.aggregators.llm_context import LLMContext
|
||||||
from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair
|
from pipecat.processors.aggregators.llm_response_universal import (
|
||||||
|
AssistantTurnStoppedMessage,
|
||||||
|
LLMContextAggregatorPair,
|
||||||
|
UserTurnStoppedMessage,
|
||||||
|
)
|
||||||
from pipecat.runner.types import RunnerArguments
|
from pipecat.runner.types import RunnerArguments
|
||||||
from pipecat.runner.utils import create_transport
|
from pipecat.runner.utils import create_transport
|
||||||
from pipecat.services.aws.nova_sonic.llm import AWSNovaSonicLLMService
|
from pipecat.services.aws.nova_sonic.llm import AWSNovaSonicLLMService
|
||||||
@@ -154,14 +158,17 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
|||||||
)
|
)
|
||||||
context_aggregator = LLMContextAggregatorPair(context)
|
context_aggregator = LLMContextAggregatorPair(context)
|
||||||
|
|
||||||
|
user_aggregator = context_aggregator.user()
|
||||||
|
assistant_aggregator = context_aggregator.assistant()
|
||||||
|
|
||||||
# Build the pipeline
|
# Build the pipeline
|
||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
context_aggregator.user(),
|
user_aggregator,
|
||||||
llm,
|
llm,
|
||||||
transport.output(),
|
transport.output(),
|
||||||
context_aggregator.assistant(),
|
assistant_aggregator,
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -192,6 +199,18 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
|||||||
logger.info(f"Client disconnected")
|
logger.info(f"Client disconnected")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
|
@user_aggregator.event_handler("on_user_turn_stopped")
|
||||||
|
async def on_user_turn_stopped(aggregator, strategy, message: UserTurnStoppedMessage):
|
||||||
|
timestamp = f"[{message.timestamp}] " if message.timestamp else ""
|
||||||
|
line = f"{timestamp}user: {message.content}"
|
||||||
|
logger.info(f"Transcript: {line}")
|
||||||
|
|
||||||
|
@assistant_aggregator.event_handler("on_assistant_turn_stopped")
|
||||||
|
async def on_assistant_turn_stopped(aggregator, message: AssistantTurnStoppedMessage):
|
||||||
|
timestamp = f"[{message.timestamp}] " if message.timestamp else ""
|
||||||
|
line = f"{timestamp}assistant: {message.content}"
|
||||||
|
logger.info(f"Transcript: {line}")
|
||||||
|
|
||||||
# Run the pipeline
|
# Run the pipeline
|
||||||
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnal
|
|||||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.audio.vad.vad_analyzer import VADParams
|
from pipecat.audio.vad.vad_analyzer import VADParams
|
||||||
from pipecat.frames.frames import LLMRunFrame
|
from pipecat.frames.frames import LLMRunFrame
|
||||||
|
from pipecat.observers.loggers.transcription_log_observer import TranscriptionLogObserver
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
@@ -21,7 +22,6 @@ from pipecat.processors.aggregators.llm_response_universal import (
|
|||||||
LLMContextAggregatorPair,
|
LLMContextAggregatorPair,
|
||||||
LLMUserAggregatorParams,
|
LLMUserAggregatorParams,
|
||||||
)
|
)
|
||||||
from pipecat.processors.transcript_processor import TranscriptProcessor
|
|
||||||
from pipecat.runner.types import RunnerArguments
|
from pipecat.runner.types import RunnerArguments
|
||||||
from pipecat.runner.utils import create_transport
|
from pipecat.runner.utils import create_transport
|
||||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||||
@@ -36,6 +36,7 @@ from pipecat.turns.user_turn_strategies import UserTurnStrategies
|
|||||||
|
|
||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
|
|
||||||
# We store functions so objects (e.g. SileroVADAnalyzer) don't get
|
# We store functions so objects (e.g. SileroVADAnalyzer) don't get
|
||||||
# instantiated. The function will be called when the desired transport gets
|
# instantiated. The function will be called when the desired transport gets
|
||||||
# selected.
|
# selected.
|
||||||
@@ -70,8 +71,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
|||||||
|
|
||||||
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
|
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
|
||||||
|
|
||||||
transcript = TranscriptProcessor()
|
|
||||||
|
|
||||||
messages = [
|
messages = [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
@@ -94,7 +93,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
|||||||
[
|
[
|
||||||
transport.input(), # Transport user input
|
transport.input(), # Transport user input
|
||||||
stt,
|
stt,
|
||||||
transcript.user(), # User transcripts
|
|
||||||
context_aggregator.user(), # User responses
|
context_aggregator.user(), # User responses
|
||||||
llm, # LLM
|
llm, # LLM
|
||||||
tts, # TTS
|
tts, # TTS
|
||||||
@@ -110,6 +108,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
|||||||
enable_usage_metrics=True,
|
enable_usage_metrics=True,
|
||||||
),
|
),
|
||||||
idle_timeout_secs=runner_args.pipeline_idle_timeout_secs,
|
idle_timeout_secs=runner_args.pipeline_idle_timeout_secs,
|
||||||
|
observers=[TranscriptionLogObserver()],
|
||||||
)
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_client_connected")
|
@transport.event_handler("on_client_connected")
|
||||||
@@ -124,12 +123,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
|||||||
logger.info(f"Client disconnected")
|
logger.info(f"Client disconnected")
|
||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
# Register event handler for transcript updates
|
|
||||||
@transcript.event_handler("on_transcript_update")
|
|
||||||
async def on_transcript_update(processor, frame):
|
|
||||||
for message in frame.messages:
|
|
||||||
logger.info(f"Transcription [{message.role}]: {message.content}")
|
|
||||||
|
|
||||||
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
||||||
|
|
||||||
await runner.run(task)
|
await runner.run(task)
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
|||||||
|
|
||||||
# Note: Grok has built-in server-side VAD, so we don't need local VAD
|
# Note: Grok has built-in server-side VAD, so we don't need local VAD
|
||||||
# from pipecat.audio.vad.silero import SileroVADAnalyzer
|
# from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
from pipecat.frames.frames import LLMRunFrame, TranscriptionMessage
|
from pipecat.frames.frames import LLMRunFrame
|
||||||
from pipecat.observers.loggers.transcription_log_observer import (
|
from pipecat.observers.loggers.transcription_log_observer import (
|
||||||
TranscriptionLogObserver,
|
TranscriptionLogObserver,
|
||||||
)
|
)
|
||||||
@@ -45,9 +45,10 @@ from pipecat.pipeline.runner import PipelineRunner
|
|||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.llm_context import LLMContext
|
from pipecat.processors.aggregators.llm_context import LLMContext
|
||||||
from pipecat.processors.aggregators.llm_response_universal import (
|
from pipecat.processors.aggregators.llm_response_universal import (
|
||||||
|
AssistantTurnStoppedMessage,
|
||||||
LLMContextAggregatorPair,
|
LLMContextAggregatorPair,
|
||||||
|
UserTurnStoppedMessage,
|
||||||
)
|
)
|
||||||
from pipecat.processors.transcript_processor import TranscriptProcessor
|
|
||||||
from pipecat.runner.types import RunnerArguments
|
from pipecat.runner.types import RunnerArguments
|
||||||
from pipecat.runner.utils import create_transport
|
from pipecat.runner.utils import create_transport
|
||||||
from pipecat.services.grok.realtime.events import (
|
from pipecat.services.grok.realtime.events import (
|
||||||
@@ -208,9 +209,6 @@ Always be helpful and proactive in offering assistance.""",
|
|||||||
llm.register_function("get_current_time", get_current_time)
|
llm.register_function("get_current_time", get_current_time)
|
||||||
llm.register_function("get_restaurant_recommendation", get_restaurant_recommendation)
|
llm.register_function("get_restaurant_recommendation", get_restaurant_recommendation)
|
||||||
|
|
||||||
# Create transcript processor for logging
|
|
||||||
transcript = TranscriptProcessor()
|
|
||||||
|
|
||||||
# Create context with initial message and tools
|
# Create context with initial message and tools
|
||||||
context = LLMContext(
|
context = LLMContext(
|
||||||
[{"role": "user", "content": "Say hello and introduce yourself!"}],
|
[{"role": "user", "content": "Say hello and introduce yourself!"}],
|
||||||
@@ -219,18 +217,19 @@ Always be helpful and proactive in offering assistance.""",
|
|||||||
|
|
||||||
context_aggregator = LLMContextAggregatorPair(context)
|
context_aggregator = LLMContextAggregatorPair(context)
|
||||||
|
|
||||||
|
user_aggregator = context_aggregator.user()
|
||||||
|
assistant_aggregator = context_aggregator.assistant()
|
||||||
|
|
||||||
# Build the pipeline
|
# Build the pipeline
|
||||||
# Note: In realtime mode, transcription comes from Grok (upstream),
|
# Note: In realtime mode, transcription comes from Grok (upstream),
|
||||||
# so transcript.user() goes BEFORE llm
|
# so transcript.user() goes BEFORE llm
|
||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(), # Transport user input (audio)
|
transport.input(), # Transport user input (audio)
|
||||||
context_aggregator.user(),
|
user_aggregator,
|
||||||
transcript.user(), # Transcription from Grok goes upstream
|
|
||||||
llm, # Grok Realtime LLM (handles STT + LLM + TTS)
|
llm, # Grok Realtime LLM (handles STT + LLM + TTS)
|
||||||
transport.output(), # Transport bot output (audio)
|
transport.output(), # Transport bot output (audio)
|
||||||
transcript.assistant(), # Log assistant speech
|
assistant_aggregator,
|
||||||
context_aggregator.assistant(),
|
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -256,13 +255,17 @@ Always be helpful and proactive in offering assistance.""",
|
|||||||
await task.cancel()
|
await task.cancel()
|
||||||
|
|
||||||
# Log transcript updates
|
# Log transcript updates
|
||||||
@transcript.event_handler("on_transcript_update")
|
@user_aggregator.event_handler("on_user_turn_stopped")
|
||||||
async def on_transcript_update(processor, frame):
|
async def on_user_turn_stopped(aggregator, strategy, message: UserTurnStoppedMessage):
|
||||||
for msg in frame.messages:
|
timestamp = f"[{message.timestamp}] " if message.timestamp else ""
|
||||||
if isinstance(msg, TranscriptionMessage):
|
line = f"{timestamp}user: {message.content}"
|
||||||
timestamp = f"[{msg.timestamp}] " if msg.timestamp else ""
|
logger.info(f"Transcript: {line}")
|
||||||
line = f"{timestamp}{msg.role}: {msg.content}"
|
|
||||||
logger.info(f"Transcript: {line}")
|
@assistant_aggregator.event_handler("on_assistant_turn_stopped")
|
||||||
|
async def on_assistant_turn_stopped(aggregator, message: AssistantTurnStoppedMessage):
|
||||||
|
timestamp = f"[{message.timestamp}] " if message.timestamp else ""
|
||||||
|
line = f"{timestamp}assistant: {message.content}"
|
||||||
|
logger.info(f"Transcript: {line}")
|
||||||
|
|
||||||
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user