first vesion
This commit is contained in:
Xin Wang
2
engine/__init__.py
Normal file
2
engine/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
"""Minimal Pipecat-based voice engine."""
|
||||
|
||||
118
engine/config.py
Normal file
118
engine/config.py
Normal file
@@ -0,0 +1,118 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ServerConfig:
|
||||
host: str = "0.0.0.0"
|
||||
port: int = 8000
|
||||
cors_origins: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AudioConfig:
|
||||
sample_rate_hz: int = 16000
|
||||
channels: int = 1
|
||||
frame_ms: int = 20
|
||||
|
||||
@property
|
||||
def frame_bytes(self) -> int:
|
||||
return int(self.sample_rate_hz * self.frame_ms / 1000) * self.channels * 2
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SessionConfig:
|
||||
inactivity_timeout_sec: int = 60
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AgentConfig:
|
||||
system_prompt: str = "You are a helpful, friendly voice assistant."
|
||||
greeting: str | None = None
|
||||
greeting_mode: str = "generated"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LLMConfig:
|
||||
provider: str = "openai"
|
||||
api_key: str = ""
|
||||
base_url: str | None = None
|
||||
model: str = "gpt-4o-mini"
|
||||
temperature: float | None = 0.7
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class STTConfig:
|
||||
provider: str = "openai"
|
||||
api_key: str = ""
|
||||
base_url: str | None = None
|
||||
model: str = "gpt-4o-mini-transcribe"
|
||||
language: str | None = "en"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TTSConfig:
|
||||
provider: str = "openai"
|
||||
api_key: str = ""
|
||||
base_url: str | None = None
|
||||
model: str = "gpt-4o-mini-tts"
|
||||
voice: str = "alloy"
|
||||
source_sample_rate_hz: int | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ServicesConfig:
|
||||
llm: LLMConfig = field(default_factory=LLMConfig)
|
||||
stt: STTConfig = field(default_factory=STTConfig)
|
||||
tts: TTSConfig = field(default_factory=TTSConfig)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EngineConfig:
|
||||
server: ServerConfig = field(default_factory=ServerConfig)
|
||||
audio: AudioConfig = field(default_factory=AudioConfig)
|
||||
session: SessionConfig = field(default_factory=SessionConfig)
|
||||
agent: AgentConfig = field(default_factory=AgentConfig)
|
||||
services: ServicesConfig = field(default_factory=ServicesConfig)
|
||||
|
||||
|
||||
def load_config(path: str | Path = "config.json") -> EngineConfig:
|
||||
config_path = Path(path)
|
||||
if not config_path.exists() and str(path) == "config.json":
|
||||
config_path = Path(__file__).resolve().parent.parent / "config.json"
|
||||
data = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"Config file must contain a JSON object: {config_path}")
|
||||
return config_from_dict(data)
|
||||
|
||||
|
||||
def config_from_dict(data: dict) -> EngineConfig:
|
||||
services = _dict(data.get("services"))
|
||||
agent = _dict(data.get("agent"))
|
||||
if agent.get("greeting") == "":
|
||||
agent["greeting"] = None
|
||||
if agent.get("greeting_mode") not in (None, "generated", "fixed", "off"):
|
||||
raise ValueError("agent.greeting_mode must be one of: generated, fixed, off")
|
||||
|
||||
stt = _dict(services.get("stt") or services.get("asr"))
|
||||
if stt.get("language") == "":
|
||||
stt["language"] = None
|
||||
|
||||
return EngineConfig(
|
||||
server=ServerConfig(**_dict(data.get("server"))),
|
||||
audio=AudioConfig(**_dict(data.get("audio"))),
|
||||
session=SessionConfig(**_dict(data.get("session"))),
|
||||
agent=AgentConfig(**agent),
|
||||
services=ServicesConfig(
|
||||
llm=LLMConfig(**_dict(services.get("llm"))),
|
||||
stt=STTConfig(**stt),
|
||||
tts=TTSConfig(**_dict(services.get("tts"))),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _dict(value: object) -> dict:
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
80
engine/main.py
Normal file
80
engine/main.py
Normal file
@@ -0,0 +1,80 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from functools import lru_cache
|
||||
|
||||
from fastapi import FastAPI, WebSocket
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from .config import EngineConfig, load_config
|
||||
from .pipeline import run_product_voice_pipeline, run_voice_pipeline
|
||||
|
||||
|
||||
@lru_cache(maxsize=8)
|
||||
def get_config(path: str = "config.json") -> EngineConfig:
|
||||
return load_config(path)
|
||||
|
||||
|
||||
def create_app(config_path: str = "config.json") -> FastAPI:
|
||||
config = get_config(config_path)
|
||||
app = FastAPI(title="AI VideoAssistant Engine v5 Pipecat Minimal", version="0.1.0")
|
||||
app.state.config = config
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=config.server.cors_origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
@app.get("/health")
|
||||
async def health() -> dict[str, object]:
|
||||
return {
|
||||
"status": "healthy",
|
||||
"protocols": {
|
||||
"/ws": "pipecat.websocket.protobuf",
|
||||
"/ws-product": "va.ws.v1.json_base64",
|
||||
},
|
||||
"features": {
|
||||
"product_text_input": True,
|
||||
"product_text_interrupt": True,
|
||||
},
|
||||
"llm_provider": config.services.llm.provider,
|
||||
"stt_provider": config.services.stt.provider,
|
||||
"tts_provider": config.services.tts.provider,
|
||||
}
|
||||
|
||||
@app.websocket("/ws")
|
||||
async def websocket_endpoint(websocket: WebSocket) -> None:
|
||||
await websocket.accept()
|
||||
await run_voice_pipeline(websocket, config)
|
||||
|
||||
@app.websocket("/ws-product")
|
||||
async def product_websocket_endpoint(websocket: WebSocket) -> None:
|
||||
await websocket.accept()
|
||||
await run_product_voice_pipeline(websocket, config)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
app = create_app()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
import uvicorn
|
||||
|
||||
parser = argparse.ArgumentParser(description="Run the minimal Pipecat voice engine.")
|
||||
parser.add_argument("--config", default="config.json")
|
||||
args = parser.parse_args()
|
||||
|
||||
config = load_config(args.config)
|
||||
uvicorn.run(
|
||||
create_app(args.config),
|
||||
host=config.server.host,
|
||||
port=config.server.port,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
160
engine/pipeline.py
Normal file
160
engine/pipeline.py
Normal file
@@ -0,0 +1,160 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||
from pipecat.frames.frames import (
|
||||
LLMRunFrame,
|
||||
OutputTransportMessageUrgentFrame,
|
||||
TTSSpeakFrame,
|
||||
)
|
||||
from pipecat.pipeline.pipeline import Pipeline
|
||||
from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||
from pipecat.processors.aggregators.llm_context import LLMContext
|
||||
from pipecat.processors.aggregators.llm_response_universal import (
|
||||
AssistantTurnStoppedMessage,
|
||||
LLMContextAggregatorPair,
|
||||
LLMUserAggregatorParams,
|
||||
UserTurnStoppedMessage,
|
||||
)
|
||||
from pipecat.serializers.protobuf import ProtobufFrameSerializer
|
||||
from pipecat.serializers.base_serializer import FrameSerializer
|
||||
from pipecat.transports.websocket.fastapi import (
|
||||
FastAPIWebsocketParams,
|
||||
FastAPIWebsocketTransport,
|
||||
)
|
||||
|
||||
from .config import EngineConfig
|
||||
from .product_protocol import ProductWebsocketSerializer
|
||||
from .services import create_llm_service, create_stt_service, create_tts_service
|
||||
from .text_input import ProductTextInputProcessor
|
||||
from .text_stream import ProductTextStreamProcessor
|
||||
|
||||
|
||||
async def run_voice_pipeline(websocket, config: EngineConfig) -> None:
|
||||
await run_pipeline_with_serializer(
|
||||
websocket,
|
||||
config,
|
||||
serializer=ProtobufFrameSerializer(),
|
||||
client_label="Pipecat protobuf",
|
||||
)
|
||||
|
||||
|
||||
async def run_product_voice_pipeline(websocket, config: EngineConfig) -> None:
|
||||
await run_pipeline_with_serializer(
|
||||
websocket,
|
||||
config,
|
||||
serializer=ProductWebsocketSerializer(
|
||||
sample_rate=config.audio.sample_rate_hz,
|
||||
channels=config.audio.channels,
|
||||
),
|
||||
client_label="Product JSON",
|
||||
)
|
||||
|
||||
|
||||
async def run_pipeline_with_serializer(
|
||||
websocket,
|
||||
config: EngineConfig,
|
||||
*,
|
||||
serializer: FrameSerializer,
|
||||
client_label: str,
|
||||
) -> None:
|
||||
transport = FastAPIWebsocketTransport(
|
||||
websocket=websocket,
|
||||
params=FastAPIWebsocketParams(
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
audio_in_sample_rate=config.audio.sample_rate_hz,
|
||||
audio_out_sample_rate=config.audio.sample_rate_hz,
|
||||
audio_in_channels=config.audio.channels,
|
||||
audio_out_channels=config.audio.channels,
|
||||
serializer=serializer,
|
||||
session_timeout=None,
|
||||
),
|
||||
)
|
||||
|
||||
stt = create_stt_service(config.services.stt)
|
||||
llm = create_llm_service(config.services.llm)
|
||||
tts = create_tts_service(config.services.tts, config.audio)
|
||||
|
||||
messages = [{"role": "developer", "content": config.agent.system_prompt}]
|
||||
if config.agent.greeting and config.agent.greeting_mode == "generated":
|
||||
messages.append({"role": "developer", "content": config.agent.greeting})
|
||||
|
||||
context = LLMContext(messages)
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()),
|
||||
)
|
||||
|
||||
pipeline = Pipeline(
|
||||
[
|
||||
transport.input(),
|
||||
ProductTextInputProcessor(),
|
||||
stt,
|
||||
user_aggregator,
|
||||
llm,
|
||||
ProductTextStreamProcessor(),
|
||||
tts,
|
||||
transport.output(),
|
||||
assistant_aggregator,
|
||||
]
|
||||
)
|
||||
|
||||
task = PipelineTask(
|
||||
pipeline,
|
||||
params=PipelineParams(
|
||||
audio_in_sample_rate=config.audio.sample_rate_hz,
|
||||
audio_out_sample_rate=config.audio.sample_rate_hz,
|
||||
enable_metrics=True,
|
||||
enable_usage_metrics=True,
|
||||
enable_heartbeats=True,
|
||||
),
|
||||
idle_timeout_secs=config.session.inactivity_timeout_sec,
|
||||
)
|
||||
|
||||
@transport.event_handler("on_client_connected")
|
||||
async def on_client_connected(_transport, _client):
|
||||
logger.info(f"{client_label} websocket client connected")
|
||||
if config.agent.greeting_mode == "fixed" and config.agent.greeting:
|
||||
await task.queue_frames([TTSSpeakFrame(config.agent.greeting)])
|
||||
elif config.agent.greeting_mode == "generated":
|
||||
await task.queue_frames([LLMRunFrame()])
|
||||
|
||||
@transport.event_handler("on_client_disconnected")
|
||||
async def on_client_disconnected(_transport, _client):
|
||||
logger.info(f"{client_label} websocket client disconnected")
|
||||
await task.cancel()
|
||||
|
||||
@transport.event_handler("on_session_timeout")
|
||||
async def on_session_timeout(_transport, _client):
|
||||
logger.info(f"{client_label} websocket session timed out")
|
||||
await task.cancel()
|
||||
|
||||
@user_aggregator.event_handler("on_user_turn_stopped")
|
||||
async def on_user_turn_stopped(_aggregator, _strategy, message: UserTurnStoppedMessage):
|
||||
logger.info(f"User: {message.content}")
|
||||
text = (message.content or "").strip()
|
||||
if not text:
|
||||
return
|
||||
await task.queue_frame(
|
||||
OutputTransportMessageUrgentFrame(
|
||||
message={
|
||||
"type": "input.transcript.final",
|
||||
"text": text,
|
||||
"user_id": message.user_id,
|
||||
"timestamp": message.timestamp,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
# NOTE: assistant turn started/final events are emitted by
|
||||
# ProductTextStreamProcessor, upstream of TTS, so text streams to the
|
||||
# client ahead of audio. This logger is kept for server-side visibility.
|
||||
@assistant_aggregator.event_handler("on_assistant_turn_stopped")
|
||||
async def on_assistant_turn_stopped(_aggregator, message: AssistantTurnStoppedMessage):
|
||||
logger.info(f"Assistant: {message.content}")
|
||||
|
||||
runner = PipelineRunner(handle_sigint=False)
|
||||
await runner.run(task)
|
||||
164
engine/product_protocol.py
Normal file
164
engine/product_protocol.py
Normal file
@@ -0,0 +1,164 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
CancelFrame,
|
||||
BotStartedSpeakingFrame,
|
||||
BotStoppedSpeakingFrame,
|
||||
EndFrame,
|
||||
Frame,
|
||||
InputAudioRawFrame,
|
||||
InputTransportMessageFrame,
|
||||
OutputAudioRawFrame,
|
||||
OutputTransportMessageFrame,
|
||||
OutputTransportMessageUrgentFrame,
|
||||
TextFrame,
|
||||
TranscriptionFrame,
|
||||
)
|
||||
from pipecat.serializers.base_serializer import FrameSerializer
|
||||
|
||||
|
||||
class ProductWebsocketSerializer(FrameSerializer):
|
||||
"""Stable app-facing JSON/base64 protocol adapter for Pipecat websocket transport."""
|
||||
|
||||
protocol = "va.ws.v1"
|
||||
|
||||
def __init__(self, *, sample_rate: int, channels: int):
|
||||
super().__init__()
|
||||
self._sample_rate = sample_rate
|
||||
self._channels = channels
|
||||
self._sequence = 0
|
||||
|
||||
async def serialize(self, frame: Frame) -> str | bytes | None:
|
||||
if isinstance(frame, OutputAudioRawFrame):
|
||||
return self._event(
|
||||
"response.audio.delta",
|
||||
audio=base64.b64encode(frame.audio).decode("ascii"),
|
||||
bytes=len(frame.audio),
|
||||
sample_rate=frame.sample_rate,
|
||||
channels=frame.num_channels,
|
||||
)
|
||||
|
||||
if isinstance(frame, BotStartedSpeakingFrame):
|
||||
return self._event("response.audio.started")
|
||||
|
||||
if isinstance(frame, BotStoppedSpeakingFrame):
|
||||
return self._event("response.audio.stopped")
|
||||
|
||||
if isinstance(frame, TranscriptionFrame):
|
||||
return self._event(
|
||||
"input.transcript.final",
|
||||
text=frame.text,
|
||||
user_id=frame.user_id,
|
||||
timestamp=frame.timestamp,
|
||||
)
|
||||
|
||||
if isinstance(frame, TextFrame):
|
||||
return self._event("response.text.delta", text=frame.text)
|
||||
|
||||
if isinstance(frame, (OutputTransportMessageFrame, OutputTransportMessageUrgentFrame)):
|
||||
if self.should_ignore_frame(frame):
|
||||
return None
|
||||
message = frame.message
|
||||
# Allow callers to emit any named protocol event by pushing a
|
||||
# transport-message frame whose payload already carries a `type`.
|
||||
# The payload's other fields are merged alongside `type`, so e.g.
|
||||
# `{"type": "response.text.final", "text": "..."}` is sent verbatim.
|
||||
if isinstance(message, dict) and isinstance(message.get("type"), str):
|
||||
event_type = message["type"]
|
||||
payload = {k: v for k, v in message.items() if k != "type"}
|
||||
return self._event(event_type, **payload)
|
||||
return self._event("transport.message", message=message)
|
||||
|
||||
return None
|
||||
|
||||
async def deserialize(self, data: str | bytes) -> Frame | None:
|
||||
if isinstance(data, bytes):
|
||||
return InputAudioRawFrame(
|
||||
audio=data,
|
||||
sample_rate=self._sample_rate,
|
||||
num_channels=self._channels,
|
||||
)
|
||||
|
||||
try:
|
||||
message = json.loads(data)
|
||||
except json.JSONDecodeError as exc:
|
||||
logger.warning(f"Invalid product websocket JSON: {exc}")
|
||||
return None
|
||||
|
||||
if not isinstance(message, dict):
|
||||
logger.warning("Product websocket message must be a JSON object")
|
||||
return None
|
||||
|
||||
message_type = message.get("type")
|
||||
if message_type == "session.start":
|
||||
return InputTransportMessageFrame(
|
||||
message={
|
||||
"type": "session.started",
|
||||
"protocol": self.protocol,
|
||||
"audio": {
|
||||
"encoding": "pcm_s16le",
|
||||
"sample_rate": self._sample_rate,
|
||||
"channels": self._channels,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
if message_type == "session.stop":
|
||||
return EndFrame()
|
||||
|
||||
if message_type == "response.cancel":
|
||||
return CancelFrame(reason="client_cancelled")
|
||||
|
||||
if message_type == "input.audio":
|
||||
audio = message.get("audio") or message.get("data")
|
||||
if not isinstance(audio, str):
|
||||
logger.warning("input.audio requires base64 'audio' or 'data'")
|
||||
return None
|
||||
try:
|
||||
pcm = base64.b64decode(audio)
|
||||
except ValueError as exc:
|
||||
logger.warning(f"Invalid input.audio base64: {exc}")
|
||||
return None
|
||||
return InputAudioRawFrame(
|
||||
audio=pcm,
|
||||
sample_rate=int(message.get("sample_rate") or self._sample_rate),
|
||||
num_channels=int(message.get("channels") or self._channels),
|
||||
)
|
||||
|
||||
if message_type == "input.text":
|
||||
text = message.get("text")
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
logger.warning("input.text requires non-empty 'text'")
|
||||
return None
|
||||
return InputTransportMessageFrame(
|
||||
message={
|
||||
"type": "input.text",
|
||||
"text": text,
|
||||
"interrupt": bool(message.get("interrupt", True)),
|
||||
}
|
||||
)
|
||||
|
||||
if message_type == "transport.message":
|
||||
payload = message.get("message")
|
||||
return InputTransportMessageFrame(message=payload if isinstance(payload, dict) else message)
|
||||
|
||||
logger.warning(f"Unsupported product websocket message type: {message_type!r}")
|
||||
return None
|
||||
|
||||
def _event(self, event_type: str, **payload: Any) -> str:
|
||||
self._sequence += 1
|
||||
return json.dumps(
|
||||
{
|
||||
"type": event_type,
|
||||
"protocol": self.protocol,
|
||||
"seq": self._sequence,
|
||||
**payload,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
126
engine/services.py
Normal file
126
engine/services.py
Normal file
@@ -0,0 +1,126 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
from openai import BadRequestError
|
||||
from openai import NOT_GIVEN
|
||||
|
||||
from pipecat.frames.frames import ErrorFrame, Frame, TTSAudioRawFrame
|
||||
from pipecat.services.openai._constants import OPENAI_SAMPLE_RATE
|
||||
from pipecat.services.openai.llm import OpenAILLMService
|
||||
from pipecat.services.openai.stt import OpenAISTTService
|
||||
from pipecat.services.openai.tts import VALID_VOICES, OpenAITTSService
|
||||
from pipecat.transcriptions.language import Language
|
||||
|
||||
from .config import AudioConfig, LLMConfig, STTConfig, TTSConfig
|
||||
|
||||
|
||||
def create_stt_service(config: STTConfig):
|
||||
_require_provider(config.provider, "openai", "stt")
|
||||
return OpenAISTTService(
|
||||
api_key=config.api_key or None,
|
||||
base_url=config.base_url,
|
||||
settings=OpenAISTTService.Settings(
|
||||
model=config.model,
|
||||
language=_language(config.language),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def create_llm_service(config: LLMConfig):
|
||||
_require_provider(config.provider, "openai", "llm")
|
||||
return OpenAILLMService(
|
||||
api_key=config.api_key or None,
|
||||
base_url=config.base_url,
|
||||
settings=OpenAILLMService.Settings(
|
||||
model=config.model,
|
||||
temperature=config.temperature if config.temperature is not None else NOT_GIVEN,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def create_tts_service(config: TTSConfig, audio: AudioConfig):
|
||||
_require_provider(config.provider, "openai", "tts")
|
||||
service_class = OpenAITTSService if config.voice in VALID_VOICES else OpenAICompatibleTTSService
|
||||
return service_class(
|
||||
api_key=config.api_key or None,
|
||||
base_url=config.base_url,
|
||||
sample_rate=audio.sample_rate_hz,
|
||||
source_sample_rate=config.source_sample_rate_hz,
|
||||
settings=OpenAITTSService.Settings(
|
||||
model=config.model,
|
||||
voice=config.voice,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class OpenAICompatibleTTSService(OpenAITTSService):
|
||||
"""OpenAI-compatible TTS service that permits provider-specific voice ids."""
|
||||
|
||||
def __init__(self, *, source_sample_rate: int | None = None, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self._source_sample_rate = source_sample_rate or OPENAI_SAMPLE_RATE
|
||||
|
||||
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
|
||||
voice = self._settings.voice
|
||||
if not voice:
|
||||
yield ErrorFrame(error="TTS voice must be specified")
|
||||
return
|
||||
|
||||
try:
|
||||
create_params = {
|
||||
"input": text,
|
||||
"model": self._settings.model,
|
||||
"voice": voice,
|
||||
"response_format": "pcm",
|
||||
}
|
||||
|
||||
if self._settings.instructions:
|
||||
create_params["instructions"] = self._settings.instructions
|
||||
|
||||
if self._settings.speed:
|
||||
create_params["speed"] = self._settings.speed
|
||||
|
||||
async with self._client.audio.speech.with_streaming_response.create(
|
||||
**create_params
|
||||
) as response:
|
||||
if response.status_code != 200:
|
||||
error = await response.text()
|
||||
yield ErrorFrame(
|
||||
error=f"TTS request failed (status: {response.status_code}, error: {error})"
|
||||
)
|
||||
return
|
||||
|
||||
await self.start_tts_usage_metrics(text)
|
||||
|
||||
async def audio_chunks():
|
||||
async for chunk in response.iter_bytes(self.chunk_size):
|
||||
if chunk:
|
||||
yield chunk
|
||||
|
||||
first_frame = True
|
||||
async for frame in self._stream_audio_frames_from_iterator(
|
||||
audio_chunks(),
|
||||
in_sample_rate=self._source_sample_rate,
|
||||
context_id=context_id,
|
||||
):
|
||||
if first_frame:
|
||||
await self.stop_ttfb_metrics()
|
||||
first_frame = False
|
||||
yield frame
|
||||
except BadRequestError as exc:
|
||||
yield ErrorFrame(error=f"TTS request failed: {exc}")
|
||||
except Exception as exc:
|
||||
yield ErrorFrame(error=f"TTS request failed: {exc}")
|
||||
|
||||
|
||||
def _require_provider(actual: str, expected: str, service_name: str) -> None:
|
||||
if actual != expected:
|
||||
raise ValueError(f"Unsupported {service_name} provider {actual!r}; expected {expected!r}")
|
||||
|
||||
|
||||
def _language(value: str | None) -> Language | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = value.replace("-", "_").upper()
|
||||
return getattr(Language, normalized, value)
|
||||
39
engine/text_input.py
Normal file
39
engine/text_input.py
Normal file
@@ -0,0 +1,39 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.frames.frames import Frame, InputTransportMessageFrame, LLMMessagesAppendFrame
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
|
||||
|
||||
class ProductTextInputProcessor(FrameProcessor):
|
||||
"""Converts product text-input transport messages into LLM turns."""
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
if not isinstance(frame, InputTransportMessageFrame):
|
||||
await self.push_frame(frame, direction)
|
||||
return
|
||||
|
||||
message = frame.message
|
||||
if not isinstance(message, dict) or message.get("type") != "input.text":
|
||||
await self.push_frame(frame, direction)
|
||||
return
|
||||
|
||||
text = str(message.get("text") or "").strip()
|
||||
if not text:
|
||||
return
|
||||
|
||||
if message.get("interrupt", True):
|
||||
logger.info("Text input interrupting current response")
|
||||
await self.broadcast_interruption()
|
||||
|
||||
await self.push_frame(
|
||||
LLMMessagesAppendFrame(
|
||||
messages=[{"role": "user", "content": text}],
|
||||
run_llm=True,
|
||||
),
|
||||
FrameDirection.DOWNSTREAM,
|
||||
)
|
||||
|
||||
93
engine/text_stream.py
Normal file
93
engine/text_stream.py
Normal file
@@ -0,0 +1,93 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
Frame,
|
||||
InterruptionFrame,
|
||||
LLMFullResponseEndFrame,
|
||||
LLMFullResponseStartFrame,
|
||||
LLMTextFrame,
|
||||
OutputTransportMessageUrgentFrame,
|
||||
TTSSpeakFrame,
|
||||
)
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
|
||||
|
||||
class ProductTextStreamProcessor(FrameProcessor):
|
||||
"""Mirrors LLM text frames as streaming protocol events.
|
||||
|
||||
Placed between the LLM service and the TTS service, this processor
|
||||
observes the LLM's text frames as they're emitted and forwards them
|
||||
downstream as ``OutputTransportMessageUrgentFrame``s that the product
|
||||
serializer turns into ``response.text.{started,delta,final}`` events.
|
||||
|
||||
Because the events are emitted before the TTS holds onto
|
||||
``LLMFullResponseEndFrame`` to drain its audio queue, text reaches the
|
||||
client well ahead of (or at worst, alongside) the synthesized audio.
|
||||
|
||||
``TTSSpeakFrame`` (used by the fixed-greeting code path, which bypasses
|
||||
the LLM entirely) is also handled: the processor synthesizes a single
|
||||
started/delta/final sequence for its fixed text.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self._aggregation: list[str] = []
|
||||
self._turn_active = False
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
if isinstance(frame, LLMFullResponseStartFrame):
|
||||
await self._start_turn()
|
||||
elif isinstance(frame, LLMTextFrame):
|
||||
if frame.text:
|
||||
await self._delta(frame.text)
|
||||
elif isinstance(frame, LLMFullResponseEndFrame):
|
||||
await self._end_turn(interrupted=False)
|
||||
elif isinstance(frame, InterruptionFrame):
|
||||
await self._end_turn(interrupted=True)
|
||||
elif isinstance(frame, TTSSpeakFrame):
|
||||
# Fixed-text / direct-speech path: there's no LLM cycle, so
|
||||
# synthesize one started/delta/final sequence for the spoken text.
|
||||
text = frame.text or ""
|
||||
await self._start_turn()
|
||||
if text:
|
||||
await self._delta(text)
|
||||
await self._end_turn(interrupted=False)
|
||||
|
||||
await self.push_frame(frame, direction)
|
||||
|
||||
async def _start_turn(self) -> None:
|
||||
if self._turn_active:
|
||||
return
|
||||
self._turn_active = True
|
||||
self._aggregation = []
|
||||
await self._emit("response.text.started")
|
||||
|
||||
async def _delta(self, text: str) -> None:
|
||||
if not self._turn_active:
|
||||
# A text frame outside a turn shouldn't happen, but if it does,
|
||||
# synthesize a started boundary so the client renders sensibly.
|
||||
await self._start_turn()
|
||||
self._aggregation.append(text)
|
||||
await self._emit("response.text.delta", text=text)
|
||||
|
||||
async def _end_turn(self, *, interrupted: bool) -> None:
|
||||
if not self._turn_active:
|
||||
return
|
||||
full_text = "".join(self._aggregation)
|
||||
self._turn_active = False
|
||||
self._aggregation = []
|
||||
await self._emit(
|
||||
"response.text.final",
|
||||
text=full_text,
|
||||
interrupted=interrupted,
|
||||
)
|
||||
|
||||
async def _emit(self, event_type: str, **payload: object) -> None:
|
||||
await self.push_frame(
|
||||
OutputTransportMessageUrgentFrame(
|
||||
message={"type": event_type, **payload},
|
||||
),
|
||||
FrameDirection.DOWNSTREAM,
|
||||
)
|
||||
Reference in New Issue
Block a user