wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Additional LLM and TTS metrics (#343)

Browse Source 
* added llm and tts usage metrics

* Metrics debug logging

* cleanup
This commit is contained in:
chadbailey59
2024-08-07 08:55:51 -05:00
committed by GitHub
parent 83a037a7ce
commit 3958bb7903
11 changed files with 101 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
examples/foundational/06-listen-and-respond.py
View File

@@ -9,14 +9,15 @@ import aiohttp
import os
import sys
from pipecat.frames.frames import LLMMessagesFrame
from pipecat.frames.frames import Frame, LLMMessagesFrame, MetricsFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.llm_response import (
LLMAssistantResponseAggregator,
LLMUserResponseAggregator,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.logger import FrameLogger
from pipecat.services.elevenlabs import ElevenLabsTTSService
from pipecat.services.openai import OpenAILLMService
@@ -34,6 +35,14 @@ logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
class MetricsLogger(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection):
if isinstance(frame, MetricsFrame):
print(
f"!!! MetricsFrame: {frame}, ttfb: {frame.ttfb}, processing: {frame.processing}, tokens: {frame.tokens}, characters: {frame.characters}")
await self.push_frame(frame, direction)
async def main():
async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session)
@@ -58,11 +67,10 @@ async def main():
llm = OpenAILLMService(
api_key=os.getenv("OPENAI_API_KEY"),
model="gpt-4o")
model="gpt-4o"
)
fl = FrameLogger("!!! after LLM", "red")
fltts = FrameLogger("@@@ out of tts", "green")
flend = FrameLogger("### out of the end", "magenta")
ml = MetricsLogger()
messages = [
{
@@ -77,15 +85,18 @@ async def main():
transport.input(),
tma_in,
llm,
fl,
tts,
fltts,
ml,
transport.output(),
tma_out,
flend
])
task = PipelineTask(pipeline)
task = PipelineTask(pipeline, PipelineParams(
allow_interruptions=True,
enable_metrics=True,
report_only_initial_ttfb=False,
))
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):

2
src/pipecat/frames/frames.py
View File

@@ -276,6 +276,8 @@ class MetricsFrame(SystemFrame):
"""
ttfb: List[Mapping[str, Any]] | None = None
processing: List[Mapping[str, Any]] | None = None
tokens: List[Mapping[str, Any]] | None = None
characters: List[Mapping[str, Any]] | None = None
#
# Control frames

6
src/pipecat/processors/frameworks/rtvi.py
View File

@@ -537,7 +537,11 @@ class RTVIProcessor(FrameProcessor):
processors.extend(pipeline.processors_with_metrics())
ttfb = [{"processor": p.name, "value": 0.0} for p in processors]
processing = [{"processor": p.name, "value": 0.0} for p in processors]
await self.push_frame(MetricsFrame(ttfb=ttfb, processing=processing))
tokens = [{"processor": p.name, "value": {"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0}} for p in processors]
characters = [{"processor": p.name, "value": 0} for p in processors]
await self.push_frame(MetricsFrame(ttfb=ttfb, processing=processing, tokens=tokens, characters=characters))
self._pipeline = pipeline

9
src/pipecat/services/azure.py
View File

@@ -18,6 +18,7 @@ from pipecat.frames.frames import (
EndFrame,
ErrorFrame,
Frame,
MetricsFrame,
StartFrame,
SystemFrame,
TranscriptionFrame,
@@ -87,7 +88,13 @@ class AzureTTSService(TTSService):
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
if self.can_generate_metrics() and self.metrics_enabled:
characters = {
"processor": self.name,
"value": len(text),
}
logger.debug(f"{self.name} Characters: {characters['value']}")
await self.push_frame(MetricsFrame(characters=[characters]))
await self.start_ttfb_metrics()
ssml = (

8
src/pipecat/services/cartesia.py
View File

@@ -21,6 +21,7 @@ from pipecat.frames.frames import (
StartFrame,
EndFrame,
TextFrame,
MetricsFrame,
LLMFullResponseEndFrame
)
from pipecat.services.ai_services import TTSService
@@ -200,6 +201,13 @@ class CartesiaTTSService(TTSService):
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
if self.can_generate_metrics() and self.metrics_enabled:
characters = {
"processor": self.name,
"value": len(text),
}
logger.debug(f"{self.name} Characters: {characters['value']}")
await self.push_frame(MetricsFrame(characters=[characters]))
try:
if not self._websocket:

9
src/pipecat/services/deepgram.py
View File

@@ -15,6 +15,7 @@ from pipecat.frames.frames import (
ErrorFrame,
Frame,
InterimTranscriptionFrame,
MetricsFrame,
StartFrame,
SystemFrame,
TranscriptionFrame)
@@ -70,7 +71,13 @@ class DeepgramTTSService(TTSService):
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
if self.can_generate_metrics() and self.metrics_enabled:
characters = {
"processor": self.name,
"value": len(text),
}
logger.debug(f"{self.name} Characters: {characters['value']}")
await self.push_frame(MetricsFrame(characters=[characters]))
base_url = self._base_url
request_url = f"{base_url}?model={self._voice}&encoding={self._encoding}&container=none&sample_rate={self._sample_rate}"
headers = {"authorization": f"token {self._api_key}"}

10
src/pipecat/services/elevenlabs.py
View File

@@ -8,7 +8,7 @@ import aiohttp
from typing import AsyncGenerator
from pipecat.frames.frames import AudioRawFrame, ErrorFrame, Frame
from pipecat.frames.frames import AudioRawFrame, ErrorFrame, Frame, MetricsFrame
from pipecat.services.ai_services import TTSService
from loguru import logger
@@ -40,7 +40,13 @@ class ElevenLabsTTSService(TTSService):
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
if self.can_generate_metrics() and self.metrics_enabled:
characters = {
"processor": self.name,
"value": len(text),
}
logger.debug(f"{self.name} Characters: {characters['value']}")
await self.push_frame(MetricsFrame(characters=[characters]))
url = f"https://api.elevenlabs.io/v1/text-to-speech/{self._voice_id}/stream"
payload = {"text": text, "model_id": self._model}

23
src/pipecat/services/openai.py
View File

@@ -23,6 +23,7 @@ from pipecat.frames.frames import (
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMModelUpdateFrame,
MetricsFrame,
TextFrame,
URLImageRawFrame,
VisionImageRawFrame
@@ -95,6 +96,7 @@ class BaseOpenAILLMService(LLMService):
messages=messages,
tools=context.tools,
tool_choice=context.tool_choice,
stream_options={"include_usage": True}
)
return chunks
@@ -132,6 +134,19 @@ class BaseOpenAILLMService(LLMService):
)
async for chunk in chunk_stream:
if chunk.usage:
if self.can_generate_metrics() and self.metrics_enabled:
tokens = {
"processor": self.name,
"prompt_tokens": chunk.usage.prompt_tokens,
"completion_tokens": chunk.usage.completion_tokens,
"total_tokens": chunk.usage.total_tokens
}
logger.debug(
f"{self.name} prompt tokens: {tokens['prompt_tokens']}, completion tokens: {tokens['completion_tokens']}")
await self.push_frame(MetricsFrame(tokens=[tokens]))
if len(chunk.choices) == 0:
continue
@@ -323,7 +338,13 @@ class OpenAITTSService(TTSService):
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
if self.can_generate_metrics() and self.metrics_enabled:
characters = {
"processor": self.name,
"value": len(text),
}
logger.debug(f"{self.name} Characters: {characters['value']}")
await self.push_frame(MetricsFrame(characters=[characters]))
try:
await self.start_ttfb_metrics()

10
src/pipecat/services/playht.py
View File

@@ -9,7 +9,7 @@ import struct
from typing import AsyncGenerator
from pipecat.frames.frames import AudioRawFrame, Frame
from pipecat.frames.frames import AudioRawFrame, Frame, MetricsFrame
from pipecat.services.ai_services import TTSService
from loguru import logger
@@ -48,7 +48,13 @@ class PlayHTTTSService(TTSService):
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
if self.can_generate_metrics() and self.metrics_enabled:
characters = {
"processor": self.name,
"value": len(text),
}
logger.debug(f"{self.name} Characters: {characters['value']}")
await self.push_frame(MetricsFrame(characters=[characters]))
try:
b = bytearray()
in_header = True

10
src/pipecat/services/xtts.py
View File

@@ -8,7 +8,7 @@ import aiohttp
from typing import Any, AsyncGenerator, Dict
from pipecat.frames.frames import AudioRawFrame, ErrorFrame, Frame, StartFrame
from pipecat.frames.frames import AudioRawFrame, ErrorFrame, Frame, MetricsFrame, StartFrame
from pipecat.services.ai_services import TTSService
from loguru import logger
@@ -70,7 +70,13 @@ class XTTSService(TTSService):
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
if self.can_generate_metrics() and self.metrics_enabled:
characters = {
"processor": self.name,
"value": len(text),
}
logger.debug(f"{self.name} Characters: {characters['value']}")
await self.push_frame(MetricsFrame(characters=[characters]))
if not self._studio_speakers:
logger.error(f"{self} no studio speakers available")
return

4
src/pipecat/transports/services/daily.py
View File

@@ -699,6 +699,10 @@ class DailyOutputTransport(BaseOutputTransport):
metrics["ttfb"] = frame.ttfb
if frame.processing:
metrics["processing"] = frame.processing
if frame.tokens:
metrics["tokens"] = frame.tokens
if frame.characters:
metrics["characters"] = frame.characters
message = DailyTransportMessageFrame(message={
"type": "pipecat-metrics",