wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cleanup on aisle METRICS. Note: See below, this is a breaking change

Browse Source 
1. Fleshed out MetricsFrames and broke it into a proper set of types
2. Add model_name as a property to the AIService so that it can be
   automatically included in metrics and also remove that
   overhead from all the various services themselves

Breaking change!

Because of the types improvements, the MetricsFrame type has
changed. Each frame will have a list of metrics simlilar to before
except each item in the list will only contain one type of metric:
"ttfb", "tokens", "characters", or "processing". Previously these
fields would be in every entry but set to None if they didn't apply.

While this changes internal handling of the MetricsFrame, it does NOT
break the RTVI/daily messaging of metrics. That format remains the same.

Also. Remember to use model_name for accessing a service's current
model and set_model_name for setting it.
This commit is contained in:
mattie ruth backman
2024-09-17 14:49:08 -04:00
parent ed409d0460
commit a4edb3dab1
21 changed files with 190 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
examples/foundational/06-listen-and-respond.py
View File

@@ -10,6 +10,7 @@ import os
import sys
from pipecat.frames.frames import Frame, LLMMessagesFrame, MetricsFrame
from pipecat.metrics.metrics import TTFBMetricsData, ProcessingMetricsData, LLMUsageMetricsData, TTSUsageMetricsData
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
@@ -37,8 +38,19 @@ logger.add(sys.stderr, level="DEBUG")
class MetricsLogger(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection):
if isinstance(frame, MetricsFrame):
print(
f"!!! MetricsFrame: {frame}, ttfb: {frame.ttfb}, processing: {frame.processing}, tokens: {frame.tokens}, characters: {frame.characters}")
for d in frame.data:
if isinstance(d, TTFBMetricsData):
print(f"!!! MetricsFrame: {frame}, ttfb: {d.value}")
elif isinstance(d, ProcessingMetricsData):
print(f"!!! MetricsFrame: {frame}, processing: {d.value}")
elif isinstance(d, LLMUsageMetricsData):
tokens = d.value
print(
f"!!! MetricsFrame: {frame}, tokens: {
tokens.prompt_tokens}, characters: {
tokens.completion_tokens}")
elif isinstance(d, TTSUsageMetricsData):
print(f"!!! MetricsFrame: {frame}, characters: {d.value}")
await self.push_frame(frame, direction)

9
src/pipecat/frames/frames.py
View File

@@ -4,11 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import Any, List, Mapping, Optional, Tuple
from typing import Any, List, Optional, Tuple
from dataclasses import dataclass, field
from pipecat.clocks.base_clock import BaseClock
from pipecat.metrics.metrics import MetricsData
from pipecat.transcriptions.language import Language
from pipecat.utils.time import nanoseconds_to_str
from pipecat.utils.utils import obj_count, obj_id
@@ -333,10 +334,8 @@ class BotInterruptionFrame(SystemFrame):
class MetricsFrame(SystemFrame):
"""Emitted by processor that can compute metrics like latencies.
"""
ttfb: List[Mapping[str, Any]] | None = None
processing: List[Mapping[str, Any]] | None = None
tokens: List[Mapping[str, Any]] | None = None
characters: List[Mapping[str, Any]] | None = None
data: List[MetricsData]
#
# Control frames

0
src/pipecat/metrics/__init__.py Normal file
View File

31
src/pipecat/metrics/metrics.py Normal file
View File

@@ -0,0 +1,31 @@
from typing import Optional
from pydantic import BaseModel
class MetricsData(BaseModel):
processor: str
model: Optional[str] = None
class TTFBMetricsData(MetricsData):
value: float
class ProcessingMetricsData(MetricsData):
value: float
class LLMTokenUsage(BaseModel):
prompt_tokens: int
completion_tokens: int
total_tokens: int
cache_read_input_tokens: Optional[int] = None
cache_creation_input_tokens: Optional[int] = None
class LLMUsageMetricsData(MetricsData):
value: LLMTokenUsage
class TTSUsageMetricsData(MetricsData):
value: int

9
src/pipecat/pipeline/task.py
View File

@@ -20,6 +20,7 @@ from pipecat.frames.frames import (
MetricsFrame,
StartFrame,
StopTaskFrame)
from pipecat.metrics.metrics import TTFBMetricsData, ProcessingMetricsData
from pipecat.pipeline.base_pipeline import BasePipeline
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.utils import obj_count, obj_id
@@ -118,9 +119,11 @@ class PipelineTask:
def _initial_metrics_frame(self) -> MetricsFrame:
processors = self._pipeline.processors_with_metrics()
ttfb = [{"processor": p.name, "value": 0.0} for p in processors]
processing = [{"processor": p.name, "value": 0.0} for p in processors]
return MetricsFrame(ttfb=ttfb, processing=processing)
data = []
for p in processors:
data.append(TTFBMetricsData(processor=p.name, value=0.0))
data.append(ProcessingMetricsData(processor=p.name, value=0.0))
return MetricsFrame(data=data)
async def _process_down_queue(self):
self._clock.start()

67
src/pipecat/processors/frame_processor.py
View File

@@ -19,6 +19,13 @@ from pipecat.frames.frames import (
StartInterruptionFrame,
StopInterruptionFrame,
SystemFrame)
from pipecat.metrics.metrics import (
LLMTokenUsage,
LLMUsageMetricsData,
MetricsData,
ProcessingMetricsData,
TTFBMetricsData,
TTSUsageMetricsData)
from pipecat.utils.utils import obj_count, obj_id
from loguru import logger
@@ -31,11 +38,20 @@ class FrameDirection(Enum):
class FrameProcessorMetrics:
def __init__(self, name: str):
self._name = name
self._core_metrics_data = MetricsData(processor=name)
self._start_ttfb_time = 0
self._start_processing_time = 0
self._should_report_ttfb = True
def _processor_name(self):
return self._core_metrics_data.processor
def _model_name(self):
return self._core_metrics_data.model
def set_core_metrics_data(self, data: MetricsData):
self._core_metrics_data = data
async def start_ttfb_metrics(self, report_only_initial_ttfb):
if self._should_report_ttfb:
self._start_ttfb_time = time.time()
@@ -46,13 +62,13 @@ class FrameProcessorMetrics:
return None
value = time.time() - self._start_ttfb_time
logger.debug(f"{self._name} TTFB: {value}")
ttfb = {
"processor": self._name,
"value": value
}
logger.debug(f"{self._processor_name()} TTFB: {value}")
ttfb = TTFBMetricsData(
processor=self._processor_name(),
value=value,
model=self._model_name())
self._start_ttfb_time = 0
return MetricsFrame(ttfb=[ttfb])
return MetricsFrame(data=[ttfb])
async def start_processing_metrics(self):
self._start_processing_time = time.time()
@@ -62,26 +78,28 @@ class FrameProcessorMetrics:
return None
value = time.time() - self._start_processing_time
logger.debug(f"{self._name} processing time: {value}")
processing = {
"processor": self._name,
"value": value
}
logger.debug(f"{self._processor_name()} processing time: {value}")
processing = ProcessingMetricsData(
processor=self._processor_name(), value=value, model=self._model_name())
self._start_processing_time = 0
return MetricsFrame(processing=[processing])
return MetricsFrame(data=[processing])
async def start_llm_usage_metrics(self, tokens: dict):
async def start_llm_usage_metrics(self, tokens: LLMTokenUsage):
logger.debug(
f"{self._name} prompt tokens: {tokens['prompt_tokens']}, completion tokens: {tokens['completion_tokens']}")
return MetricsFrame(tokens=[tokens])
f"{self._processor_name()} prompt tokens: {tokens.prompt_tokens}, completion tokens: {tokens.completion_tokens}")
value = LLMUsageMetricsData(
processor=self._processor_name(),
model=self._model_name(),
value=tokens)
return MetricsFrame(data=[value])
async def start_tts_usage_metrics(self, text: str):
characters = {
"processor": self._name,
"value": len(text),
}
logger.debug(f"{self._name} usage characters: {characters['value']}")
return MetricsFrame(characters=[characters])
characters = TTSUsageMetricsData(
processor=self._processor_name(),
model=self._model_name(),
value=len(text))
logger.debug(f"{self._processor_name()} usage characters: {characters.value}")
return MetricsFrame(data=[characters])
class FrameProcessor:
@@ -140,6 +158,9 @@ class FrameProcessor:
def can_generate_metrics(self) -> bool:
return False
def set_core_metrics_data(self, data: MetricsData):
self._metrics.set_core_metrics_data(data)
async def start_ttfb_metrics(self):
if self.can_generate_metrics() and self.metrics_enabled:
await self._metrics.start_ttfb_metrics(self._report_only_initial_ttfb)
@@ -160,7 +181,7 @@ class FrameProcessor:
if frame:
await self.push_frame(frame)
async def start_llm_usage_metrics(self, tokens: dict):
async def start_llm_usage_metrics(self, tokens: LLMTokenUsage):
if self.can_generate_metrics() and self.usage_metrics_enabled:
frame = await self._metrics.start_llm_usage_metrics(tokens)
if frame:

14
src/pipecat/services/ai_services.py
View File

@@ -32,6 +32,7 @@ from pipecat.frames.frames import (
UserImageRequestFrame,
VisionImageRawFrame
)
from pipecat.metrics.metrics import MetricsData
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.transcriptions.language import Language
from pipecat.utils.audio import calculate_audio_volume
@@ -46,6 +47,15 @@ from loguru import logger
class AIService(FrameProcessor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._model_name: str = ""
@property
def model_name(self) -> str:
return self._model_name
def set_model_name(self, model: str):
self._model_name = model
self.set_core_metrics_data(MetricsData(processor=self.name, model=self._model_name))
async def start(self, frame: StartFrame):
pass
@@ -158,7 +168,7 @@ class TTSService(AIService):
@abstractmethod
async def set_model(self, model: str):
pass
self.set_model_name(model)
@abstractmethod
async def set_voice(self, voice: str):
@@ -367,7 +377,7 @@ class STTService(AIService):
@abstractmethod
async def set_model(self, model: str):
pass
self.set_model_name(model)
@abstractmethod
async def set_language(self, language: Language):

23
src/pipecat/services/anthropic.py
View File

@@ -29,6 +29,7 @@ from pipecat.frames.frames import (
FunctionCallInProgressFrame,
StartInterruptionFrame
)
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import LLMService
from pipecat.processors.aggregators.openai_llm_context import (
@@ -84,7 +85,7 @@ class AnthropicLLMService(LLMService):
**kwargs):
super().__init__(**kwargs)
self._client = AsyncAnthropic(api_key=api_key)
self._model = model
self.set_model_name(model)
self._max_tokens = max_tokens
self._enable_prompt_caching_beta = enable_prompt_caching_beta
@@ -137,7 +138,7 @@ class AnthropicLLMService(LLMService):
tools=context.tools or [],
system=context.system,
messages=messages,
model=self._model,
model=self.model_name,
max_tokens=self._max_tokens,
stream=True)
@@ -231,7 +232,7 @@ class AnthropicLLMService(LLMService):
context = AnthropicLLMContext.from_image_frame(frame)
elif isinstance(frame, LLMModelUpdateFrame):
logger.debug(f"Switching LLM model to: [{frame.model}]")
self._model = frame.model
self.set_model_name(frame.model)
elif isinstance(frame, LLMEnablePromptCachingFrame):
logger.debug(f"Setting enable prompt caching to: [{frame.enable}]")
self._enable_prompt_caching_beta = frame.enable
@@ -251,15 +252,13 @@ class AnthropicLLMService(LLMService):
cache_creation_input_tokens: int,
cache_read_input_tokens: int):
if prompt_tokens or completion_tokens or cache_creation_input_tokens or cache_read_input_tokens:
tokens = {
"processor": self.name,
"model": self._model,
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"cache_creation_input_tokens": cache_creation_input_tokens,
"cache_read_input_tokens": cache_read_input_tokens,
"total_tokens": prompt_tokens + completion_tokens
}
tokens = LLMTokenUsage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
cache_creation_input_tokens=cache_creation_input_tokens,
cache_read_input_tokens=cache_read_input_tokens,
total_tokens=prompt_tokens + completion_tokens
)
await self.start_llm_usage_metrics(tokens)

4
src/pipecat/services/azure.py
View File

@@ -22,6 +22,8 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
TranscriptionFrame,
URLImageRawFrame)
from pipecat.metrics.metrics import TTSUsageMetricsData
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import STTService, TTSService, ImageGenService
from pipecat.services.openai import BaseOpenAILLMService
from pipecat.utils.time import time_now_iso8601
@@ -190,7 +192,7 @@ class AzureImageGenServiceREST(ImageGenService):
self._api_key = api_key
self._azure_endpoint = endpoint
self._api_version = api_version
self._model = model
self.set_model_name(model)
self._image_size = image_size
self._aiohttp_session = aiohttp_session

17
src/pipecat/services/cartesia.py
View File

@@ -89,7 +89,7 @@ class CartesiaTTSService(AsyncWordTTSService):
self._cartesia_version = cartesia_version
self._url = url
self._voice_id = voice_id
self._model_id = model_id
self.set_model_name(model_id)
self._output_format = {
"container": "raw",
"encoding": encoding,
@@ -105,8 +105,8 @@ class CartesiaTTSService(AsyncWordTTSService):
return True
async def set_model(self, model: str):
await super().set_model(model)
logger.debug(f"Switching TTS model to: [{model}]")
self._model_id = model
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
@@ -155,6 +155,11 @@ class CartesiaTTSService(AsyncWordTTSService):
except Exception as e:
logger.error(f"{self} error closing websocket: {e}")
def _get_websocket(self):
if self._websocket:
return self._websocket
raise Exception("Websocket not connected")
async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
await super()._handle_interruption(frame, direction)
await self.stop_all_metrics()
@@ -169,7 +174,7 @@ class CartesiaTTSService(AsyncWordTTSService):
"transcript": "",
"continue": False,
"context_id": self._context_id,
"model_id": self._model_id,
"model_id": self.model_name,
"voice": {
"mode": "id",
"id": self._voice_id
@@ -182,7 +187,7 @@ class CartesiaTTSService(AsyncWordTTSService):
async def _receive_task_handler(self):
try:
async for message in self._websocket:
async for message in self._get_websocket():
msg = json.loads(message)
if not msg or msg["context_id"] != self._context_id:
continue
@@ -235,7 +240,7 @@ class CartesiaTTSService(AsyncWordTTSService):
"transcript": text + " ",
"continue": True,
"context_id": self._context_id,
"model_id": self._model_id,
"model_id": self.model_name,
"voice": {
"mode": "id",
"id": self._voice_id
@@ -245,7 +250,7 @@ class CartesiaTTSService(AsyncWordTTSService):
"add_timestamps": True,
}
try:
await self._websocket.send(json.dumps(msg))
await self._get_websocket().send(json.dumps(msg))
await self.start_tts_usage_metrics(text)
except Exception as e:
logger.error(f"{self} error sending message: {e}")

1
src/pipecat/services/deepgram.py
View File

@@ -135,6 +135,7 @@ class DeepgramSTTService(STTService):
self._connection.on(LiveTranscriptionEvents.Transcript, self._on_message)
async def set_model(self, model: str):
await super().set_model(model)
logger.debug(f"Switching STT model to: [{model}]")
self._live_options.model = model
await self._disconnect()

6
src/pipecat/services/elevenlabs.py
View File

@@ -107,7 +107,7 @@ class ElevenLabsTTSService(AsyncWordTTSService):
self._api_key = api_key
self._voice_id = voice_id
self._model = model
self.set_model_name(model)
self._url = url
self._params = params
@@ -122,8 +122,8 @@ class ElevenLabsTTSService(AsyncWordTTSService):
return True
async def set_model(self, model: str):
await super().set_model(model)
logger.debug(f"Switching TTS model to: [{model}]")
self._model = model
await self._disconnect()
await self._connect()
@@ -160,7 +160,7 @@ class ElevenLabsTTSService(AsyncWordTTSService):
async def _connect(self):
try:
voice_id = self._voice_id
model = self._model
model = self.model_name
output_format = self._params.output_format
url = f"{self._url}/v1/text-to-speech/{voice_id}/stream-input?model_id={model}&output_format={output_format}"
self._websocket = await websockets.connect(url)

4
src/pipecat/services/fal.py
View File

@@ -46,7 +46,7 @@ class FalImageGenService(ImageGenService):
**kwargs
):
super().__init__(**kwargs)
self._model = model
self.set_model_name(model)
self._params = params
self._aiohttp_session = aiohttp_session
if key:
@@ -56,7 +56,7 @@ class FalImageGenService(ImageGenService):
logger.debug(f"Generating image from prompt: {prompt}")
response = await fal_client.run_async(
self._model,
self.model_name,
arguments={"prompt": prompt, **self._params.model_dump(exclude_none=True)}
)

2
src/pipecat/services/fireworks.py
View File

@@ -22,4 +22,4 @@ class FireworksLLMService(BaseOpenAILLMService):
*,
model: str = "accounts/fireworks/models/firefunction-v1",
base_url: str = "https://api.fireworks.ai/inference/v1"):
super().__init__(model, base_url)
super().__init__(model=model, base_url=base_url)

1
src/pipecat/services/google.py
View File

@@ -50,6 +50,7 @@ class GoogleLLMService(LLMService):
return True
def _create_client(self, model: str):
self.set_model_name(model)
self._client = gai.GenerativeModel(model)
def _get_messages_from_openai_context(

4
src/pipecat/services/moondream.py
View File

@@ -54,6 +54,8 @@ class MoondreamService(VisionService):
):
super().__init__(**kwargs)
self.set_model_name(model)
if not use_cpu:
device, dtype = detect_device()
else:
@@ -73,7 +75,7 @@ class MoondreamService(VisionService):
async def run_vision(self, frame: VisionImageRawFrame) -> AsyncGenerator[Frame, None]:
if not self._model:
logger.error(f"{self} error: Moondream model not available")
logger.error(f"{self} error: Moondream model not available ({self.model_name})")
yield ErrorFrame("Moondream model not available")
return

27
src/pipecat/services/openai.py
View File

@@ -33,6 +33,7 @@ from pipecat.frames.frames import (
FunctionCallInProgressFrame,
StartInterruptionFrame
)
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_response import LLMUserContextAggregator, LLMAssistantContextAggregator
from pipecat.processors.aggregators.openai_llm_context import (
@@ -83,7 +84,7 @@ class BaseOpenAILLMService(LLMService):
def __init__(self, *, model: str, api_key=None, base_url=None, **kwargs):
super().__init__(**kwargs)
self._model: str = model
self.set_model_name(model)
self._client = self.create_client(api_key=api_key, base_url=base_url, **kwargs)
def create_client(self, api_key=None, base_url=None, **kwargs):
@@ -104,7 +105,7 @@ class BaseOpenAILLMService(LLMService):
context: OpenAILLMContext,
messages: List[ChatCompletionMessageParam]) -> AsyncStream[ChatCompletionChunk]:
chunks = await self._client.chat.completions.create(
model=self._model,
model=self.model_name,
stream=True,
messages=messages,
tools=context.tools,
@@ -148,13 +149,11 @@ class BaseOpenAILLMService(LLMService):
async for chunk in chunk_stream:
if chunk.usage:
tokens = {
"processor": self.name,
"model": self._model,
"prompt_tokens": chunk.usage.prompt_tokens,
"completion_tokens": chunk.usage.completion_tokens,
"total_tokens": chunk.usage.total_tokens
}
tokens = LLMTokenUsage(
prompt_tokens=chunk.usage.prompt_tokens,
completion_tokens=chunk.usage.completion_tokens,
total_tokens=chunk.usage.total_tokens
)
await self.start_llm_usage_metrics(tokens)
if len(chunk.choices) == 0:
@@ -223,7 +222,7 @@ class BaseOpenAILLMService(LLMService):
context = OpenAILLMContext.from_image_frame(frame)
elif isinstance(frame, LLMModelUpdateFrame):
logger.debug(f"Switching LLM model to: [{frame.model}]")
self._model = frame.model
self.set_model_name(frame.model)
else:
await self.push_frame(frame, direction)
@@ -273,7 +272,7 @@ class OpenAIImageGenService(ImageGenService):
model: str = "dall-e-3",
):
super().__init__()
self._model = model
self.set_model_name(model)
self._image_size = image_size
self._client = AsyncOpenAI(api_key=api_key)
self._aiohttp_session = aiohttp_session
@@ -283,7 +282,7 @@ class OpenAIImageGenService(ImageGenService):
image = await self._client.images.generate(
prompt=prompt,
model=self._model,
model=self.model_name,
n=1,
size=self._image_size
)
@@ -325,7 +324,7 @@ class OpenAITTSService(TTSService):
super().__init__(sample_rate=sample_rate, **kwargs)
self._voice: ValidVoice = VALID_VOICES.get(voice, "alloy")
self._model = model
self.set_model_name(model)
self._sample_rate = sample_rate
self._client = AsyncOpenAI(api_key=api_key)
@@ -348,7 +347,7 @@ class OpenAITTSService(TTSService):
async with self._client.audio.speech.with_streaming_response.create(
input=text,
model=self._model,
model=self.model_name,
voice=self._voice,
response_format="pcm",
) as r:

2
src/pipecat/services/openpipe.py
View File

@@ -60,7 +60,7 @@ class OpenPipeLLMService(BaseOpenAILLMService):
context: OpenAILLMContext,
messages: List[ChatCompletionMessageParam]) -> AsyncStream[ChatCompletionChunk]:
chunks = await self._client.chat.completions.create(
model=self._model,
model=self.model_name,
stream=True,
messages=messages,
openpipe={

21
src/pipecat/services/together.py
View File

@@ -18,9 +18,7 @@ from pipecat.frames.frames import (
Frame,
LLMModelUpdateFrame,
TextFrame,
VisionImageRawFrame,
UserImageRequestFrame,
UserImageRawFrame,
LLMMessagesFrame,
LLMFullResponseStartFrame,
LLMFullResponseEndFrame,
@@ -28,6 +26,7 @@ from pipecat.frames.frames import (
FunctionCallInProgressFrame,
StartInterruptionFrame
)
from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import LLMService
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext, OpenAILLMContextFrame
@@ -69,7 +68,7 @@ class TogetherLLMService(LLMService):
**kwargs):
super().__init__(**kwargs)
self._client = AsyncTogether(api_key=api_key)
self._model = model
self.set_model_name(model)
self._max_tokens = max_tokens
def can_generate_metrics(self) -> bool:
@@ -95,7 +94,7 @@ class TogetherLLMService(LLMService):
stream = await self._client.chat.completions.create(
messages=context.messages,
model=self._model,
model=self.model_name,
max_tokens=self._max_tokens,
stream=True,
)
@@ -108,13 +107,11 @@ class TogetherLLMService(LLMService):
async for chunk in stream:
# logger.debug(f"Together LLM event: {chunk}")
if chunk.usage:
tokens = {
"processor": self.name,
"model": self._model,
"prompt_tokens": chunk.usage.prompt_tokens,
"completion_tokens": chunk.usage.completion_tokens,
"total_tokens": chunk.usage.total_tokens
}
tokens = LLMTokenUsage(
prompt_tokens=chunk.usage.prompt_tokens,
completion_tokens=chunk.usage.completion_tokens,
total_tokens=chunk.usage.total_tokens
)
await self.start_llm_usage_metrics(tokens)
if len(chunk.choices) == 0:
@@ -156,7 +153,7 @@ class TogetherLLMService(LLMService):
context = TogetherLLMContext.from_messages(frame.messages)
elif isinstance(frame, LLMModelUpdateFrame):
logger.debug(f"Switching LLM model to: [{frame.model}]")
self._model = frame.model
self.set_model_name(frame.model)
else:
await self.push_frame(frame, direction)

4
src/pipecat/services/whisper.py
View File

@@ -52,7 +52,7 @@ class WhisperSTTService(SegmentedSTTService):
super().__init__(**kwargs)
self._device: str = device
self._compute_type = compute_type
self._model_name: str | Model = model
self.set_model_name(model if isinstance(model, str) else model.value)
self._no_speech_prob = no_speech_prob
self._model: WhisperModel | None = None
self._load()
@@ -65,7 +65,7 @@ class WhisperSTTService(SegmentedSTTService):
this model is being run, it will take time to download."""
logger.debug("Loading Whisper model...")
self._model = WhisperModel(
self._model_name.value if isinstance(self._model_name, Enum) else self._model_name,
self.model_name,
device=self._device,
compute_type=self._compute_type)
logger.debug("Loaded Whisper model")

26
src/pipecat/transports/services/daily.py
View File

@@ -35,6 +35,7 @@ from pipecat.frames.frames import (
TransportMessageFrame,
UserImageRawFrame,
UserImageRequestFrame)
from pipecat.metrics.metrics import LLMUsageMetricsData, ProcessingMetricsData, TTFBMetricsData, TTSUsageMetricsData
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.transcriptions.language import Language
from pipecat.transports.base_input import BaseInputTransport
@@ -731,14 +732,23 @@ class DailyOutputTransport(BaseOutputTransport):
async def send_metrics(self, frame: MetricsFrame):
metrics = {}
if frame.ttfb:
metrics["ttfb"] = frame.ttfb
if frame.processing:
metrics["processing"] = frame.processing
if frame.tokens:
metrics["tokens"] = frame.tokens
if frame.characters:
metrics["characters"] = frame.characters
for d in frame.data:
if isinstance(d, TTFBMetricsData):
if "ttfb" not in metrics:
metrics["ttfb"] = []
metrics["ttfb"].append(d.model_dump())
elif isinstance(d, ProcessingMetricsData):
if "processing" not in metrics:
metrics["processing"] = []
metrics["processing"].append(d.model_dump())
elif isinstance(d, LLMUsageMetricsData):
if "tokens" not in metrics:
metrics["tokens"] = []
metrics["tokens"].append(d.value.model_dump(exclude_none=True))
elif isinstance(d, TTSUsageMetricsData):
if "characters" not in metrics:
metrics["characters"] = []
metrics["characters"].append(d.model_dump())
message = DailyTransportMessageFrame(message={
"type": "pipecat-metrics",