Cleanup on aisle METRICS. Note: See below, this is a breaking change

1. Fleshed out MetricsFrames and broke it into a proper set of types
2. Add model_name as a property to the AIService so that it can be
   automatically included in metrics and also remove that
   overhead from all the various services themselves

Breaking change!

Because of the types improvements, the MetricsFrame type has
changed. Each frame will have a list of metrics simlilar to before
except each item in the list will only contain one type of metric:
"ttfb", "tokens", "characters", or "processing". Previously these
fields would be in every entry but set to None if they didn't apply.

While this changes internal handling of the MetricsFrame, it does NOT
break the RTVI/daily messaging of metrics. That format remains the same.

Also. Remember to use model_name for accessing a service's current
model and set_model_name for setting it.

examples/foundational/06-listen-and-respond.py

@@ -10,6 +10,7 @@ import os
 import sys
 
 from pipecat.frames.frames import Frame, LLMMessagesFrame, MetricsFrame
+from pipecat.metrics.metrics import TTFBMetricsData, ProcessingMetricsData, LLMUsageMetricsData, TTSUsageMetricsData
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineParams, PipelineTask
@@ -37,8 +38,19 @@ logger.add(sys.stderr, level="DEBUG")
 class MetricsLogger(FrameProcessor):
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         if isinstance(frame, MetricsFrame):
-            print(
+            for d in frame.data:
-                f"!!! MetricsFrame: {frame}, ttfb: {frame.ttfb}, processing: {frame.processing}, tokens: {frame.tokens}, characters: {frame.characters}")
+                if isinstance(d, TTFBMetricsData):
+                    print(f"!!! MetricsFrame: {frame}, ttfb: {d.value}")
+                elif isinstance(d, ProcessingMetricsData):
+                    print(f"!!! MetricsFrame: {frame}, processing: {d.value}")
+                elif isinstance(d, LLMUsageMetricsData):
+                    tokens = d.value
+                    print(
+                        f"!!! MetricsFrame: {frame}, tokens: {
+                            tokens.prompt_tokens}, characters: {
+                            tokens.completion_tokens}")
+                elif isinstance(d, TTSUsageMetricsData):
+                    print(f"!!! MetricsFrame: {frame}, characters: {d.value}")
         await self.push_frame(frame, direction)
 
 

src/pipecat/frames/frames.py

@@ -4,11 +4,12 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-from typing import Any, List, Mapping, Optional, Tuple
+from typing import Any, List, Optional, Tuple
 
 from dataclasses import dataclass, field
 
 from pipecat.clocks.base_clock import BaseClock
+from pipecat.metrics.metrics import MetricsData
 from pipecat.transcriptions.language import Language
 from pipecat.utils.time import nanoseconds_to_str
 from pipecat.utils.utils import obj_count, obj_id
@@ -333,10 +334,8 @@ class BotInterruptionFrame(SystemFrame):
 class MetricsFrame(SystemFrame):
     """Emitted by processor that can compute metrics like latencies.
     """
-    ttfb: List[Mapping[str, Any]] | None = None
+    data: List[MetricsData]
-    processing: List[Mapping[str, Any]] | None = None
+
-    tokens: List[Mapping[str, Any]] | None = None
-    characters: List[Mapping[str, Any]] | None = None
 
 #
 # Control frames

src/pipecat/metrics/metrics.py

@@ -0,0 +1,31 @@
+from typing import Optional
+from pydantic import BaseModel
+
+
+class MetricsData(BaseModel):
+    processor: str
+    model: Optional[str] = None
+
+
+class TTFBMetricsData(MetricsData):
+    value: float
+
+
+class ProcessingMetricsData(MetricsData):
+    value: float
+
+
+class LLMTokenUsage(BaseModel):
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
+    cache_read_input_tokens: Optional[int] = None
+    cache_creation_input_tokens: Optional[int] = None
+
+
+class LLMUsageMetricsData(MetricsData):
+    value: LLMTokenUsage
+
+
+class TTSUsageMetricsData(MetricsData):
+    value: int

src/pipecat/pipeline/task.py

@@ -20,6 +20,7 @@ from pipecat.frames.frames import (
     MetricsFrame,
     StartFrame,
     StopTaskFrame)
+from pipecat.metrics.metrics import TTFBMetricsData, ProcessingMetricsData
 from pipecat.pipeline.base_pipeline import BasePipeline
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 from pipecat.utils.utils import obj_count, obj_id
@@ -118,9 +119,11 @@ class PipelineTask:
 
     def _initial_metrics_frame(self) -> MetricsFrame:
         processors = self._pipeline.processors_with_metrics()
-        ttfb = [{"processor": p.name, "value": 0.0} for p in processors]
+        data = []
-        processing = [{"processor": p.name, "value": 0.0} for p in processors]
+        for p in processors:
-        return MetricsFrame(ttfb=ttfb, processing=processing)
+            data.append(TTFBMetricsData(processor=p.name, value=0.0))
+            data.append(ProcessingMetricsData(processor=p.name, value=0.0))
+        return MetricsFrame(data=data)
 
     async def _process_down_queue(self):
         self._clock.start()

src/pipecat/processors/frame_processor.py

@@ -19,6 +19,13 @@ from pipecat.frames.frames import (
     StartInterruptionFrame,
     StopInterruptionFrame,
     SystemFrame)
+from pipecat.metrics.metrics import (
+    LLMTokenUsage,
+    LLMUsageMetricsData,
+    MetricsData,
+    ProcessingMetricsData,
+    TTFBMetricsData,
+    TTSUsageMetricsData)
 from pipecat.utils.utils import obj_count, obj_id
 
 from loguru import logger
@@ -31,11 +38,20 @@ class FrameDirection(Enum):
 
 class FrameProcessorMetrics:
     def __init__(self, name: str):
-        self._name = name
+        self._core_metrics_data = MetricsData(processor=name)
         self._start_ttfb_time = 0
         self._start_processing_time = 0
         self._should_report_ttfb = True
 
+    def _processor_name(self):
+        return self._core_metrics_data.processor
+
+    def _model_name(self):
+        return self._core_metrics_data.model
+
+    def set_core_metrics_data(self, data: MetricsData):
+        self._core_metrics_data = data
+
     async def start_ttfb_metrics(self, report_only_initial_ttfb):
         if self._should_report_ttfb:
             self._start_ttfb_time = time.time()
@@ -46,13 +62,13 @@ class FrameProcessorMetrics:
             return None
 
         value = time.time() - self._start_ttfb_time
-        logger.debug(f"{self._name} TTFB: {value}")
+        logger.debug(f"{self._processor_name()} TTFB: {value}")
-        ttfb = {
+        ttfb = TTFBMetricsData(
-            "processor": self._name,
+            processor=self._processor_name(),
-            "value": value
+            value=value,
-        }
+            model=self._model_name())
         self._start_ttfb_time = 0
-        return MetricsFrame(ttfb=[ttfb])
+        return MetricsFrame(data=[ttfb])
 
     async def start_processing_metrics(self):
         self._start_processing_time = time.time()
@@ -62,26 +78,28 @@ class FrameProcessorMetrics:
             return None
 
         value = time.time() - self._start_processing_time
-        logger.debug(f"{self._name} processing time: {value}")
+        logger.debug(f"{self._processor_name()} processing time: {value}")
-        processing = {
+        processing = ProcessingMetricsData(
-            "processor": self._name,
+            processor=self._processor_name(), value=value, model=self._model_name())
-            "value": value
-        }
         self._start_processing_time = 0
-        return MetricsFrame(processing=[processing])
+        return MetricsFrame(data=[processing])
 
-    async def start_llm_usage_metrics(self, tokens: dict):
+    async def start_llm_usage_metrics(self, tokens: LLMTokenUsage):
         logger.debug(
-            f"{self._name} prompt tokens: {tokens['prompt_tokens']}, completion tokens: {tokens['completion_tokens']}")
+            f"{self._processor_name()} prompt tokens: {tokens.prompt_tokens}, completion tokens: {tokens.completion_tokens}")
-        return MetricsFrame(tokens=[tokens])
+        value = LLMUsageMetricsData(
+            processor=self._processor_name(),
+            model=self._model_name(),
+            value=tokens)
+        return MetricsFrame(data=[value])
 
     async def start_tts_usage_metrics(self, text: str):
-        characters = {
+        characters = TTSUsageMetricsData(
-            "processor": self._name,
+            processor=self._processor_name(),
-            "value": len(text),
+            model=self._model_name(),
-        }
+            value=len(text))
-        logger.debug(f"{self._name} usage characters: {characters['value']}")
+        logger.debug(f"{self._processor_name()} usage characters: {characters.value}")
-        return MetricsFrame(characters=[characters])
+        return MetricsFrame(data=[characters])
 
 
 class FrameProcessor:
@@ -140,6 +158,9 @@ class FrameProcessor:
     def can_generate_metrics(self) -> bool:
         return False
 
+    def set_core_metrics_data(self, data: MetricsData):
+        self._metrics.set_core_metrics_data(data)
+
     async def start_ttfb_metrics(self):
         if self.can_generate_metrics() and self.metrics_enabled:
             await self._metrics.start_ttfb_metrics(self._report_only_initial_ttfb)
@@ -160,7 +181,7 @@ class FrameProcessor:
             if frame:
                 await self.push_frame(frame)
 
-    async def start_llm_usage_metrics(self, tokens: dict):
+    async def start_llm_usage_metrics(self, tokens: LLMTokenUsage):
         if self.can_generate_metrics() and self.usage_metrics_enabled:
             frame = await self._metrics.start_llm_usage_metrics(tokens)
             if frame:

src/pipecat/services/ai_services.py

@@ -32,6 +32,7 @@ from pipecat.frames.frames import (
     UserImageRequestFrame,
     VisionImageRawFrame
 )
+from pipecat.metrics.metrics import MetricsData
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 from pipecat.transcriptions.language import Language
 from pipecat.utils.audio import calculate_audio_volume
@@ -46,6 +47,15 @@ from loguru import logger
 class AIService(FrameProcessor):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+        self._model_name: str = ""
+
+    @property
+    def model_name(self) -> str:
+        return self._model_name
+
+    def set_model_name(self, model: str):
+        self._model_name = model
+        self.set_core_metrics_data(MetricsData(processor=self.name, model=self._model_name))
 
     async def start(self, frame: StartFrame):
         pass
@@ -158,7 +168,7 @@ class TTSService(AIService):
 
     @abstractmethod
     async def set_model(self, model: str):
-        pass
+        self.set_model_name(model)
 
     @abstractmethod
     async def set_voice(self, voice: str):
@@ -367,7 +377,7 @@ class STTService(AIService):
 
     @abstractmethod
     async def set_model(self, model: str):
-        pass
+        self.set_model_name(model)
 
     @abstractmethod
     async def set_language(self, language: Language):

src/pipecat/services/anthropic.py

@@ -29,6 +29,7 @@ from pipecat.frames.frames import (
     FunctionCallInProgressFrame,
     StartInterruptionFrame
 )
+from pipecat.metrics.metrics import LLMTokenUsage
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.ai_services import LLMService
 from pipecat.processors.aggregators.openai_llm_context import (
@@ -84,7 +85,7 @@ class AnthropicLLMService(LLMService):
             **kwargs):
         super().__init__(**kwargs)
         self._client = AsyncAnthropic(api_key=api_key)
-        self._model = model
+        self.set_model_name(model)
         self._max_tokens = max_tokens
         self._enable_prompt_caching_beta = enable_prompt_caching_beta
 
@@ -137,7 +138,7 @@ class AnthropicLLMService(LLMService):
                 tools=context.tools or [],
                 system=context.system,
                 messages=messages,
-                model=self._model,
+                model=self.model_name,
                 max_tokens=self._max_tokens,
                 stream=True)
 
@@ -231,7 +232,7 @@ class AnthropicLLMService(LLMService):
             context = AnthropicLLMContext.from_image_frame(frame)
         elif isinstance(frame, LLMModelUpdateFrame):
             logger.debug(f"Switching LLM model to: [{frame.model}]")
-            self._model = frame.model
+            self.set_model_name(frame.model)
         elif isinstance(frame, LLMEnablePromptCachingFrame):
             logger.debug(f"Setting enable prompt caching to: [{frame.enable}]")
             self._enable_prompt_caching_beta = frame.enable
@@ -251,15 +252,13 @@ class AnthropicLLMService(LLMService):
             cache_creation_input_tokens: int,
             cache_read_input_tokens: int):
         if prompt_tokens or completion_tokens or cache_creation_input_tokens or cache_read_input_tokens:
-            tokens = {
+            tokens = LLMTokenUsage(
-                "processor": self.name,
+                prompt_tokens=prompt_tokens,
-                "model": self._model,
+                completion_tokens=completion_tokens,
-                "prompt_tokens": prompt_tokens,
+                cache_creation_input_tokens=cache_creation_input_tokens,
-                "completion_tokens": completion_tokens,
+                cache_read_input_tokens=cache_read_input_tokens,
-                "cache_creation_input_tokens": cache_creation_input_tokens,
+                total_tokens=prompt_tokens + completion_tokens
-                "cache_read_input_tokens": cache_read_input_tokens,
+            )
-                "total_tokens": prompt_tokens + completion_tokens
-            }
             await self.start_llm_usage_metrics(tokens)
 
 

src/pipecat/services/azure.py

@@ -22,6 +22,8 @@ from pipecat.frames.frames import (
     TTSStoppedFrame,
     TranscriptionFrame,
     URLImageRawFrame)
+from pipecat.metrics.metrics import TTSUsageMetricsData
+from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.ai_services import STTService, TTSService, ImageGenService
 from pipecat.services.openai import BaseOpenAILLMService
 from pipecat.utils.time import time_now_iso8601
@@ -190,7 +192,7 @@ class AzureImageGenServiceREST(ImageGenService):
         self._api_key = api_key
         self._azure_endpoint = endpoint
         self._api_version = api_version
-        self._model = model
+        self.set_model_name(model)
         self._image_size = image_size
         self._aiohttp_session = aiohttp_session
 

src/pipecat/services/cartesia.py

@@ -89,7 +89,7 @@ class CartesiaTTSService(AsyncWordTTSService):
         self._cartesia_version = cartesia_version
         self._url = url
         self._voice_id = voice_id
-        self._model_id = model_id
+        self.set_model_name(model_id)
         self._output_format = {
             "container": "raw",
             "encoding": encoding,
@@ -105,8 +105,8 @@ class CartesiaTTSService(AsyncWordTTSService):
         return True
 
     async def set_model(self, model: str):
+        await super().set_model(model)
         logger.debug(f"Switching TTS model to: [{model}]")
-        self._model_id = model
 
     async def set_voice(self, voice: str):
         logger.debug(f"Switching TTS voice to: [{voice}]")
@@ -155,6 +155,11 @@ class CartesiaTTSService(AsyncWordTTSService):
         except Exception as e:
             logger.error(f"{self} error closing websocket: {e}")
 
+    def _get_websocket(self):
+        if self._websocket:
+            return self._websocket
+        raise Exception("Websocket not connected")
+
     async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
         await super()._handle_interruption(frame, direction)
         await self.stop_all_metrics()
@@ -169,7 +174,7 @@ class CartesiaTTSService(AsyncWordTTSService):
             "transcript": "",
             "continue": False,
             "context_id": self._context_id,
-            "model_id": self._model_id,
+            "model_id": self.model_name,
             "voice": {
                 "mode": "id",
                 "id": self._voice_id
@@ -182,7 +187,7 @@ class CartesiaTTSService(AsyncWordTTSService):
 
     async def _receive_task_handler(self):
         try:
-            async for message in self._websocket:
+            async for message in self._get_websocket():
                 msg = json.loads(message)
                 if not msg or msg["context_id"] != self._context_id:
                     continue
@@ -235,7 +240,7 @@ class CartesiaTTSService(AsyncWordTTSService):
                 "transcript": text + " ",
                 "continue": True,
                 "context_id": self._context_id,
-                "model_id": self._model_id,
+                "model_id": self.model_name,
                 "voice": {
                     "mode": "id",
                     "id": self._voice_id
@@ -245,7 +250,7 @@ class CartesiaTTSService(AsyncWordTTSService):
                 "add_timestamps": True,
             }
             try:
-                await self._websocket.send(json.dumps(msg))
+                await self._get_websocket().send(json.dumps(msg))
                 await self.start_tts_usage_metrics(text)
             except Exception as e:
                 logger.error(f"{self} error sending message: {e}")

src/pipecat/services/deepgram.py

@@ -135,6 +135,7 @@ class DeepgramSTTService(STTService):
         self._connection.on(LiveTranscriptionEvents.Transcript, self._on_message)
 
     async def set_model(self, model: str):
+        await super().set_model(model)
         logger.debug(f"Switching STT model to: [{model}]")
         self._live_options.model = model
         await self._disconnect()

src/pipecat/services/elevenlabs.py

@@ -107,7 +107,7 @@ class ElevenLabsTTSService(AsyncWordTTSService):
 
         self._api_key = api_key
         self._voice_id = voice_id
-        self._model = model
+        self.set_model_name(model)
         self._url = url
         self._params = params
 
@@ -122,8 +122,8 @@ class ElevenLabsTTSService(AsyncWordTTSService):
         return True
 
     async def set_model(self, model: str):
+        await super().set_model(model)
         logger.debug(f"Switching TTS model to: [{model}]")
-        self._model = model
         await self._disconnect()
         await self._connect()
 
@@ -160,7 +160,7 @@ class ElevenLabsTTSService(AsyncWordTTSService):
     async def _connect(self):
         try:
             voice_id = self._voice_id
-            model = self._model
+            model = self.model_name
             output_format = self._params.output_format
             url = f"{self._url}/v1/text-to-speech/{voice_id}/stream-input?model_id={model}&output_format={output_format}"
             self._websocket = await websockets.connect(url)

src/pipecat/services/fal.py

@@ -46,7 +46,7 @@ class FalImageGenService(ImageGenService):
         **kwargs
     ):
         super().__init__(**kwargs)
-        self._model = model
+        self.set_model_name(model)
         self._params = params
         self._aiohttp_session = aiohttp_session
         if key:
@@ -56,7 +56,7 @@ class FalImageGenService(ImageGenService):
         logger.debug(f"Generating image from prompt: {prompt}")
 
         response = await fal_client.run_async(
-            self._model,
+            self.model_name,
             arguments={"prompt": prompt, **self._params.model_dump(exclude_none=True)}
         )
 

src/pipecat/services/fireworks.py

@@ -22,4 +22,4 @@ class FireworksLLMService(BaseOpenAILLMService):
                  *,
                  model: str = "accounts/fireworks/models/firefunction-v1",
                  base_url: str = "https://api.fireworks.ai/inference/v1"):
-        super().__init__(model, base_url)
+        super().__init__(model=model, base_url=base_url)

src/pipecat/services/google.py

@@ -50,6 +50,7 @@ class GoogleLLMService(LLMService):
         return True
 
     def _create_client(self, model: str):
+        self.set_model_name(model)
         self._client = gai.GenerativeModel(model)
 
     def _get_messages_from_openai_context(

src/pipecat/services/moondream.py

@@ -54,6 +54,8 @@ class MoondreamService(VisionService):
     ):
         super().__init__(**kwargs)
 
+        self.set_model_name(model)
+
         if not use_cpu:
             device, dtype = detect_device()
         else:
@@ -73,7 +75,7 @@ class MoondreamService(VisionService):
 
     async def run_vision(self, frame: VisionImageRawFrame) -> AsyncGenerator[Frame, None]:
         if not self._model:
-            logger.error(f"{self} error: Moondream model not available")
+            logger.error(f"{self} error: Moondream model not available ({self.model_name})")
             yield ErrorFrame("Moondream model not available")
             return
 

src/pipecat/services/openai.py

@@ -33,6 +33,7 @@ from pipecat.frames.frames import (
     FunctionCallInProgressFrame,
     StartInterruptionFrame
 )
+from pipecat.metrics.metrics import LLMTokenUsage
 from pipecat.processors.aggregators.llm_response import LLMUserContextAggregator, LLMAssistantContextAggregator
 
 from pipecat.processors.aggregators.openai_llm_context import (
@@ -83,7 +84,7 @@ class BaseOpenAILLMService(LLMService):
 
     def __init__(self, *, model: str, api_key=None, base_url=None, **kwargs):
         super().__init__(**kwargs)
-        self._model: str = model
+        self.set_model_name(model)
         self._client = self.create_client(api_key=api_key, base_url=base_url, **kwargs)
 
     def create_client(self, api_key=None, base_url=None, **kwargs):
@@ -104,7 +105,7 @@ class BaseOpenAILLMService(LLMService):
             context: OpenAILLMContext,
             messages: List[ChatCompletionMessageParam]) -> AsyncStream[ChatCompletionChunk]:
         chunks = await self._client.chat.completions.create(
-            model=self._model,
+            model=self.model_name,
             stream=True,
             messages=messages,
             tools=context.tools,
@@ -148,13 +149,11 @@ class BaseOpenAILLMService(LLMService):
 
         async for chunk in chunk_stream:
             if chunk.usage:
-                tokens = {
+                tokens = LLMTokenUsage(
-                    "processor": self.name,
+                    prompt_tokens=chunk.usage.prompt_tokens,
-                    "model": self._model,
+                    completion_tokens=chunk.usage.completion_tokens,
-                    "prompt_tokens": chunk.usage.prompt_tokens,
+                    total_tokens=chunk.usage.total_tokens
-                    "completion_tokens": chunk.usage.completion_tokens,
+                )
-                    "total_tokens": chunk.usage.total_tokens
-                }
                 await self.start_llm_usage_metrics(tokens)
 
             if len(chunk.choices) == 0:
@@ -223,7 +222,7 @@ class BaseOpenAILLMService(LLMService):
             context = OpenAILLMContext.from_image_frame(frame)
         elif isinstance(frame, LLMModelUpdateFrame):
             logger.debug(f"Switching LLM model to: [{frame.model}]")
-            self._model = frame.model
+            self.set_model_name(frame.model)
         else:
             await self.push_frame(frame, direction)
 
@@ -273,7 +272,7 @@ class OpenAIImageGenService(ImageGenService):
         model: str = "dall-e-3",
     ):
         super().__init__()
-        self._model = model
+        self.set_model_name(model)
         self._image_size = image_size
         self._client = AsyncOpenAI(api_key=api_key)
         self._aiohttp_session = aiohttp_session
@@ -283,7 +282,7 @@ class OpenAIImageGenService(ImageGenService):
 
         image = await self._client.images.generate(
             prompt=prompt,
-            model=self._model,
+            model=self.model_name,
             n=1,
             size=self._image_size
         )
@@ -325,7 +324,7 @@ class OpenAITTSService(TTSService):
         super().__init__(sample_rate=sample_rate, **kwargs)
 
         self._voice: ValidVoice = VALID_VOICES.get(voice, "alloy")
-        self._model = model
+        self.set_model_name(model)
         self._sample_rate = sample_rate
 
         self._client = AsyncOpenAI(api_key=api_key)
@@ -348,7 +347,7 @@ class OpenAITTSService(TTSService):
 
             async with self._client.audio.speech.with_streaming_response.create(
                     input=text,
-                    model=self._model,
+                    model=self.model_name,
                     voice=self._voice,
                     response_format="pcm",
             ) as r:

src/pipecat/services/openpipe.py

@@ -60,7 +60,7 @@ class OpenPipeLLMService(BaseOpenAILLMService):
             context: OpenAILLMContext,
             messages: List[ChatCompletionMessageParam]) -> AsyncStream[ChatCompletionChunk]:
         chunks = await self._client.chat.completions.create(
-            model=self._model,
+            model=self.model_name,
             stream=True,
             messages=messages,
             openpipe={

src/pipecat/services/together.py

@@ -18,9 +18,7 @@ from pipecat.frames.frames import (
     Frame,
     LLMModelUpdateFrame,
     TextFrame,
-    VisionImageRawFrame,
     UserImageRequestFrame,
-    UserImageRawFrame,
     LLMMessagesFrame,
     LLMFullResponseStartFrame,
     LLMFullResponseEndFrame,
@@ -28,6 +26,7 @@ from pipecat.frames.frames import (
     FunctionCallInProgressFrame,
     StartInterruptionFrame
 )
+from pipecat.metrics.metrics import LLMTokenUsage
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.ai_services import LLMService
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext, OpenAILLMContextFrame
@@ -69,7 +68,7 @@ class TogetherLLMService(LLMService):
             **kwargs):
         super().__init__(**kwargs)
         self._client = AsyncTogether(api_key=api_key)
-        self._model = model
+        self.set_model_name(model)
         self._max_tokens = max_tokens
 
     def can_generate_metrics(self) -> bool:
@@ -95,7 +94,7 @@ class TogetherLLMService(LLMService):
 
             stream = await self._client.chat.completions.create(
                 messages=context.messages,
-                model=self._model,
+                model=self.model_name,
                 max_tokens=self._max_tokens,
                 stream=True,
             )
@@ -108,13 +107,11 @@ class TogetherLLMService(LLMService):
             async for chunk in stream:
                 # logger.debug(f"Together LLM event: {chunk}")
                 if chunk.usage:
-                    tokens = {
+                    tokens = LLMTokenUsage(
-                        "processor": self.name,
+                        prompt_tokens=chunk.usage.prompt_tokens,
-                        "model": self._model,
+                        completion_tokens=chunk.usage.completion_tokens,
-                        "prompt_tokens": chunk.usage.prompt_tokens,
+                        total_tokens=chunk.usage.total_tokens
-                        "completion_tokens": chunk.usage.completion_tokens,
+                    )
-                        "total_tokens": chunk.usage.total_tokens
-                    }
                     await self.start_llm_usage_metrics(tokens)
 
                 if len(chunk.choices) == 0:
@@ -156,7 +153,7 @@ class TogetherLLMService(LLMService):
             context = TogetherLLMContext.from_messages(frame.messages)
         elif isinstance(frame, LLMModelUpdateFrame):
             logger.debug(f"Switching LLM model to: [{frame.model}]")
-            self._model = frame.model
+            self.set_model_name(frame.model)
         else:
             await self.push_frame(frame, direction)
 

src/pipecat/services/whisper.py

@@ -52,7 +52,7 @@ class WhisperSTTService(SegmentedSTTService):
         super().__init__(**kwargs)
         self._device: str = device
         self._compute_type = compute_type
-        self._model_name: str | Model = model
+        self.set_model_name(model if isinstance(model, str) else model.value)
         self._no_speech_prob = no_speech_prob
         self._model: WhisperModel | None = None
         self._load()
@@ -65,7 +65,7 @@ class WhisperSTTService(SegmentedSTTService):
         this model is being run, it will take time to download."""
         logger.debug("Loading Whisper model...")
         self._model = WhisperModel(
-            self._model_name.value if isinstance(self._model_name, Enum) else self._model_name,
+            self.model_name,
             device=self._device,
             compute_type=self._compute_type)
         logger.debug("Loaded Whisper model")

src/pipecat/transports/services/daily.py

@@ -35,6 +35,7 @@ from pipecat.frames.frames import (
     TransportMessageFrame,
     UserImageRawFrame,
     UserImageRequestFrame)
+from pipecat.metrics.metrics import LLMUsageMetricsData, ProcessingMetricsData, TTFBMetricsData, TTSUsageMetricsData
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 from pipecat.transcriptions.language import Language
 from pipecat.transports.base_input import BaseInputTransport
@@ -731,14 +732,23 @@ class DailyOutputTransport(BaseOutputTransport):
 
     async def send_metrics(self, frame: MetricsFrame):
         metrics = {}
-        if frame.ttfb:
+        for d in frame.data:
-            metrics["ttfb"] = frame.ttfb
+            if isinstance(d, TTFBMetricsData):
-        if frame.processing:
+                if "ttfb" not in metrics:
-            metrics["processing"] = frame.processing
+                    metrics["ttfb"] = []
-        if frame.tokens:
+                metrics["ttfb"].append(d.model_dump())
-            metrics["tokens"] = frame.tokens
+            elif isinstance(d, ProcessingMetricsData):
-        if frame.characters:
+                if "processing" not in metrics:
-            metrics["characters"] = frame.characters
+                    metrics["processing"] = []
+                metrics["processing"].append(d.model_dump())
+            elif isinstance(d, LLMUsageMetricsData):
+                if "tokens" not in metrics:
+                    metrics["tokens"] = []
+                metrics["tokens"].append(d.value.model_dump(exclude_none=True))
+            elif isinstance(d, TTSUsageMetricsData):
+                if "characters" not in metrics:
+                    metrics["characters"] = []
+                metrics["characters"].append(d.model_dump())
 
         message = DailyTransportMessageFrame(message={
             "type": "pipecat-metrics",