Change Gemini Live to use Gemini provided usage metrics

CHANGELOG.md

@@ -53,8 +53,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
-- `GeminiMultimodalLiveLLMService` now uses the user transcription provided by
-  Gemini Live.
+- `GeminiMultimodalLiveLLMService` now uses the user transcription and usage
+  metrics provided by Gemini Live.
 
 - `GoogleLLMService` has been updated to use `google-genai` instead of the
   deprecated `google-generativeai`.

src/pipecat/services/gemini_multimodal_live/events.py

@@ -182,10 +182,43 @@ class ToolCall(BaseModel):
     functionCalls: List[FunctionCall]
 
 
+class Modality(str, Enum):
+    """Modality types in token counts."""
+
+    UNSPECIFIED = "MODALITY_UNSPECIFIED"
+    TEXT = "TEXT"
+    IMAGE = "IMAGE"
+    AUDIO = "AUDIO"
+    VIDEO = "VIDEO"
+
+
+class ModalityTokenCount(BaseModel):
+    """Token count for a specific modality."""
+
+    modality: Modality
+    tokenCount: int
+
+
+class UsageMetadata(BaseModel):
+    """Usage metadata about the response."""
+
+    promptTokenCount: Optional[int] = None
+    cachedContentTokenCount: Optional[int] = None
+    responseTokenCount: Optional[int] = None
+    toolUsePromptTokenCount: Optional[int] = None
+    thoughtsTokenCount: Optional[int] = None
+    totalTokenCount: Optional[int] = None
+    promptTokensDetails: Optional[List[ModalityTokenCount]] = None
+    cacheTokensDetails: Optional[List[ModalityTokenCount]] = None
+    responseTokensDetails: Optional[List[ModalityTokenCount]] = None
+    toolUsePromptTokensDetails: Optional[List[ModalityTokenCount]] = None
+
+
 class ServerEvent(BaseModel):
     setupComplete: Optional[SetupComplete] = None
     serverContent: Optional[ServerContent] = None
     toolCall: Optional[ToolCall] = None
+    usageMetadata: Optional[UsageMetadata] = None
 
 
 def parse_server_event(str):

src/pipecat/services/gemini_multimodal_live/gemini.py

@@ -663,8 +663,9 @@ class GeminiMultimodalLiveLLMService(LLMService):
                 await self._handle_evt_setup_complete(evt)
             elif evt.serverContent and evt.serverContent.modelTurn:
                 await self._handle_evt_model_turn(evt)
-            elif evt.serverContent and evt.serverContent.turnComplete:
+            elif evt.serverContent and evt.serverContent.turnComplete and evt.usageMetadata:
                 await self._handle_evt_turn_complete(evt)
+                await self._handle_evt_usage_metadata(evt)
             elif evt.serverContent and evt.serverContent.inputTranscription:
                 await self._handle_evt_input_transcription(evt)
             elif evt.serverContent and evt.serverContent.outputTranscription:
@@ -858,6 +859,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
             )
 
     async def _handle_evt_turn_complete(self, evt):
+        print(f"GeminiMultimodalLiveLLMService: _handle_evt_turn_complete: {evt}")
         self._bot_is_speaking = False
         text = self._bot_text_buffer
         self._bot_text_buffer = ""
@@ -927,6 +929,19 @@ class GeminiMultimodalLiveLLMService(LLMService):
         await self.push_frame(LLMTextFrame(text=text))
         await self.push_frame(TTSTextFrame(text=text))
 
+    async def _handle_evt_usage_metadata(self, evt):
+        if not evt.usageMetadata:
+            return
+
+        usage = evt.usageMetadata
+
+        tokens = LLMTokenUsage(
+            prompt_tokens=usage.promptTokenCount,
+            completion_tokens=usage.responseTokenCount,
+            total_tokens=usage.totalTokenCount,
+        )
+        await self.start_llm_usage_metrics(tokens)
+
     def create_context_aggregator(
         self,
         context: OpenAILLMContext,