diff --git a/CHANGELOG.md b/CHANGELOG.md index fb6891905..bd0a3fe5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,8 +53,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- `GeminiMultimodalLiveLLMService` now uses the user transcription provided by - Gemini Live. +- `GeminiMultimodalLiveLLMService` now uses the user transcription and usage + metrics provided by Gemini Live. - `GoogleLLMService` has been updated to use `google-genai` instead of the deprecated `google-generativeai`. diff --git a/src/pipecat/services/gemini_multimodal_live/events.py b/src/pipecat/services/gemini_multimodal_live/events.py index d06c28325..97f7787a6 100644 --- a/src/pipecat/services/gemini_multimodal_live/events.py +++ b/src/pipecat/services/gemini_multimodal_live/events.py @@ -182,10 +182,43 @@ class ToolCall(BaseModel): functionCalls: List[FunctionCall] +class Modality(str, Enum): + """Modality types in token counts.""" + + UNSPECIFIED = "MODALITY_UNSPECIFIED" + TEXT = "TEXT" + IMAGE = "IMAGE" + AUDIO = "AUDIO" + VIDEO = "VIDEO" + + +class ModalityTokenCount(BaseModel): + """Token count for a specific modality.""" + + modality: Modality + tokenCount: int + + +class UsageMetadata(BaseModel): + """Usage metadata about the response.""" + + promptTokenCount: Optional[int] = None + cachedContentTokenCount: Optional[int] = None + responseTokenCount: Optional[int] = None + toolUsePromptTokenCount: Optional[int] = None + thoughtsTokenCount: Optional[int] = None + totalTokenCount: Optional[int] = None + promptTokensDetails: Optional[List[ModalityTokenCount]] = None + cacheTokensDetails: Optional[List[ModalityTokenCount]] = None + responseTokensDetails: Optional[List[ModalityTokenCount]] = None + toolUsePromptTokensDetails: Optional[List[ModalityTokenCount]] = None + + class ServerEvent(BaseModel): setupComplete: Optional[SetupComplete] = None serverContent: Optional[ServerContent] = None toolCall: Optional[ToolCall] = None + usageMetadata: Optional[UsageMetadata] = None def parse_server_event(str): diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index ccbe43d27..a8cf9e3d2 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -663,8 +663,9 @@ class GeminiMultimodalLiveLLMService(LLMService): await self._handle_evt_setup_complete(evt) elif evt.serverContent and evt.serverContent.modelTurn: await self._handle_evt_model_turn(evt) - elif evt.serverContent and evt.serverContent.turnComplete: + elif evt.serverContent and evt.serverContent.turnComplete and evt.usageMetadata: await self._handle_evt_turn_complete(evt) + await self._handle_evt_usage_metadata(evt) elif evt.serverContent and evt.serverContent.inputTranscription: await self._handle_evt_input_transcription(evt) elif evt.serverContent and evt.serverContent.outputTranscription: @@ -858,6 +859,7 @@ class GeminiMultimodalLiveLLMService(LLMService): ) async def _handle_evt_turn_complete(self, evt): + print(f"GeminiMultimodalLiveLLMService: _handle_evt_turn_complete: {evt}") self._bot_is_speaking = False text = self._bot_text_buffer self._bot_text_buffer = "" @@ -927,6 +929,19 @@ class GeminiMultimodalLiveLLMService(LLMService): await self.push_frame(LLMTextFrame(text=text)) await self.push_frame(TTSTextFrame(text=text)) + async def _handle_evt_usage_metadata(self, evt): + if not evt.usageMetadata: + return + + usage = evt.usageMetadata + + tokens = LLMTokenUsage( + prompt_tokens=usage.promptTokenCount, + completion_tokens=usage.responseTokenCount, + total_tokens=usage.totalTokenCount, + ) + await self.start_llm_usage_metrics(tokens) + def create_context_aggregator( self, context: OpenAILLMContext,