From 0b779a880b3b63296dc1b13bb3a75099fc98ebeb Mon Sep 17 00:00:00 2001 From: Thu Nguyen Date: Wed, 5 Nov 2025 13:24:49 +0700 Subject: [PATCH 1/5] Feat: allow accessing prob metrics for Whisper STT services with include_prob_metrics param --- src/pipecat/services/groq/stt.py | 3 +- src/pipecat/services/openai/stt.py | 9 +++ src/pipecat/services/sambanova/stt.py | 10 ++++ src/pipecat/services/whisper/utils.py | 85 +++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 src/pipecat/services/whisper/utils.py diff --git a/src/pipecat/services/groq/stt.py b/src/pipecat/services/groq/stt.py index 1267dd85f..28d0c6f83 100644 --- a/src/pipecat/services/groq/stt.py +++ b/src/pipecat/services/groq/stt.py @@ -58,7 +58,8 @@ class GroqSTTService(BaseWhisperSTTService): kwargs = { "file": ("audio.wav", audio, "audio/wav"), "model": self.model_name, - "response_format": "json", + # Use verbose_json to get probability metrics + "response_format": "verbose_json" if self._include_prob_metrics else "json", "language": self._language, } diff --git a/src/pipecat/services/openai/stt.py b/src/pipecat/services/openai/stt.py index 208c68d34..75a6f920c 100644 --- a/src/pipecat/services/openai/stt.py +++ b/src/pipecat/services/openai/stt.py @@ -61,6 +61,15 @@ class OpenAISTTService(BaseWhisperSTTService): "language": self._language, } + if self._include_prob_metrics: + # GPT-4o-transcribe models only support logprobs (not verbose_json) + if self.model_name in ("gpt-4o-transcribe", "gpt-4o-mini-transcribe"): + kwargs["response_format"] = "json" + kwargs["include"] = ["logprobs"] + else: + # Whisper models support verbose_json + kwargs["response_format"] = "verbose_json" + if self._prompt is not None: kwargs["prompt"] = self._prompt diff --git a/src/pipecat/services/sambanova/stt.py b/src/pipecat/services/sambanova/stt.py index 71a709420..2f4e70c77 100644 --- a/src/pipecat/services/sambanova/stt.py +++ b/src/pipecat/services/sambanova/stt.py @@ -8,6 +8,8 @@ from typing import Any, Optional +from loguru import logger + from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription from pipecat.transcriptions.language import Language @@ -54,6 +56,14 @@ class SambaNovaSTTService(BaseWhisperSTTService): # type: ignore async def _transcribe(self, audio: bytes) -> Transcription: assert self._language is not None # Assigned in the BaseWhisperSTTService class + if self._include_prob_metrics: + # https://docs.sambanova.ai/docs/en/features/audio#request-parameters + logger.warning( + "SambaNova STT does not support probability metrics " + "(include_prob_metrics parameter has no effect). " + "Check their docs: https://docs.sambanova.ai/docs/en/features/audio#request-parameters for more details." + ) + # Build kwargs dict with only set parameters kwargs = { "file": ("audio.wav", audio, "audio/wav"), diff --git a/src/pipecat/services/whisper/utils.py b/src/pipecat/services/whisper/utils.py new file mode 100644 index 000000000..5d685d459 --- /dev/null +++ b/src/pipecat/services/whisper/utils.py @@ -0,0 +1,85 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import math +from typing import Optional + +from pipecat.frames.frames import TranscriptionFrame + + +def extract_whisper_probability(frame: TranscriptionFrame) -> Optional[float]: + """Extract probability from Whisper-based TranscriptionFrame result. + + Works with Groq, OpenAI Whisper, or other Whisper-based services that use + verbose_json format with segments containing avg_logprob. + + Converts avg_logprob to probability. + + Args: + frame: TranscriptionFrame with result from GroqSTTService or OpenAISTTService + (when include_prob_metrics=True and using Whisper models). + + Returns: + Probability (0-1) if available, None otherwise. + + Example: + >>> from pipecat.services.groq.stt import GroqSTTService + >>> from pipecat.services.whisper.utils import extract_whisper_probability + >>> + >>> stt = GroqSTTService(include_prob_metrics=True) + >>> # ... use stt in pipeline ... + >>> # In your frame processor: + >>> if isinstance(frame, TranscriptionFrame): + >>> prob = extract_whisper_probability(frame) + >>> if prob: + >>> print(f"Transcription confidence: {prob:.2%}") + """ + if not frame.result: + return None + + # Whisper verbose_json format: response.segments[0].avg_logprob + if hasattr(frame.result, "segments") and frame.result.segments: + segment = frame.result.segments[0] + avg_logprob = getattr(segment, "avg_logprob", None) + if avg_logprob is not None: + return math.exp(avg_logprob) + + return None + + +def extract_openai_gpt4o_logprobs(frame: TranscriptionFrame) -> Optional[list]: + """Extract logprobs from OpenAI GPT-4o-transcribe TranscriptionFrame result. + + Args: + frame: TranscriptionFrame with result from OpenAISTTService + using GPT-4o-transcribe model (when include_prob_metrics=True). + + Returns: + List of logprobs if available, None otherwise. + + Example: + >>> from pipecat.services.openai.stt import OpenAISTTService + >>> from pipecat.services.whisper.utils import extract_openai_gpt4o_logprobs + >>> + >>> stt = OpenAISTTService(model="gpt-4o-transcribe", include_prob_metrics=True) + >>> # ... use stt in pipeline ... + >>> # In your frame processor: + >>> if isinstance(frame, TranscriptionFrame): + >>> logprobs = extract_openai_gpt4o_logprobs(frame) + >>> if logprobs: + >>> # Calculate average logprob + >>> avg_logprob = sum(logprobs) / len(logprobs) + >>> prob = math.exp(avg_logprob) + >>> print(f"Transcription confidence: {prob:.2%}") + """ + if not frame.result: + return None + + # OpenAI GPT-4o-transcribe format: response.logprobs + if hasattr(frame.result, "logprobs"): + return frame.result.logprobs + + return None From 842c4a34850495f8af9736a532ca9476c0a96b23 Mon Sep 17 00:00:00 2001 From: Thu Nguyen Date: Wed, 5 Nov 2025 13:26:59 +0700 Subject: [PATCH 2/5] Update base_stt --- src/pipecat/services/whisper/base_stt.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pipecat/services/whisper/base_stt.py b/src/pipecat/services/whisper/base_stt.py index 3d9151e37..64c5e5ed7 100644 --- a/src/pipecat/services/whisper/base_stt.py +++ b/src/pipecat/services/whisper/base_stt.py @@ -122,6 +122,7 @@ class BaseWhisperSTTService(SegmentedSTTService): language: Optional[Language] = Language.EN, prompt: Optional[str] = None, temperature: Optional[float] = None, + include_prob_metrics: bool = False, **kwargs, ): """Initialize the Whisper STT service. @@ -133,6 +134,9 @@ class BaseWhisperSTTService(SegmentedSTTService): language: Language of the audio input. Defaults to English. prompt: Optional text to guide the model's style or continue a previous segment. temperature: Sampling temperature between 0 and 1. Defaults to 0.0. + include_prob_metrics: If True, enables probability metrics in API response. + Each service implements this differently (see child classes). + Defaults to False. **kwargs: Additional arguments passed to SegmentedSTTService. """ super().__init__(**kwargs) @@ -141,6 +145,7 @@ class BaseWhisperSTTService(SegmentedSTTService): self._language = self.language_to_service_language(language or Language.EN) self._prompt = prompt self._temperature = temperature + self._include_prob_metrics = include_prob_metrics self._settings = { "base_url": base_url, @@ -223,6 +228,7 @@ class BaseWhisperSTTService(SegmentedSTTService): text, self._user_id, time_now_iso8601(), + result=response, ) else: logger.warning("Received empty transcription from API") From 3486d63ef6709ae4a5df3840dd3a621210a03fb7 Mon Sep 17 00:00:00 2001 From: Thu Nguyen Date: Wed, 5 Nov 2025 13:30:49 +0700 Subject: [PATCH 3/5] Add docs --- src/pipecat/services/whisper/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pipecat/services/whisper/utils.py b/src/pipecat/services/whisper/utils.py index 5d685d459..1d8c84897 100644 --- a/src/pipecat/services/whisper/utils.py +++ b/src/pipecat/services/whisper/utils.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Utility functions for extracting probability metrics from Whisper-based STT services.""" + import math from typing import Optional From c26c27fe2105d8d73fdae02a75f47c2858df1380 Mon Sep 17 00:00:00 2001 From: Thu Nguyen Date: Thu, 6 Nov 2025 00:23:20 +0700 Subject: [PATCH 4/5] Update util with new docs and extract_deepgram_probability --- src/pipecat/services/whisper/utils.py | 108 +++++++++++++++++++------- 1 file changed, 78 insertions(+), 30 deletions(-) diff --git a/src/pipecat/services/whisper/utils.py b/src/pipecat/services/whisper/utils.py index 1d8c84897..b28c945c0 100644 --- a/src/pipecat/services/whisper/utils.py +++ b/src/pipecat/services/whisper/utils.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""Utility functions for extracting probability metrics from Whisper-based STT services.""" +"""Utility functions for extracting probability metrics from STT services.""" import math from typing import Optional @@ -27,17 +27,18 @@ def extract_whisper_probability(frame: TranscriptionFrame) -> Optional[float]: Returns: Probability (0-1) if available, None otherwise. - Example: - >>> from pipecat.services.groq.stt import GroqSTTService - >>> from pipecat.services.whisper.utils import extract_whisper_probability - >>> - >>> stt = GroqSTTService(include_prob_metrics=True) - >>> # ... use stt in pipeline ... - >>> # In your frame processor: - >>> if isinstance(frame, TranscriptionFrame): - >>> prob = extract_whisper_probability(frame) - >>> if prob: - >>> print(f"Transcription confidence: {prob:.2%}") + Example:: + + from pipecat.services.groq.stt import GroqSTTService + from pipecat.services.whisper.utils import extract_whisper_probability + + stt = GroqSTTService(include_prob_metrics=True) + # ... use stt in pipeline ... + # In your frame processor: + if isinstance(frame, TranscriptionFrame): + prob = extract_whisper_probability(frame) + if prob: + print(f"Transcription confidence: {prob:.2%}") """ if not frame.result: return None @@ -52,36 +53,83 @@ def extract_whisper_probability(frame: TranscriptionFrame) -> Optional[float]: return None -def extract_openai_gpt4o_logprobs(frame: TranscriptionFrame) -> Optional[list]: - """Extract logprobs from OpenAI GPT-4o-transcribe TranscriptionFrame result. +def extract_openai_gpt4o_probability(frame: TranscriptionFrame) -> Optional[float]: + """Extract probability from OpenAI GPT-4o-transcribe TranscriptionFrame result. Args: frame: TranscriptionFrame with result from OpenAISTTService using GPT-4o-transcribe model (when include_prob_metrics=True). Returns: - List of logprobs if available, None otherwise. + Probability (0-1) if available, None otherwise. - Example: - >>> from pipecat.services.openai.stt import OpenAISTTService - >>> from pipecat.services.whisper.utils import extract_openai_gpt4o_logprobs - >>> - >>> stt = OpenAISTTService(model="gpt-4o-transcribe", include_prob_metrics=True) - >>> # ... use stt in pipeline ... - >>> # In your frame processor: - >>> if isinstance(frame, TranscriptionFrame): - >>> logprobs = extract_openai_gpt4o_logprobs(frame) - >>> if logprobs: - >>> # Calculate average logprob - >>> avg_logprob = sum(logprobs) / len(logprobs) - >>> prob = math.exp(avg_logprob) - >>> print(f"Transcription confidence: {prob:.2%}") + Example:: + + from pipecat.services.openai.stt import OpenAISTTService + from pipecat.services.whisper.utils import extract_openai_gpt4o_probability + + stt = OpenAISTTService(model="gpt-4o-transcribe", include_prob_metrics=True) + # ... use stt in pipeline ... + # In your frame processor: + if isinstance(frame, TranscriptionFrame): + prob = extract_openai_gpt4o_probability(frame) + if prob: + print(f"Transcription confidence: {prob:.2%}") """ if not frame.result: return None # OpenAI GPT-4o-transcribe format: response.logprobs if hasattr(frame.result, "logprobs"): - return frame.result.logprobs + logprobs = frame.result.logprobs + if logprobs: + # Calculate average logprob and convert to probability + avg_logprob = sum(logprobs) / len(logprobs) + return math.exp(avg_logprob) + + return None + + +def extract_deepgram_probability(frame: TranscriptionFrame) -> Optional[float]: + """Extract probability from Deepgram TranscriptionFrame result. + + Args: + frame: TranscriptionFrame with result from DeepgramSTTService. + + Returns: + Probability (0-1) if available, None otherwise. + Returns alternative-level confidence if available, otherwise calculates + average confidence from word-level confidences. + + Example:: + + from pipecat.services.deepgram.stt import DeepgramSTTService + from pipecat.services.whisper.utils import extract_deepgram_probability + + stt = DeepgramSTTService() + # ... use stt in pipeline ... + # In your frame processor: + if isinstance(frame, TranscriptionFrame): + prob = extract_deepgram_probability(frame) + if prob: + print(f"Transcription confidence: {prob:.2%}") + """ + if not frame.result: + return None + + result = frame.result + if hasattr(result, "channel") and result.channel: + if hasattr(result.channel, "alternatives") and result.channel.alternatives: + alt = result.channel.alternatives[0] + conf = getattr(alt, "confidence", None) + if conf is not None: + return float(conf) + + words = getattr(alt, "words", None) + if words: + word_confs = [getattr(w, "confidence", None) for w in words] + word_confs = [c for c in word_confs if c is not None] + if word_confs: + return float(sum(word_confs) / len(word_confs)) return None From dd8711dee1aceb06a5053819c8769a0eaf803068 Mon Sep 17 00:00:00 2001 From: Thu Nguyen Date: Thu, 6 Nov 2025 00:23:42 +0700 Subject: [PATCH 5/5] Added changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0beaeab2f..7c3b94838 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 you cancel a task with `PipelineTask.cancel(reason="cancellation your reason")`. +- Added `include_prob_metrics` parameter to Whisper STT services to enable access + to probability metrics from transcription results. + +- Added utility functions `extract_whisper_probability()`, + `extract_openai_gpt4o_probability()`, and `extract_deepgram_probability()` to + extract probability metrics from `TranscriptionFrame` objects for Whisper-based, + OpenAI GPT-4o-transcribe, and Deepgram STT services respectively. + ### Fixed - Fixed `GeminiLiveLLMService` session resumption after a connection timeout.