From d35f4c6b992cc0975b98cbee8f387e10f5845c2e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 10 Feb 2025 21:06:37 -0500 Subject: [PATCH 1/3] Add prompt and temperature args to OpenAI and Groq hosted Whisper STT services --- src/pipecat/services/base_whisper.py | 6 +++++ src/pipecat/services/groq.py | 34 ++++++++++++++++++++++------ src/pipecat/services/openai.py | 30 ++++++++++++++++++++---- 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/src/pipecat/services/base_whisper.py b/src/pipecat/services/base_whisper.py index 150ad2d50..abe9475dc 100644 --- a/src/pipecat/services/base_whisper.py +++ b/src/pipecat/services/base_whisper.py @@ -111,6 +111,8 @@ class BaseWhisperSTTService(SegmentedSTTService): api_key: Service API key. Defaults to None. base_url: Service API base URL. Defaults to None. language: Language of the audio input. Defaults to English. + prompt: Optional text to guide the model's style or continue a previous segment. + temperature: Sampling temperature between 0 and 1. Defaults to 0.0. **kwargs: Additional arguments passed to SegmentedSTTService. """ @@ -121,12 +123,16 @@ class BaseWhisperSTTService(SegmentedSTTService): api_key: Optional[str] = None, base_url: Optional[str] = None, language: Optional[Language] = Language.EN, + prompt: Optional[str] = None, + temperature: float = 0.0, **kwargs, ): super().__init__(**kwargs) self.set_model_name(model) self._client = self._create_client(api_key, base_url) self._language = self.language_to_service_language(language or Language.EN) + self._prompt = prompt + self._temperature = temperature def _create_client(self, api_key: Optional[str], base_url: Optional[str]): return AsyncOpenAI(api_key=api_key, base_url=base_url) diff --git a/src/pipecat/services/groq.py b/src/pipecat/services/groq.py index b7ee2b770..02a7e5e0f 100644 --- a/src/pipecat/services/groq.py +++ b/src/pipecat/services/groq.py @@ -54,6 +54,8 @@ class GroqSTTService(BaseWhisperSTTService): api_key: Groq API key. Defaults to None. base_url: API base URL. Defaults to "https://api.groq.com/openai/v1". language: Language of the audio input. Defaults to English. + prompt: Optional text to guide the model's style or continue a previous segment. + temperature: Sampling temperature between 0 and 1. Defaults to 0.0. **kwargs: Additional arguments passed to BaseWhisperSTTService. """ @@ -64,17 +66,35 @@ class GroqSTTService(BaseWhisperSTTService): api_key: Optional[str] = None, base_url: str = "https://api.groq.com/openai/v1", language: Optional[Language] = Language.EN, + prompt: Optional[str] = None, + temperature: float = 0.0, **kwargs, ): super().__init__( - model=model, api_key=api_key, base_url=base_url, language=language, **kwargs + model=model, + api_key=api_key, + base_url=base_url, + language=language, + prompt=prompt, + temperature=temperature, + **kwargs, ) async def _transcribe(self, audio: bytes) -> Transcription: assert self._language is not None # Assigned in the BaseWhisperSTTService class - return await self._client.audio.transcriptions.create( - file=("audio.wav", audio, "audio/wav"), - model=self.model_name, - response_format="json", - language=self._language, - ) + + # Build kwargs dict with only set parameters + kwargs = { + "file": ("audio.wav", audio, "audio/wav"), + "model": self.model_name, + "response_format": "json", + "language": self._language, + } + + if self._prompt is not None: + kwargs["prompt"] = self._prompt + + if self._temperature is not None: + kwargs["temperature"] = self._temperature + + return await self._client.audio.transcriptions.create(**kwargs) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 61f2dcd33..182488f16 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -408,6 +408,8 @@ class OpenAISTTService(BaseWhisperSTTService): api_key: OpenAI API key. Defaults to None. base_url: API base URL. Defaults to None. language: Language of the audio input. Defaults to English. + prompt: Optional text to guide the model's style or continue a previous segment. + temperature: Sampling temperature between 0 and 1. Defaults to 0.0. **kwargs: Additional arguments passed to BaseWhisperSTTService. """ @@ -418,17 +420,37 @@ class OpenAISTTService(BaseWhisperSTTService): api_key: Optional[str] = None, base_url: Optional[str] = None, language: Optional[Language] = Language.EN, + prompt: Optional[str] = None, + temperature: float = 0.0, **kwargs, ): super().__init__( - model=model, api_key=api_key, base_url=base_url, language=language, **kwargs + model=model, + api_key=api_key, + base_url=base_url, + language=language, + prompt=prompt, + temperature=temperature, + **kwargs, ) async def _transcribe(self, audio: bytes) -> Transcription: assert self._language is not None # Assigned in the BaseWhisperSTTService class - return await self._client.audio.transcriptions.create( - file=("audio.wav", audio, "audio/wav"), model=self.model_name, language=self._language - ) + + # Build kwargs dict with only set parameters + kwargs = { + "file": ("audio.wav", audio, "audio/wav"), + "model": self.model_name, + "language": self._language, + } + + if self._prompt is not None: + kwargs["prompt"] = self._prompt + + if self._temperature is not None: + kwargs["temperature"] = self._temperature + + return await self._client.audio.transcriptions.create(**kwargs) class OpenAITTSService(TTSService): From dcc63dd648dde38d2755d83de2a34bc6fbd41715 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 11 Feb 2025 14:29:33 -0500 Subject: [PATCH 2/3] Use the vendor default for temperature --- src/pipecat/services/base_whisper.py | 2 +- src/pipecat/services/groq.py | 2 +- src/pipecat/services/openai.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/base_whisper.py b/src/pipecat/services/base_whisper.py index abe9475dc..eae38be64 100644 --- a/src/pipecat/services/base_whisper.py +++ b/src/pipecat/services/base_whisper.py @@ -124,7 +124,7 @@ class BaseWhisperSTTService(SegmentedSTTService): base_url: Optional[str] = None, language: Optional[Language] = Language.EN, prompt: Optional[str] = None, - temperature: float = 0.0, + temperature: Optional[float] = None, **kwargs, ): super().__init__(**kwargs) diff --git a/src/pipecat/services/groq.py b/src/pipecat/services/groq.py index 02a7e5e0f..e72597f06 100644 --- a/src/pipecat/services/groq.py +++ b/src/pipecat/services/groq.py @@ -67,7 +67,7 @@ class GroqSTTService(BaseWhisperSTTService): base_url: str = "https://api.groq.com/openai/v1", language: Optional[Language] = Language.EN, prompt: Optional[str] = None, - temperature: float = 0.0, + temperature: Optional[float] = None, **kwargs, ): super().__init__( diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 182488f16..6d9af198f 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -421,7 +421,7 @@ class OpenAISTTService(BaseWhisperSTTService): base_url: Optional[str] = None, language: Optional[Language] = Language.EN, prompt: Optional[str] = None, - temperature: float = 0.0, + temperature: Optional[float] = None, **kwargs, ): super().__init__( From ec0ca466172d790e7fc3219b2120a5e31d30ef4f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 13 Feb 2025 09:04:20 -0500 Subject: [PATCH 3/3] Fix temperature docstrings to reference optional --- src/pipecat/services/groq.py | 2 +- src/pipecat/services/openai.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/groq.py b/src/pipecat/services/groq.py index e72597f06..66cc9357f 100644 --- a/src/pipecat/services/groq.py +++ b/src/pipecat/services/groq.py @@ -55,7 +55,7 @@ class GroqSTTService(BaseWhisperSTTService): base_url: API base URL. Defaults to "https://api.groq.com/openai/v1". language: Language of the audio input. Defaults to English. prompt: Optional text to guide the model's style or continue a previous segment. - temperature: Sampling temperature between 0 and 1. Defaults to 0.0. + temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0. **kwargs: Additional arguments passed to BaseWhisperSTTService. """ diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 6d9af198f..bc251025e 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -409,7 +409,7 @@ class OpenAISTTService(BaseWhisperSTTService): base_url: API base URL. Defaults to None. language: Language of the audio input. Defaults to English. prompt: Optional text to guide the model's style or continue a previous segment. - temperature: Sampling temperature between 0 and 1. Defaults to 0.0. + temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0. **kwargs: Additional arguments passed to BaseWhisperSTTService. """