From 72d373e5655710f9a95a869c3478b1775d921ac9 Mon Sep 17 00:00:00 2001 From: Nico <105345946+nicougou@users.noreply.github.com> Date: Tue, 25 Mar 2025 10:25:16 +0100 Subject: [PATCH 1/4] feature/support instructions in OpenAITTSService --- src/pipecat/services/openai.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index cb1edea72..3be1df243 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -391,6 +391,7 @@ class OpenAIImageGenService(ImageGenService): self, *, api_key: str, + base_url: Optional[str] = None, aiohttp_session: aiohttp.ClientSession, image_size: Literal["256x256", "512x512", "1024x1024", "1792x1024", "1024x1792"], model: str = "dall-e-3", @@ -398,7 +399,7 @@ class OpenAIImageGenService(ImageGenService): super().__init__() self.set_model_name(model) self._image_size = image_size - self._client = AsyncOpenAI(api_key=api_key) + self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) self._aiohttp_session = aiohttp_session async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: @@ -501,9 +502,12 @@ class OpenAITTSService(TTSService): self, *, api_key: Optional[str] = None, + base_url: Optional[str] = None, + websocket_base_url: Optional[str] = None, voice: str = "alloy", model: str = "gpt-4o-mini-tts", sample_rate: Optional[int] = None, + instructions: Optional[str] = None, **kwargs, ): if sample_rate and sample_rate != self.OPENAI_SAMPLE_RATE: @@ -515,8 +519,8 @@ class OpenAITTSService(TTSService): self.set_model_name(model) self.set_voice(voice) - - self._client = AsyncOpenAI(api_key=api_key) + self._instructions = instructions + self._client = AsyncOpenAI(api_key=api_key, base_url=base_url, websocket_base_url=websocket_base_url) def can_generate_metrics(self) -> bool: return True @@ -538,11 +542,17 @@ class OpenAITTSService(TTSService): try: await self.start_ttfb_metrics() + # Setup extra body parameters + extra_body = {} + if self._instructions: + extra_body["instructions"] = self._instructions + async with self._client.audio.speech.with_streaming_response.create( input=text or " ", # Text must contain at least one character model=self.model_name, voice=VALID_VOICES[self._voice_id], response_format="pcm", + extra_body=extra_body, ) as r: if r.status_code != 200: error = await r.text() From d982fc35d8aebadaadb6449201917f7e741661f8 Mon Sep 17 00:00:00 2001 From: Nico <105345946+nicougou@users.noreply.github.com> Date: Tue, 25 Mar 2025 10:35:53 +0100 Subject: [PATCH 2/4] fix: formatter --- src/pipecat/services/openai.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 3be1df243..c631fd8c0 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -520,7 +520,9 @@ class OpenAITTSService(TTSService): self.set_model_name(model) self.set_voice(voice) self._instructions = instructions - self._client = AsyncOpenAI(api_key=api_key, base_url=base_url, websocket_base_url=websocket_base_url) + self._client = AsyncOpenAI( + api_key=api_key, base_url=base_url, websocket_base_url=websocket_base_url + ) def can_generate_metrics(self) -> bool: return True From dc2ee2bf0a96393bf6e01f85d3671df0bcdf5119 Mon Sep 17 00:00:00 2001 From: Nico <105345946+nicougou@users.noreply.github.com> Date: Wed, 26 Mar 2025 15:41:37 +0100 Subject: [PATCH 3/4] review: remove websocket_base_url --- src/pipecat/services/openai.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index c631fd8c0..3f85d917c 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -503,7 +503,6 @@ class OpenAITTSService(TTSService): *, api_key: Optional[str] = None, base_url: Optional[str] = None, - websocket_base_url: Optional[str] = None, voice: str = "alloy", model: str = "gpt-4o-mini-tts", sample_rate: Optional[int] = None, @@ -520,9 +519,7 @@ class OpenAITTSService(TTSService): self.set_model_name(model) self.set_voice(voice) self._instructions = instructions - self._client = AsyncOpenAI( - api_key=api_key, base_url=base_url, websocket_base_url=websocket_base_url - ) + self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) def can_generate_metrics(self) -> bool: return True From 499e69846d0918d1b42bd295d4ab07aa67553e89 Mon Sep 17 00:00:00 2001 From: Nico <105345946+nicougou@users.noreply.github.com> Date: Wed, 26 Mar 2025 17:13:30 +0100 Subject: [PATCH 4/4] review: add changelog entries --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e88155896..fa272561d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ElevenLabs TTS services now support a sample rate of 8000. +- Added support for `instructions` in `OpenAITTSService` + +- Added support for `base_url` in `OpenAIImageGenService` and `OpenAITTSService` + ### Fixed - Fixed a `GoogleAssistantContextAggregator` issue where function calls