From ae049961b7a26978a840d4ba0edc9d52e4fed5c9 Mon Sep 17 00:00:00 2001 From: TomTom101 Date: Fri, 31 May 2024 22:30:52 +0200 Subject: [PATCH 1/4] wip: untested --- src/pipecat/services/openai.py | 87 ++++++++++++++++++++++------------ tests/test_whisper.py | 14 ++++++ 2 files changed, 71 insertions(+), 30 deletions(-) create mode 100644 tests/test_whisper.py diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 96c855fa4..0864baf53 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -3,45 +3,35 @@ # # SPDX-License-Identifier: BSD 2-Clause License # - +import base64 import io import json import time -import aiohttp -import base64 - -from PIL import Image - from typing import AsyncGenerator, List, Literal -from pipecat.frames.frames import ( - ErrorFrame, - Frame, - LLMFullResponseEndFrame, - LLMFullResponseStartFrame, - LLMMessagesFrame, - LLMResponseEndFrame, - LLMResponseStartFrame, - TextFrame, - URLImageRawFrame, - VisionImageRawFrame -) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext, OpenAILLMContextFrame -from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.ai_services import LLMService, ImageGenService - +import aiohttp from loguru import logger +from openai import OpenAI +from PIL import Image + +from pipecat.frames.frames import (AudioRawFrame, ErrorFrame, Frame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, LLMMessagesFrame, + LLMResponseEndFrame, LLMResponseStartFrame, + TextFrame, URLImageRawFrame, + VisionImageRawFrame) +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, OpenAILLMContextFrame) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.ai_services import (ImageGenService, LLMService, + TTSService) try: from openai import AsyncOpenAI, AsyncStream - - from openai.types.chat import ( - ChatCompletion, - ChatCompletionChunk, - ChatCompletionFunctionMessageParam, - ChatCompletionMessageParam, - ChatCompletionToolParam - ) + from openai.types.chat import (ChatCompletion, ChatCompletionChunk, + ChatCompletionFunctionMessageParam, + ChatCompletionMessageParam, + ChatCompletionToolParam) except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error( @@ -272,3 +262,40 @@ class OpenAIImageGenService(ImageGenService): image = Image.open(image_stream) frame = URLImageRawFrame(image_url, image.tobytes(), image.size, image.format) yield frame + + +class WhisperTTSService(TTSService): + def __init__( + self, + *, + api_key: str | None, + voice: Literal["alloy", "echo", "fable", "onyx", "nova", "shimmer"] = "alloy", + response_format: Literal["mp3", "opus", "flac", "pcm"] = "pcm", + model: Literal["tts-1", "tts-1-hd"] = "tts-1", + **kwargs): + super().__init__(**kwargs) + + self._voice = voice + self._model = model + self._response_format = response_format + + self._client = AsyncOpenAI(api_key=api_key) + + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + logger.debug(f"Generating TTS: [{text}]") + + async with self._client.audio.speech.with_streaming_response.create( + input=text, + model=self._model, + voice=self._voice, + response_format=self._response_format + ) as r: + if r.status != 200: + error = await r.text() + logger.error(f"Error getting audio (status: {r.status}, error: {error})") + yield ErrorFrame(f"Error getting audio (status: {r.status}, error: {error})") + return + for chunk in r.iter_bytes(1024): + if len(chunk) > 0: + frame = AudioRawFrame(chunk, 16000, 1) + yield frame diff --git a/tests/test_whisper.py b/tests/test_whisper.py new file mode 100644 index 000000000..62037e8a0 --- /dev/null +++ b/tests/test_whisper.py @@ -0,0 +1,14 @@ +import unittest + +from pipecat.services.openai import WhisperTTSService + + +class TestWhisperOpenAIService(unittest.IsolatedAsyncioTestCase): + async def test_whisper_tts(self): + tts = WhisperTTSService() + # tts_response = await tts.run_tts("Hello, world") + await tts.say("Hi! If you want to talk to me, just say 'Hey Robot'.") + + +if __name__ == "__main__": + unittest.main() From 8683cae719a5cb8b932d1c28eb40032e6851cbed Mon Sep 17 00:00:00 2001 From: TomTom101 Date: Sat, 1 Jun 2024 10:13:28 +0200 Subject: [PATCH 2/4] feat: OpenAITTS --- src/pipecat/services/openai.py | 43 +++++++++++++++++----------------- tests/test_openai_tts.py | 39 ++++++++++++++++++++++++++++++ tests/test_whisper.py | 14 ----------- 3 files changed, 61 insertions(+), 35 deletions(-) create mode 100644 tests/test_openai_tts.py delete mode 100644 tests/test_whisper.py diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 0864baf53..ae0cd70c1 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -11,7 +11,6 @@ from typing import AsyncGenerator, List, Literal import aiohttp from loguru import logger -from openai import OpenAI from PIL import Image from pipecat.frames.frames import (AudioRawFrame, ErrorFrame, Frame, @@ -27,7 +26,7 @@ from pipecat.services.ai_services import (ImageGenService, LLMService, TTSService) try: - from openai import AsyncOpenAI, AsyncStream + from openai import AsyncOpenAI, AsyncStream, BadRequestError from openai.types.chat import (ChatCompletion, ChatCompletionChunk, ChatCompletionFunctionMessageParam, ChatCompletionMessageParam, @@ -264,38 +263,40 @@ class OpenAIImageGenService(ImageGenService): yield frame -class WhisperTTSService(TTSService): +class OpenAITTSService(TTSService): def __init__( self, *, - api_key: str | None, + api_key: str | None = None, voice: Literal["alloy", "echo", "fable", "onyx", "nova", "shimmer"] = "alloy", - response_format: Literal["mp3", "opus", "flac", "pcm"] = "pcm", model: Literal["tts-1", "tts-1-hd"] = "tts-1", **kwargs): super().__init__(**kwargs) self._voice = voice self._model = model - self._response_format = response_format self._client = AsyncOpenAI(api_key=api_key) async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: logger.debug(f"Generating TTS: [{text}]") - async with self._client.audio.speech.with_streaming_response.create( - input=text, - model=self._model, - voice=self._voice, - response_format=self._response_format - ) as r: - if r.status != 200: - error = await r.text() - logger.error(f"Error getting audio (status: {r.status}, error: {error})") - yield ErrorFrame(f"Error getting audio (status: {r.status}, error: {error})") - return - for chunk in r.iter_bytes(1024): - if len(chunk) > 0: - frame = AudioRawFrame(chunk, 16000, 1) - yield frame + try: + async with self._client.audio.speech.with_streaming_response.create( + input=text, + model=self._model, + voice=self._voice, + response_format="pcm", + ) as r: + if r.status_code != 200: + error = await r.text() + logger.error(f"Error getting audio (status: {r.status_code}, error: {error})") + yield ErrorFrame(f"Error getting audio (status: {r.status_code}, error: {error})") + return + async for chunk in r.iter_bytes(8192): + if len(chunk) > 0: + frame = AudioRawFrame(chunk, 24_000, 1) + yield frame + except BadRequestError as e: + logger.exception(f"Error generating TTS: {e}") + raise diff --git a/tests/test_openai_tts.py b/tests/test_openai_tts.py new file mode 100644 index 000000000..5bbf449b9 --- /dev/null +++ b/tests/test_openai_tts.py @@ -0,0 +1,39 @@ +import asyncio +import unittest + +import openai +import pyaudio +from dotenv import load_dotenv + +from pipecat.frames.frames import AudioRawFrame, ErrorFrame +from pipecat.services.openai import OpenAITTSService + +load_dotenv() + + +class TestWhisperOpenAIService(unittest.IsolatedAsyncioTestCase): + async def test_whisper_tts(self): + pa = pyaudio.PyAudio() + stream = pa.open(format=pyaudio.paInt16, + channels=1, + rate=24_000, + output=True) + + tts = OpenAITTSService(voice="nova") + + async for frame in tts.run_tts("Hello, there. Nice to meet you, seems to work well"): + self.assertIsInstance(frame, AudioRawFrame) + stream.write(frame.audio) + + await asyncio.sleep(.5) + stream.stop_stream() + pa.terminate() + + tts = OpenAITTSService(voice="invalid_voice") + with self.assertRaises(openai.BadRequestError): + async for frame in tts.run_tts("wont work"): + self.assertIsInstance(frame, ErrorFrame) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_whisper.py b/tests/test_whisper.py deleted file mode 100644 index 62037e8a0..000000000 --- a/tests/test_whisper.py +++ /dev/null @@ -1,14 +0,0 @@ -import unittest - -from pipecat.services.openai import WhisperTTSService - - -class TestWhisperOpenAIService(unittest.IsolatedAsyncioTestCase): - async def test_whisper_tts(self): - tts = WhisperTTSService() - # tts_response = await tts.run_tts("Hello, world") - await tts.say("Hi! If you want to talk to me, just say 'Hey Robot'.") - - -if __name__ == "__main__": - unittest.main() From 7085b1ea3f24bfd8ce2304e414d2f882fd08da98 Mon Sep 17 00:00:00 2001 From: TomTom101 Date: Sat, 1 Jun 2024 20:35:46 +0200 Subject: [PATCH 3/4] doc(openai): Added hint re the 24kHz sample rate --- src/pipecat/services/openai.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index ae0cd70c1..44bc9d847 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -264,6 +264,16 @@ class OpenAIImageGenService(ImageGenService): class OpenAITTSService(TTSService): + """This service uses the OpenAI TTS API to generate audio from text. + The returned audio is PCM encoded at 24kHz. When using the DailyTransport, set the sample rate in the DailyParams: + ``` + DailyParams( + audio_out_enabled=True, + audio_out_sample_rate=24_000, + ) + ``` + """ + def __init__( self, *, From d462c03d00026311307ccb408f5506c91185df53 Mon Sep 17 00:00:00 2001 From: TomTom101 Date: Mon, 3 Jun 2024 20:13:15 +0200 Subject: [PATCH 4/4] chore: Review comments --- src/pipecat/services/openai.py | 46 ++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 44bc9d847..fbdd5c1ac 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -13,24 +13,39 @@ import aiohttp from loguru import logger from PIL import Image -from pipecat.frames.frames import (AudioRawFrame, ErrorFrame, Frame, - LLMFullResponseEndFrame, - LLMFullResponseStartFrame, LLMMessagesFrame, - LLMResponseEndFrame, LLMResponseStartFrame, - TextFrame, URLImageRawFrame, - VisionImageRawFrame) +from pipecat.frames.frames import ( + AudioRawFrame, + ErrorFrame, + Frame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMMessagesFrame, + LLMResponseEndFrame, + LLMResponseStartFrame, + TextFrame, + URLImageRawFrame, + VisionImageRawFrame +) from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, OpenAILLMContextFrame) + OpenAILLMContext, + OpenAILLMContextFrame +) from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.ai_services import (ImageGenService, LLMService, - TTSService) +from pipecat.services.ai_services import ( + ImageGenService, + LLMService, + TTSService +) try: from openai import AsyncOpenAI, AsyncStream, BadRequestError - from openai.types.chat import (ChatCompletion, ChatCompletionChunk, - ChatCompletionFunctionMessageParam, - ChatCompletionMessageParam, - ChatCompletionToolParam) + from openai.types.chat import ( + ChatCompletion, + ChatCompletionChunk, + ChatCompletionFunctionMessageParam, + ChatCompletionMessageParam, + ChatCompletionToolParam + ) except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error( @@ -265,7 +280,7 @@ class OpenAIImageGenService(ImageGenService): class OpenAITTSService(TTSService): """This service uses the OpenAI TTS API to generate audio from text. - The returned audio is PCM encoded at 24kHz. When using the DailyTransport, set the sample rate in the DailyParams: + The returned audio is PCM encoded at 24kHz. When using the DailyTransport, set the sample rate in the DailyParams accordingly: ``` DailyParams( audio_out_enabled=True, @@ -308,5 +323,4 @@ class OpenAITTSService(TTSService): frame = AudioRawFrame(chunk, 24_000, 1) yield frame except BadRequestError as e: - logger.exception(f"Error generating TTS: {e}") - raise + logger.error(f"Error generating TTS: {e}")