From ae049961b7a26978a840d4ba0edc9d52e4fed5c9 Mon Sep 17 00:00:00 2001
From: TomTom101 <thobra@gmail.com>
Date: Fri, 31 May 2024 22:30:52 +0200
Subject: [PATCH] wip: untested

---
 src/pipecat/services/openai.py | 87 ++++++++++++++++++++++------------
 tests/test_whisper.py          | 14 ++++++
 2 files changed, 71 insertions(+), 30 deletions(-)
 create mode 100644 tests/test_whisper.py

diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py
index 96c855fa4..0864baf53 100644
--- a/src/pipecat/services/openai.py
+++ b/src/pipecat/services/openai.py
@@ -3,45 +3,35 @@
 #
 # SPDX-License-Identifier: BSD 2-Clause License
 #
-
+import base64
 import io
 import json
 import time
-import aiohttp
-import base64
-
-from PIL import Image
-
 from typing import AsyncGenerator, List, Literal
 
-from pipecat.frames.frames import (
-    ErrorFrame,
-    Frame,
-    LLMFullResponseEndFrame,
-    LLMFullResponseStartFrame,
-    LLMMessagesFrame,
-    LLMResponseEndFrame,
-    LLMResponseStartFrame,
-    TextFrame,
-    URLImageRawFrame,
-    VisionImageRawFrame
-)
-from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext, OpenAILLMContextFrame
-from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.ai_services import LLMService, ImageGenService
-
+import aiohttp
 from loguru import logger
+from openai import OpenAI
+from PIL import Image
+
+from pipecat.frames.frames import (AudioRawFrame, ErrorFrame, Frame,
+                                   LLMFullResponseEndFrame,
+                                   LLMFullResponseStartFrame, LLMMessagesFrame,
+                                   LLMResponseEndFrame, LLMResponseStartFrame,
+                                   TextFrame, URLImageRawFrame,
+                                   VisionImageRawFrame)
+from pipecat.processors.aggregators.openai_llm_context import (
+    OpenAILLMContext, OpenAILLMContextFrame)
+from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.ai_services import (ImageGenService, LLMService,
+                                          TTSService)
 
 try:
     from openai import AsyncOpenAI, AsyncStream
-
-    from openai.types.chat import (
-        ChatCompletion,
-        ChatCompletionChunk,
-        ChatCompletionFunctionMessageParam,
-        ChatCompletionMessageParam,
-        ChatCompletionToolParam
-    )
+    from openai.types.chat import (ChatCompletion, ChatCompletionChunk,
+                                   ChatCompletionFunctionMessageParam,
+                                   ChatCompletionMessageParam,
+                                   ChatCompletionToolParam)
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
@@ -272,3 +262,40 @@ class OpenAIImageGenService(ImageGenService):
             image = Image.open(image_stream)
             frame = URLImageRawFrame(image_url, image.tobytes(), image.size, image.format)
             yield frame
+
+
+class WhisperTTSService(TTSService):
+    def __init__(
+            self,
+            *,
+            api_key: str | None,
+            voice: Literal["alloy", "echo", "fable", "onyx", "nova", "shimmer"] = "alloy",
+            response_format: Literal["mp3", "opus", "flac", "pcm"] = "pcm",
+            model: Literal["tts-1", "tts-1-hd"] = "tts-1",
+            **kwargs):
+        super().__init__(**kwargs)
+
+        self._voice = voice
+        self._model = model
+        self._response_format = response_format
+
+        self._client = AsyncOpenAI(api_key=api_key)
+
+    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+        logger.debug(f"Generating TTS: [{text}]")
+
+        async with self._client.audio.speech.with_streaming_response.create(
+                input=text,
+                model=self._model,
+                voice=self._voice,
+                response_format=self._response_format
+        ) as r:
+            if r.status != 200:
+                error = await r.text()
+                logger.error(f"Error getting audio (status: {r.status}, error: {error})")
+                yield ErrorFrame(f"Error getting audio (status: {r.status}, error: {error})")
+                return
+            for chunk in r.iter_bytes(1024):
+                if len(chunk) > 0:
+                    frame = AudioRawFrame(chunk, 16000, 1)
+                    yield frame
diff --git a/tests/test_whisper.py b/tests/test_whisper.py
new file mode 100644
index 000000000..62037e8a0
--- /dev/null
+++ b/tests/test_whisper.py
@@ -0,0 +1,14 @@
+import unittest
+
+from pipecat.services.openai import WhisperTTSService
+
+
+class TestWhisperOpenAIService(unittest.IsolatedAsyncioTestCase):
+    async def test_whisper_tts(self):
+        tts = WhisperTTSService()
+        # tts_response = await tts.run_tts("Hello, world")
+        await tts.say("Hi! If you want to talk to me, just say 'Hey Robot'.")
+
+
+if __name__ == "__main__":
+    unittest.main()