From 3cb189eb1fe37229e4697e665a6185d44004b511 Mon Sep 17 00:00:00 2001 From: Jin Kim Date: Tue, 4 Feb 2025 10:27:28 +0900 Subject: [PATCH 1/3] Add whisper STT service using OpenAI API --- src/pipecat/services/openai.py | 64 +++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index ab81a1abb..2e138cd99 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -29,6 +29,7 @@ from pipecat.frames.frames import ( LLMUpdateSettingsFrame, OpenAILLMContextAssistantTimestampFrame, StartInterruptionFrame, + TranscriptionFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, @@ -47,7 +48,12 @@ from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContextFrame, ) from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.ai_services import ImageGenService, LLMService, TTSService +from pipecat.services.ai_services import ( + ImageGenService, + LLMService, + SegmentedSTTService, + TTSService, +) from pipecat.utils.time import time_now_iso8601 try: @@ -58,6 +64,7 @@ try: BadRequestError, DefaultAsyncHttpxClient, ) + from openai.types.audio import Transcription from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam except ModuleNotFoundError as e: logger.error(f"Exception: {e}") @@ -390,6 +397,61 @@ class OpenAIImageGenService(ImageGenService): yield frame +class OpenAISTTService(SegmentedSTTService): + """OpenAI Speech-to-Text (STT) service. + + This service uses OpenAI's Whisper API to convert audio to text. + + Args: + model: Whisper model to use. Defaults to "whisper-1". + api_key: OpenAI API key. Defaults to None. + base_url: API base URL. Defaults to None. + **kwargs: Additional arguments passed to SegmentedSTTService. + """ + + def __init__( + self, + *, + model: str = "whisper-1", + api_key: Optional[str] = None, + base_url: Optional[str] = None, + **kwargs, + ): + super().__init__(**kwargs) + self.set_model_name(model) + self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) + + async def set_model(self, model: str): + self.set_model_name(model) + + def can_generate_metrics(self) -> bool: + return True + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + try: + await self.start_processing_metrics() + await self.start_ttfb_metrics() + + response: Transcription = await self._client.audio.transcriptions.create( + file=("audio.wav", audio, "audio/wav"), model=self.model_name + ) + + await self.stop_ttfb_metrics() + await self.stop_processing_metrics() + + text = response.text.strip() + + if text: + logger.debug(f"Transcription: [{text}]") + yield TranscriptionFrame(text, "", time_now_iso8601()) + else: + logger.warning("Received empty transcription from API") + + except Exception as e: + logger.exception(f"Exception during transcription: {e}") + yield ErrorFrame(f"Error during transcription: {str(e)}") + + class OpenAITTSService(TTSService): """OpenAI Text-to-Speech service that generates audio from text. From 823b763b25eba164ba7dcacc040a53d00cc56fb2 Mon Sep 17 00:00:00 2001 From: Jin Kim Date: Tue, 4 Feb 2025 10:28:06 +0900 Subject: [PATCH 2/3] Change OpenAI example file name --- ...7g-interruptible-openai-tts.py => 07g-interruptible-openai.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/foundational/{07g-interruptible-openai-tts.py => 07g-interruptible-openai.py} (100%) diff --git a/examples/foundational/07g-interruptible-openai-tts.py b/examples/foundational/07g-interruptible-openai.py similarity index 100% rename from examples/foundational/07g-interruptible-openai-tts.py rename to examples/foundational/07g-interruptible-openai.py From ef1e4277d32a593c7230e2fdd5b588f560ef0084 Mon Sep 17 00:00:00 2001 From: Jin Kim Date: Tue, 4 Feb 2025 10:32:55 +0900 Subject: [PATCH 3/3] Add an example for Whisper using OpenAI API --- examples/foundational/07g-interruptible-openai.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/examples/foundational/07g-interruptible-openai.py b/examples/foundational/07g-interruptible-openai.py index 78a23b65f..01d869e91 100644 --- a/examples/foundational/07g-interruptible-openai.py +++ b/examples/foundational/07g-interruptible-openai.py @@ -18,7 +18,7 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.openai import OpenAILLMService, OpenAITTSService +from pipecat.services.openai import OpenAILLMService, OpenAISTTService, OpenAITTSService from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) @@ -38,12 +38,21 @@ async def main(): DailyParams( audio_out_enabled=True, audio_out_sample_rate=24000, - transcription_enabled=True, + transcription_enabled=False, vad_enabled=True, vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, ), ) + # You can use the OpenAI compatible API like Groq. + # stt = OpenAISTTService( + # base_url="https://api.groq.com/openai/v1", + # api_key="gsk_***", + # model="whisper-large-v3", + # ) + stt = OpenAISTTService(api_key=os.getenv("OPENAI_API_KEY"), model="whisper-1") + tts = OpenAITTSService(api_key=os.getenv("OPENAI_API_KEY"), voice="alloy") llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") @@ -61,6 +70,7 @@ async def main(): pipeline = Pipeline( [ transport.input(), # Transport user input + stt, # STT context_aggregator.user(), # User responses llm, # LLM tts, # TTS