Merge pull request #1475 from pipecat-ai/khk/whisper-mlx-example

Example and CHANGELOG for WhisperSTTServiceMLX service

CHANGELOG.md

@@ -12,6 +12,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added `Mem0MemoryService`. Mem0 is a self-improving memory layer for LLM
   applications. Learn more at: https://mem0.ai/.
 
+- Added `WhisperSTTServiceMLX` for whisper transcription on Apple Silicon.
+  See example in `examples/foundational/13e-whisper-mlx.py`. Latency of
+  completed transcription using whisper large-v3-turbo on an M4 macbook is
+  ~500ms.
+
+- Added `SmallWebRTCTransport`, a new P2P WebRTC transport.
+
+  - Created two examples in `p2p-webrtc`:
+    - **video-transform**: Demonstrates sending and receiving audio/video with
+      `SmallWebRTCTransport` using `TypeScript`. Includes video frame
+      processing with OpenCV.
+    - **voice-agent**: A minimal example of creating a voice agent with
+      `SmallWebRTCTransport`.
+
 - Added `SmallWebRTCTransport`, a new P2P WebRTC transport.
 
   - Created two examples in `p2p-webrtc`:

examples/foundational/13e-whisper-mlx.py

@@ -0,0 +1,95 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import sys
+import time
+
+import aiohttp
+from dotenv import load_dotenv
+from loguru import logger
+from runner import configure
+
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.frames.frames import Frame, TranscriptionFrame, UserStoppedSpeakingFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.services.whisper import MLXModel, WhisperSTTServiceMLX
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+STOP_SECS = 2.0
+
+
+class TranscriptionLogger(FrameProcessor):
+    """Measures transcription latency.
+
+    Uses the (intentionally) long STOP_SECS parameter to give the transcription time to finish,
+    then outputs the timing between when the VAD first classified audio input as not-speech and
+    the delivery of the last transcription frame.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._last_transcription_time = time.time()
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, UserStoppedSpeakingFrame):
+            logger.debug(
+                f"Transcription latency: {(STOP_SECS - (time.time() - self._last_transcription_time)):.2f}"
+            )
+
+        if isinstance(frame, TranscriptionFrame):
+            self._last_transcription_time = time.time()
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        (room_url, _) = await configure(session)
+
+        transport = DailyTransport(
+            room_url,
+            None,
+            "Transcription bot",
+            DailyParams(
+                audio_in_enabled=True,
+                vad_enabled=True,
+                vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=STOP_SECS)),
+                vad_audio_passthrough=True,
+            ),
+        )
+
+        stt = WhisperSTTServiceMLX(model=MLXModel.LARGE_V3_TURBO)
+
+        tl = TranscriptionLogger()
+
+        pipeline = Pipeline([transport.input(), stt, tl])
+
+        task = PipelineTask(
+            pipeline,
+            params=PipelineParams(
+                enable_metrics=True,
+                report_only_initial_ttfb=False,
+            ),
+        )
+
+        runner = PipelineRunner()
+
+        await runner.run(task)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())