Merge pull request #2103 from pipecat-ai/mb/add-user-id-to-transcript

Add user_id to transcription frames

CHANGELOG.md

@@ -40,6 +40,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   tools = ToolsSchema(standard_tools=[do_something])
   ```
 
+  - `user_id` is now populated in the `TranscriptionFrame` and
+    `InterimTranscriptionFrame` when using a transport that provides a
+    `user_id`, like `DailyTransport` or `LiveKitTransport`.
+
 - Added `watchdog_coroutine()`. This is a watchdog helper for couroutines. So,
   if you have a coroutine that is waiting for a result and that takes a long
   time, you will need to wrap it with `watchdog_coroutine()` so the watchdog

src/pipecat/services/assemblyai/stt.py

@@ -311,7 +311,7 @@ class AssemblyAISTTService(STTService):
             await self.push_frame(
                 TranscriptionFrame(
                     message.transcript,
-                    "",  # participant
+                    self._user_id,
                     time_now_iso8601(),
                     self._language,
                     message,
@@ -323,7 +323,7 @@ class AssemblyAISTTService(STTService):
             await self.push_frame(
                 InterimTranscriptionFrame(
                     message.transcript,
-                    "",  # participant
+                    self._user_id,
                     time_now_iso8601(),
                     self._language,
                     message,

src/pipecat/services/aws/stt.py

@@ -366,7 +366,7 @@ class AWSTranscribeSTTService(STTService):
                                     await self.push_frame(
                                         TranscriptionFrame(
                                             transcript,
-                                            "",
+                                            self._user_id,
                                             time_now_iso8601(),
                                             self._settings["language"],
                                             result=result,
@@ -382,7 +382,7 @@ class AWSTranscribeSTTService(STTService):
                                     await self.push_frame(
                                         InterimTranscriptionFrame(
                                             transcript,
-                                            "",
+                                            self._user_id,
                                             time_now_iso8601(),
                                             self._settings["language"],
                                             result=result,

src/pipecat/services/azure/stt.py

@@ -183,7 +183,7 @@ class AzureSTTService(STTService):
             language = getattr(event.result, "language", None) or self._settings.get("language")
             frame = TranscriptionFrame(
                 event.result.text,
-                "",
+                self._user_id,
                 time_now_iso8601(),
                 language,
                 result=event,

src/pipecat/services/cartesia/stt.py

@@ -289,14 +289,24 @@ class CartesiaSTTService(STTService):
             await self.stop_ttfb_metrics()
             if is_final:
                 await self.push_frame(
-                    TranscriptionFrame(transcript, "", time_now_iso8601(), language)
+                    TranscriptionFrame(
+                        transcript,
+                        self._user_id,
+                        time_now_iso8601(),
+                        language,
+                    )
                 )
                 await self._handle_transcription(transcript, is_final, language)
                 await self.stop_processing_metrics()
             else:
                 # For interim transcriptions, just push the frame without tracing
                 await self.push_frame(
-                    InterimTranscriptionFrame(transcript, "", time_now_iso8601(), language)
+                    InterimTranscriptionFrame(
+                        transcript,
+                        self._user_id,
+                        time_now_iso8601(),
+                        language,
+                    )
                 )
 
     async def _disconnect(self):

src/pipecat/services/deepgram/stt.py

@@ -278,7 +278,7 @@ class DeepgramSTTService(STTService):
                 await self.push_frame(
                     TranscriptionFrame(
                         transcript,
-                        "",
+                        self._user_id,
                         time_now_iso8601(),
                         language,
                         result=result,
@@ -291,7 +291,7 @@ class DeepgramSTTService(STTService):
                 await self.push_frame(
                     InterimTranscriptionFrame(
                         transcript,
-                        "",
+                        self._user_id,
                         time_now_iso8601(),
                         language,
                         result=result,

src/pipecat/services/fal/stt.py

@@ -291,7 +291,7 @@ class FalSTTService(SegmentedSTTService):
                     logger.debug(f"Transcription: [{text}]")
                     yield TranscriptionFrame(
                         text,
-                        "",
+                        self._user_id,
                         time_now_iso8601(),
                         Language(self._settings["language"]),
                         result=response,

src/pipecat/services/gladia/stt.py

@@ -567,7 +567,7 @@ class GladiaSTTService(STTService):
                             await self.push_frame(
                                 TranscriptionFrame(
                                     transcript,
-                                    "",
+                                    self._user_id,
                                     time_now_iso8601(),
                                     language,
                                     result=content,
@@ -582,7 +582,7 @@ class GladiaSTTService(STTService):
                             await self.push_frame(
                                 InterimTranscriptionFrame(
                                     transcript,
-                                    "",
+                                    self._user_id,
                                     time_now_iso8601(),
                                     language,
                                     result=content,

src/pipecat/services/google/stt.py

@@ -862,7 +862,7 @@ class GoogleSTTService(STTService):
                         await self.push_frame(
                             TranscriptionFrame(
                                 transcript,
-                                "",
+                                self._user_id,
                                 time_now_iso8601(),
                                 primary_language,
                                 result=result,
@@ -880,7 +880,7 @@ class GoogleSTTService(STTService):
                         await self.push_frame(
                             InterimTranscriptionFrame(
                                 transcript,
-                                "",
+                                self._user_id,
                                 time_now_iso8601(),
                                 primary_language,
                                 result=result,

src/pipecat/services/riva/stt.py

@@ -314,7 +314,7 @@ class RivaSTTService(STTService):
                     await self.push_frame(
                         TranscriptionFrame(
                             transcript,
-                            "",
+                            self._user_id,
                             time_now_iso8601(),
                             self._language_code,
                             result=result,
@@ -329,7 +329,7 @@ class RivaSTTService(STTService):
                     await self.push_frame(
                         InterimTranscriptionFrame(
                             transcript,
-                            "",
+                            self._user_id,
                             time_now_iso8601(),
                             self._language_code,
                             result=result,
@@ -636,7 +636,10 @@ class RivaSegmentedSTTService(SegmentedSTTService):
                         if text:
                             logger.debug(f"Transcription: [{text}]")
                             yield TranscriptionFrame(
-                                text, "", time_now_iso8601(), self._language_enum
+                                text,
+                                self._user_id,
+                                time_now_iso8601(),
+                                self._language_enum,
                             )
                             transcription_found = True
 

src/pipecat/services/stt_service.py

@@ -57,6 +57,7 @@ class STTService(AIService):
         self._sample_rate = 0
         self._settings: Dict[str, Any] = {}
         self._muted: bool = False
+        self._user_id: str = ""
 
     @property
     def is_muted(self) -> bool:
@@ -132,6 +133,11 @@ class STTService(AIService):
     async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection):
         """Process an audio frame for speech recognition.
 
+        If the service is muted, this method does nothing. Otherwise, it
+        processes the audio frame and runs speech-to-text on it, yielding
+        transcription results. If the frame has a user_id, it is stored
+        for later use in transcription.
+
         Args:
             frame: The audio frame to process.
             direction: The direction of frame processing.
@@ -139,6 +145,13 @@ class STTService(AIService):
         if self._muted:
             return
 
+        # UserAudioRawFrame contains a user_id (e.g. Daily, Livekit)
+        if hasattr(frame, "user_id"):
+            self._user_id = frame.user_id
+        # AudioRawFrame does not have a user_id (e.g. SmallWebRTCTransport, websockets)
+        else:
+            self._user_id = ""
+
         await self.process_generator(self.run_stt(frame.audio))
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -241,10 +254,19 @@ class SegmentedSTTService(STTService):
         Continuously buffers audio, growing the buffer while user is speaking and
         maintaining a small buffer when not speaking to account for VAD delay.
 
+        If the frame has a user_id, it is stored for later use in transcription.
+
         Args:
             frame: The audio frame to process.
             direction: The direction of frame processing.
         """
+        # UserAudioRawFrame contains a user_id (e.g. Daily, Livekit)
+        if hasattr(frame, "user_id"):
+            self._user_id = frame.user_id
+        # AudioRawFrame does not have a user_id (e.g. SmallWebRTCTransport, websockets)
+        else:
+            self._user_id = ""
+
         # If the user is speaking the audio buffer will keep growing.
         self._audio_buffer += frame.audio
 

src/pipecat/services/whisper/base_stt.py

@@ -219,7 +219,11 @@ class BaseWhisperSTTService(SegmentedSTTService):
             if text:
                 await self._handle_transcription(text, True, self._language)
                 logger.debug(f"Transcription: [{text}]")
-                yield TranscriptionFrame(text, "", time_now_iso8601())
+                yield TranscriptionFrame(
+                    text,
+                    self._user_id,
+                    time_now_iso8601(),
+                )
             else:
                 logger.warning("Received empty transcription from API")
 

src/pipecat/services/whisper/stt.py

@@ -395,7 +395,12 @@ class WhisperSTTService(SegmentedSTTService):
         if text:
             await self._handle_transcription(text, True, self._settings["language"])
             logger.debug(f"Transcription: [{text}]")
-            yield TranscriptionFrame(text, "", time_now_iso8601(), self._settings["language"])
+            yield TranscriptionFrame(
+                text,
+                self._user_id,
+                time_now_iso8601(),
+                self._settings["language"],
+            )
 
 
 class WhisperSTTServiceMLX(WhisperSTTService):
@@ -500,7 +505,12 @@ class WhisperSTTServiceMLX(WhisperSTTService):
             if text:
                 await self._handle_transcription(text, True, self._settings["language"])
                 logger.debug(f"Transcription: [{text}]")
-                yield TranscriptionFrame(text, "", time_now_iso8601(), self._settings["language"])
+                yield TranscriptionFrame(
+                    text,
+                    self._user_id,
+                    time_now_iso8601(),
+                    self._settings["language"],
+                )
 
         except Exception as e:
             logger.exception(f"MLX Whisper transcription error: {e}")

src/pipecat/utils/tracing/service_attributes.py

@@ -125,6 +125,7 @@ def add_stt_span_attributes(
     transcript: Optional[str] = None,
     is_final: Optional[bool] = None,
     language: Optional[str] = None,
+    user_id: Optional[str] = None,
     settings: Optional[Dict[str, Any]] = None,
     vad_enabled: bool = False,
     ttfb: Optional[float] = None,
@@ -140,6 +141,7 @@ def add_stt_span_attributes(
         transcript: The transcribed text.
         is_final: Whether this is a final transcript.
         language: Detected or configured language.
+        user_id: User ID associated with the audio being transcribed.
         settings: Service configuration settings.
         vad_enabled: Whether voice activity detection is enabled.
         ttfb: Time to first byte in seconds.
@@ -161,6 +163,9 @@ def add_stt_span_attributes(
     if language:
         span.set_attribute("language", language)
 
+    if user_id:
+        span.set_attribute("user_id", user_id)
+
     if ttfb is not None:
         span.set_attribute("metrics.ttfb", ttfb)
 

src/pipecat/utils/tracing/service_decorators.py

@@ -270,6 +270,7 @@ def traced_stt(func: Optional[Callable] = None, *, name: Optional[str] = None) -
                             transcript=transcript,
                             is_final=is_final,
                             language=str(language) if language else None,
+                            user_id=getattr(self, "_user_id", None),
                             vad_enabled=getattr(self, "vad_enabled", False),
                             settings=settings,
                             ttfb=ttfb,