diff --git a/CHANGELOG.md b/CHANGELOG.md index eff100b2c..251539ba7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `TranslationFrame`, a new frame type that contains a translated + transcription. + - Added `TransportParams.audio_in_passthrough`. If set (the default), incoming audio will be pushed downstream. @@ -17,6 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Updated `GladiaSTTService` to output a `TranslationFrame` when specifying a + `translation` and `translation_config`. + - STT services now passthrough audio frames by default. This allows you to add audio recording without worrying about what's wrong in your pipeline when it doesn't work the first time. @@ -49,6 +55,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added 04 foundational examples for client/server transports. Also, renamed `29-livekit-audio-chat.py` to `04b-transports-livekit.py`. +- Added foundational example `13c-gladia-translation.py` showing how to use + `TranscriptionFrame` and `TranslationFrame`. + ## [0.0.65] - 2025-04-23 "Sant Jordi's release" 🌹📕 https://en.wikipedia.org/wiki/Saint_George%27s_Day_in_Catalonia diff --git a/README.md b/README.md index 5a0375c71..ac2444f87 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,12 @@ pip install "pipecat-ai[option,...]" ### Running tests +Install the test dependencies: + +```shell +pip install -r test-requirements.txt +``` + From the root directory, run: ```shell diff --git a/examples/foundational/13c-gladia-translation.py b/examples/foundational/13c-gladia-translation.py new file mode 100644 index 000000000..8ebd17aa4 --- /dev/null +++ b/examples/foundational/13c-gladia-translation.py @@ -0,0 +1,90 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.frames.frames import Frame, TranscriptionFrame, TranslationFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineTask +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.services.gladia.config import ( + GladiaInputParams, + LanguageConfig, + RealtimeProcessingConfig, + TranslationConfig, +) +from pipecat.services.gladia.stt import GladiaSTTService +from pipecat.transcriptions.language import Language +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection + +load_dotenv(override=True) + + +class TranscriptionLogger(FrameProcessor): + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, TranscriptionFrame): + print(f"Transcription ({frame.language}): {frame.text}") + elif isinstance(frame, TranslationFrame): + print(f"Translation ({frame.language}): {frame.text}") + + +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") + + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams(audio_in_enabled=True), + ) + + stt = GladiaSTTService( + api_key=os.getenv("GLADIA_API_KEY"), + params=GladiaInputParams( + language_config=LanguageConfig( + languages=[Language.EN], # Input in English + code_switching=False, + ), + realtime_processing=RealtimeProcessingConfig( + translation=True, # Enable translation + translation_config=TranslationConfig( + target_languages=[Language.ES], # Translate to Spanish + model="enhanced", # Use the enhanced translation model + ), + ), + ), + ) + + tl = TranscriptionLogger() + + pipeline = Pipeline([transport.input(), stt, tl]) + + task = PipelineTask(pipeline) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) + + +if __name__ == "__main__": + from run import main + + main() diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 72acf1a2a..073112a9f 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -256,6 +256,22 @@ class InterimTranscriptionFrame(TextFrame): return f"{self.name}(user: {self.user_id}, text: [{self.text}], language: {self.language}, timestamp: {self.timestamp})" +@dataclass +class TranslationFrame(TextFrame): + """A text frame with translated transcription data. + + Will be placed in the transport's receive queue when a participant speaks. + + """ + + user_id: str + timestamp: str + language: Optional[Language] = None + + def __str__(self): + return f"{self.name}(user: {self.user_id}, text: [{self.text}], language: {self.language}, timestamp: {self.timestamp})" + + @dataclass class OpenAILLMContextAssistantTimestampFrame(DataFrame): """Timestamp information for assistant message in LLM context.""" diff --git a/src/pipecat/services/gladia/stt.py b/src/pipecat/services/gladia/stt.py index df8038bd3..f27664a25 100644 --- a/src/pipecat/services/gladia/stt.py +++ b/src/pipecat/services/gladia/stt.py @@ -20,6 +20,7 @@ from pipecat.frames.frames import ( InterimTranscriptionFrame, StartFrame, TranscriptionFrame, + TranslationFrame, ) from pipecat.services.gladia.config import GladiaInputParams from pipecat.services.stt_service import STTService @@ -384,16 +385,31 @@ class GladiaSTTService(STTService): if content["type"] == "transcript": utterance = content["data"]["utterance"] confidence = utterance.get("confidence", 0) + language = utterance["language"] transcript = utterance["text"] if confidence >= self._confidence: if content["data"]["is_final"]: await self.push_frame( - TranscriptionFrame(transcript, "", time_now_iso8601()) + TranscriptionFrame(transcript, "", time_now_iso8601(), language) ) else: await self.push_frame( - InterimTranscriptionFrame(transcript, "", time_now_iso8601()) + InterimTranscriptionFrame( + transcript, "", time_now_iso8601(), language + ) ) + elif content["type"] == "translation": + translated_utterance = content["data"]["translated_utterance"] + original_language = content["data"]["original_language"] + translated_language = translated_utterance["language"] + confidence = translated_utterance.get("confidence", 0) + translation = translated_utterance["text"] + if translated_language != original_language and confidence >= self._confidence: + await self.push_frame( + TranslationFrame( + translation, "", time_now_iso8601(), translated_language + ) + ) except websockets.exceptions.ConnectionClosed: # Expected when closing the connection pass