From f733e774967d4e6e2267b39aa25e3f35d9a0716f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 19 Jan 2026 09:13:41 -0500 Subject: [PATCH 1/3] AzureTTS: work around word ordering issue at 8khz sample rate --- src/pipecat/services/azure/tts.py | 38 +++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/pipecat/services/azure/tts.py b/src/pipecat/services/azure/tts.py index 93c421c1e..b9733db07 100644 --- a/src/pipecat/services/azure/tts.py +++ b/src/pipecat/services/azure/tts.py @@ -277,6 +277,11 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): self._started = False self._first_chunk = True self._cumulative_audio_offset: float = 0.0 # Cumulative audio duration in seconds + self._current_sentence_base_offset: float = 0.0 # Base offset for current sentence + self._current_sentence_duration: float = 0.0 # Duration from Azure callback + self._current_sentence_max_word_offset: float = ( + 0.0 # Max word boundary offset seen in current sentence (for 8kHz workaround) + ) self._last_word: Optional[str] = None # Track last word for punctuation merging self._last_timestamp: Optional[float] = None # Track last timestamp @@ -386,8 +391,14 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): word = evt.text sentence_relative_seconds = evt.audio_offset / 10_000_000.0 - # Add cumulative offset to get absolute timestamp across sentences - absolute_seconds = self._cumulative_audio_offset + sentence_relative_seconds + # Use base offset captured at start of run_tts to avoid race conditions + # with callbacks from overlapping TTS requests + absolute_seconds = self._current_sentence_base_offset + sentence_relative_seconds + + # Track max word offset for accurate cumulative timing + # (audio_duration from Azure doesn't always match word boundary offsets at 8kHz) + if sentence_relative_seconds > self._current_sentence_max_word_offset: + self._current_sentence_max_word_offset = sentence_relative_seconds if not word: return @@ -492,9 +503,9 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): self._last_word = None self._last_timestamp = None - # Update cumulative audio offset for next sentence + # Store duration for cumulative offset calculation if evt.result and evt.result.audio_duration: - self._cumulative_audio_offset += evt.result.audio_duration.total_seconds() + self._current_sentence_duration = evt.result.audio_duration.total_seconds() self._audio_queue.put_nowait(None) # Signal completion @@ -530,6 +541,9 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): self._started = False self._first_chunk = True self._cumulative_audio_offset = 0.0 + self._current_sentence_base_offset = 0.0 + self._current_sentence_duration = 0.0 + self._current_sentence_max_word_offset = 0.0 self._last_word = None self._last_timestamp = None @@ -604,6 +618,12 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): self._started = True self._first_chunk = True + # Capture base offset BEFORE starting synthesis to avoid race conditions + # Word boundary callbacks will use this value + self._current_sentence_base_offset = self._cumulative_audio_offset + self._current_sentence_duration = 0.0 + self._current_sentence_max_word_offset = 0.0 + ssml = self._construct_ssml(text) self._speech_synthesizer.speak_ssml_async(ssml) await self.start_tts_usage_metrics(text) @@ -627,6 +647,16 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService): ) yield frame + # Update cumulative offset for next sentence + # At 8kHz, Azure's audio_duration doesn't match word boundary offsets, + # so we use max_word_offset as a workaround. At other sample rates, + # audio_duration is accurate. + # TODO: Remove after Azure fixes word boundary timing at 8kHz + if self.sample_rate == 8000: + self._cumulative_audio_offset += self._current_sentence_max_word_offset + else: + self._cumulative_audio_offset += self._current_sentence_duration + except Exception as e: yield ErrorFrame(error=f"Unknown error occurred: {e}") yield TTSStoppedFrame() From 14bd3b1b32e4486a1eadf1d92e290916b20ef867 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 19 Jan 2026 09:19:57 -0500 Subject: [PATCH 2/3] Set Azure TTS default prosody rate to None --- src/pipecat/services/azure/tts.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/pipecat/services/azure/tts.py b/src/pipecat/services/azure/tts.py index b9733db07..dc0c56e7b 100644 --- a/src/pipecat/services/azure/tts.py +++ b/src/pipecat/services/azure/tts.py @@ -90,7 +90,7 @@ class AzureBaseTTSService: emphasis: Emphasis level for speech ("strong", "moderate", "reduced"). language: Language for synthesis. Defaults to English (US). pitch: Voice pitch adjustment (e.g., "+10%", "-5Hz", "high"). - rate: Speech rate multiplier. Defaults to "1.05". + rate: Speech rate adjustment (e.g., "1.0", "1.25", "slow", "fast"). role: Voice role for expression (e.g., "YoungAdultFemale"). style: Speaking style (e.g., "cheerful", "sad", "excited"). style_degree: Intensity of the speaking style (0.01 to 2.0). @@ -100,7 +100,7 @@ class AzureBaseTTSService: emphasis: Optional[str] = None language: Optional[Language] = Language.EN_US pitch: Optional[str] = None - rate: Optional[str] = "1.05" + rate: Optional[str] = None role: Optional[str] = None style: Optional[str] = None style_degree: Optional[str] = None @@ -185,7 +185,9 @@ class AzureBaseTTSService: if self._settings["volume"]: prosody_attrs.append(f"volume='{self._settings['volume']}'") - ssml += f"" + # Only wrap in prosody tag if there are prosody attributes + if prosody_attrs: + ssml += f"" if self._settings["emphasis"]: ssml += f"" @@ -195,7 +197,8 @@ class AzureBaseTTSService: if self._settings["emphasis"]: ssml += "" - ssml += "" + if prosody_attrs: + ssml += "" if self._settings["style"]: ssml += "" From 0b1a4792b8d2971e8fa97576e35a413e48ceff27 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 19 Jan 2026 09:51:39 -0500 Subject: [PATCH 3/3] Bump to latest azure-cognitiveservices-speech version, 1.47.0 --- pyproject.toml | 2 +- uv.lock | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e99ab62bc..0cb985f34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ assemblyai = [ "pipecat-ai[websockets-base]" ] asyncai = [ "pipecat-ai[websockets-base]" ] aws = [ "aioboto3~=15.5.0", "pipecat-ai[websockets-base]" ] aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.2.0; python_version>='3.12'" ] -azure = [ "azure-cognitiveservices-speech~=1.44.0"] +azure = [ "azure-cognitiveservices-speech~=1.47.0"] cartesia = [ "cartesia~=2.0.3", "pipecat-ai[websockets-base]" ] camb = [ "camb-sdk>=1.5.4" ] cerebras = [] diff --git a/uv.lock b/uv.lock index 45c1f892f..0ca04667a 100644 --- a/uv.lock +++ b/uv.lock @@ -531,18 +531,18 @@ wheels = [ [[package]] name = "azure-cognitiveservices-speech" -version = "1.44.0" +version = "1.47.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "azure-core" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/0d/0752835f079e8d2cc42bb634f3ccd761c8d6e9d0d46a2d6cf7b3ed8e714c/azure_cognitiveservices_speech-1.44.0-py3-none-macosx_10_14_x86_64.whl", hash = "sha256:78037a147ba72abb57e8c10b693d43a1bb029986fae0918f1f9b7d6342737bfe", size = 7492396, upload-time = "2025-05-19T15:46:11.318Z" }, - { url = "https://files.pythonhosted.org/packages/76/1d/d0ed4ec0f51303a2a532dc845eeb72c7729a3c8639b08050f3c1cd96db79/azure_cognitiveservices_speech-1.44.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:2c9b436326cd8dd82dfa88454b7b68359dfc7149e2ac9029f9bcff155ebd5c95", size = 7347577, upload-time = "2025-05-19T15:46:13.644Z" }, - { url = "https://files.pythonhosted.org/packages/89/c8/f0a4ea8bea014b912046f737e429378ceadad68258395454d62acf7f65bb/azure_cognitiveservices_speech-1.44.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:e5f07fc0587067850288c17aebf33d307d2c1ef9e0b2d11d9f44bff2af400568", size = 40977193, upload-time = "2025-05-19T15:46:15.878Z" }, - { url = "https://files.pythonhosted.org/packages/6a/0d/0a0394e8102d6660afeec6b780c451401f6074b1e19f00e90785529e459e/azure_cognitiveservices_speech-1.44.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:3461e22cf04816f69a964d936218d920240f987c0656fdaaf46571529ff0f7e6", size = 40747860, upload-time = "2025-05-19T15:46:19.316Z" }, - { url = "https://files.pythonhosted.org/packages/55/ad/3b7f6eca73040821358ce01f22067446a03d876bfed41cd784291706db4c/azure_cognitiveservices_speech-1.44.0-py3-none-win32.whl", hash = "sha256:a3fe7fd67ba7db281ae490de3d71b5a22648454ec2630eb6a70797f666330586", size = 2164045, upload-time = "2025-05-19T15:46:22.373Z" }, - { url = "https://files.pythonhosted.org/packages/83/ac/f491487d7d0e25ae2929b4f07e7f9b7456feb38e65b36fb605b2c9685b10/azure_cognitiveservices_speech-1.44.0-py3-none-win_amd64.whl", hash = "sha256:77cfb5dd40733b7ccc21edc427e9fb4720997832ea8a1ba460dc94345f3588ae", size = 2422937, upload-time = "2025-05-19T15:46:23.657Z" }, + { url = "https://files.pythonhosted.org/packages/cc/e3/b6a3d1ef4f135f8ef00ed084b9284e65409e9cd52bc96cd0453a5c6637c6/azure_cognitiveservices_speech-1.47.0-py3-none-macosx_10_14_x86_64.whl", hash = "sha256:656577ed01ed4b8cd7c70fab2c921b300181b906f101758a16406bc99b133681", size = 3574346, upload-time = "2025-11-11T21:13:37.717Z" }, + { url = "https://files.pythonhosted.org/packages/82/fa/9cc0c5400e9d433bd98a1239bedf97b34abf410dbc8932a50886ae43e115/azure_cognitiveservices_speech-1.47.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:afd91653ceca482ccea5459eedda1ec9aa95ee07df12a15fc588c42d4f90f0a9", size = 3506219, upload-time = "2025-11-11T21:13:39.702Z" }, + { url = "https://files.pythonhosted.org/packages/6b/d6/b8f55421b8cb40b478f4fb793c52b1bb0ed794263a5475ae2a6490a4cd53/azure_cognitiveservices_speech-1.47.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:577b702ee30d35ecc581e7e2ac23f4387782f93c241d7f8f3c86f72bb883d02d", size = 35399363, upload-time = "2025-11-11T21:13:41.915Z" }, + { url = "https://files.pythonhosted.org/packages/98/91/c36be146824797f57b194128a173baf289a260c2540c86c166f8c7fbebe3/azure_cognitiveservices_speech-1.47.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:ff72c74abe4b4c0f5a527eabf8511a8c0e689d884a95c54a46495b293e302e73", size = 35196906, upload-time = "2025-11-11T21:13:45.31Z" }, + { url = "https://files.pythonhosted.org/packages/fb/19/dd6f08dc623f2b336cc9cd5cf765712df5262fd675583e701922491e455d/azure_cognitiveservices_speech-1.47.0-py3-none-win_amd64.whl", hash = "sha256:ecfce57d66907afe305fb2950cc781ea8f327274facd2db66950e701b6cfd715", size = 2182376, upload-time = "2025-11-11T21:13:47.753Z" }, + { url = "https://files.pythonhosted.org/packages/1b/16/a6d1f7ab7eae21b00da2eee7186a7db9c9a2434e0ef833f071ff686b833f/azure_cognitiveservices_speech-1.47.0-py3-none-win_arm64.whl", hash = "sha256:4351734cf240d11340a057ecb388397e5ecf40e97e4b67a6a990fffe2791b56c", size = 1978493, upload-time = "2025-11-11T21:13:49.445Z" }, ] [[package]] @@ -4504,7 +4504,7 @@ requires-dist = [ { name = "audioop-lts", marker = "python_full_version >= '3.13'", specifier = "~=0.2.1" }, { name = "aws-sdk-bedrock-runtime", marker = "python_full_version >= '3.12' and extra == 'aws-nova-sonic'", specifier = "~=0.2.0" }, { name = "aws-sdk-sagemaker-runtime-http2", marker = "python_full_version >= '3.12' and extra == 'sagemaker'" }, - { name = "azure-cognitiveservices-speech", marker = "extra == 'azure'", specifier = "~=1.44.0" }, + { name = "azure-cognitiveservices-speech", marker = "extra == 'azure'", specifier = "~=1.47.0" }, { name = "camb-sdk", marker = "extra == 'camb'", specifier = ">=1.5.4" }, { name = "cartesia", marker = "extra == 'cartesia'", specifier = "~=2.0.3" }, { name = "coremltools", marker = "extra == 'local-smart-turn'", specifier = ">=8.0" },