Merge pull request #528 from pipecat-ai/khk/sentence-splits

TTS sentence aggregation fix

examples/foundational/07a-interruptible-anthropic.py

@@ -5,29 +5,24 @@
 #
 
 import asyncio
-import aiohttp
 import os
 import sys
 
+import aiohttp
+from dotenv import load_dotenv
+from loguru import logger
+from runner import configure
+
 from pipecat.frames.frames import LLMMessagesFrame
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineParams, PipelineTask
-from pipecat.processors.aggregators.llm_response import (
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
-    LLMAssistantResponseAggregator,
-    LLMUserResponseAggregator,
-)
-from pipecat.services.cartesia import CartesiaTTSService
 from pipecat.services.anthropic import AnthropicLLMService
+from pipecat.services.cartesia import CartesiaTTSService
 from pipecat.transports.services.daily import DailyParams, DailyTransport
 from pipecat.vad.silero import SileroVADAnalyzer
 
-from runner import configure
-
-from loguru import logger
-
-from dotenv import load_dotenv
-
 load_dotenv(override=True)
 
 logger.remove(0)
@@ -69,17 +64,17 @@ async def main():
             },
         ]
 
-        tma_in = LLMUserResponseAggregator(messages)
+        context = OpenAILLMContext(messages)
-        tma_out = LLMAssistantResponseAggregator(messages)
+        context_aggregator = llm.create_context_aggregator(context)
 
         pipeline = Pipeline(
             [
                 transport.input(),  # Transport user input
-                tma_in,  # User responses
+                context_aggregator.user(),  # User responses
                 llm,  # LLM
                 tts,  # TTS
                 transport.output(),  # Transport bot output
-                tma_out,  # Assistant spoken responses
+                context_aggregator.assistant(),  # Assistant spoken responses
             ]
         )
 

src/pipecat/services/ai_services.py

@@ -297,16 +297,18 @@ class TTSService(AIService):
             text = frame.text
         else:
             self._current_sentence += frame.text
-            if match_endofsentence(self._current_sentence):
+            eos_end_marker = match_endofsentence(self._current_sentence)
-                text = self._current_sentence
+            if eos_end_marker:
-                self._current_sentence = ""
+                text = self._current_sentence[:eos_end_marker]
+                self._current_sentence = self._current_sentence[eos_end_marker:]
 
         if text:
             await self._push_tts_frames(text)
 
     async def _push_tts_frames(self, text: str):
-        text = text.strip()
+        # Don't send only whitespace. This causes problems for some TTS models. But also don't
-        if not text:
+        # strip all whitespace, as whitespace can influence prosody.
+        if not text.strip():
             return
 
         await self.start_processing_metrics()

src/pipecat/utils/string.py

@@ -6,7 +6,6 @@
 
 import re
 
-
 ENDOFSENTENCE_PATTERN_STR = r"""
     (?<![A-Z])       # Negative lookbehind: not preceded by an uppercase letter (e.g., "U.S.A.")
     (?<!\d)          # Negative lookbehind: not preceded by a digit (e.g., "1. Let's start")
@@ -21,5 +20,6 @@ ENDOFSENTENCE_PATTERN_STR = r"""
 ENDOFSENTENCE_PATTERN = re.compile(ENDOFSENTENCE_PATTERN_STR, re.VERBOSE)
 
 
-def match_endofsentence(text: str) -> bool:
+def match_endofsentence(text: str) -> int:
-    return ENDOFSENTENCE_PATTERN.search(text.rstrip()) is not None
+    match = ENDOFSENTENCE_PATTERN.search(text.rstrip())
+    return match.end() if match else 0