From 84705427c56ef21dabb9301f8c048caf6209c2ff Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 11 Oct 2024 10:23:34 -0400 Subject: [PATCH 1/6] Add code filtering option to MarkdownTextFilter --- src/pipecat/services/ai_services.py | 3 + src/pipecat/utils/text/base_text_filter.py | 8 ++ .../utils/text/markdown_text_filter.py | 80 +++++++++++++++++++ 3 files changed, 91 insertions(+) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 20a587912..b5ef17b48 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -294,6 +294,8 @@ class TTSService(AIService): async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): self._current_sentence = "" + if self._text_filter: + self._text_filter.handle_interruption() await self.push_frame(frame, direction) async def _process_text_frame(self, frame: TextFrame): @@ -318,6 +320,7 @@ class TTSService(AIService): await self.start_processing_metrics() if self._text_filter: + self._text_filter.reset_interruption() text = self._text_filter.filter(text) await self.process_generator(self.run_tts(text)) await self.stop_processing_metrics() diff --git a/src/pipecat/utils/text/base_text_filter.py b/src/pipecat/utils/text/base_text_filter.py index 69d5d4fe1..4e814d7f1 100644 --- a/src/pipecat/utils/text/base_text_filter.py +++ b/src/pipecat/utils/text/base_text_filter.py @@ -16,3 +16,11 @@ class BaseTextFilter(ABC): @abstractmethod def filter(self, text: str) -> str: pass + + @abstractmethod + def handle_interruption(self): + pass + + @abstractmethod + def reset_interruption(self): + pass diff --git a/src/pipecat/utils/text/markdown_text_filter.py b/src/pipecat/utils/text/markdown_text_filter.py index 3018b8788..d1f5d0a5b 100644 --- a/src/pipecat/utils/text/markdown_text_filter.py +++ b/src/pipecat/utils/text/markdown_text_filter.py @@ -23,10 +23,13 @@ class MarkdownTextFilter(BaseTextFilter): class InputParams(BaseModel): enable_text_filter: bool = True + filter_code: bool = False def __init__(self, params: InputParams = InputParams(), **kwargs): super().__init__(**kwargs) self._settings = params + self._in_code_block = False + self._interrupted = False def update_settings(self, settings: Mapping[str, Any]): for key, value in settings.items(): @@ -38,6 +41,9 @@ class MarkdownTextFilter(BaseTextFilter): # Replace newlines with spaces only when there's no text before or after text = re.sub(r"^\s*\n", " ", text, flags=re.MULTILINE) + # Remove backticks from inline code, but not from code blocks + text = re.sub(r"(? str: + """ + Main method to remove code blocks from the input text. + Handles interruptions and delegates to specific methods based on the current state. + """ + if self._interrupted: + self._in_code_block = False + return text + + # Pattern to match three consecutive backticks (code block delimiter) + code_block_pattern = r"```" + match = re.search(code_block_pattern, text) + + if self._in_code_block: + return self._handle_in_code_block(match, text) + + return self._handle_not_in_code_block(match, text, code_block_pattern) + + def _handle_in_code_block(self, match, text): + """ + Handle text when we're currently inside a code block. + If we find the end of the block, return text after it. Otherwise, skip the content. + """ + if match: + self._in_code_block = False + end_index = match.end() + return text[end_index:].strip() + return " " # Skip content inside code block + + def _handle_not_in_code_block(self, match, text, code_block_pattern): + """ + Handle text when we're not currently inside a code block. + Delegate to specific methods based on whether we find a code block delimiter. + """ + if not match: + return text # No code block found, return original text + + start_index = match.start() + if start_index == 0 or text[:start_index].isspace(): + return self._handle_start_of_code_block(text, start_index) + + return self._handle_code_block_within_text(text, code_block_pattern) + + def _handle_start_of_code_block(self, text, start_index): + """ + Handle the case where we find the start of a code block. + Return any text before the code block and set the state to inside a code block. + """ + self._in_code_block = True + return text[:start_index].strip() + + def _handle_code_block_within_text(self, text, code_block_pattern): + """ + Handle the case where we find a code block within the text. + If it's a complete code block, remove it and return surrounding text. + If it's the start of a code block, return text before it and set state. + """ + parts = re.split(code_block_pattern, text) + if len(parts) > 2: + return (parts[0] + " " + parts[-1]).strip() + self._in_code_block = True + return parts[0].strip() From d10c7ac7ce018c32fd6ecb3a96572c514c8b477f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 11 Oct 2024 13:28:34 -0400 Subject: [PATCH 2/6] Add Changelog entry --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5a8b9dfc..b700e6110 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Added new input params to the `MarkdownTextFilter` utility. You can set + `filter_code` to filter code from text. + ## [0.0.43] - 2024-10-10 ### Added From 74d47b725fb0804906a0c4cd4cd33a80a113ba94 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 11 Oct 2024 14:10:47 -0400 Subject: [PATCH 3/6] Add table filtering --- CHANGELOG.md | 3 +- .../utils/text/markdown_text_filter.py | 72 +++++++++++++++---- 2 files changed, 62 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b700e6110..1f761a5f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added new input params to the `MarkdownTextFilter` utility. You can set - `filter_code` to filter code from text. + `filter_code` to filter code from text and `filter_tables` to filter tables + from text. ## [0.0.43] - 2024-10-10 diff --git a/src/pipecat/utils/text/markdown_text_filter.py b/src/pipecat/utils/text/markdown_text_filter.py index d1f5d0a5b..d7203e6e7 100644 --- a/src/pipecat/utils/text/markdown_text_filter.py +++ b/src/pipecat/utils/text/markdown_text_filter.py @@ -24,11 +24,13 @@ class MarkdownTextFilter(BaseTextFilter): class InputParams(BaseModel): enable_text_filter: bool = True filter_code: bool = False + filter_tables: bool = False def __init__(self, params: InputParams = InputParams(), **kwargs): super().__init__(**kwargs) self._settings = params self._in_code_block = False + self._in_table = False self._interrupted = False def update_settings(self, settings: Mapping[str, Any]): @@ -38,30 +40,35 @@ class MarkdownTextFilter(BaseTextFilter): def filter(self, text: str) -> str: if self._settings.enable_text_filter: - # Replace newlines with spaces only when there's no text before or after - text = re.sub(r"^\s*\n", " ", text, flags=re.MULTILINE) + # Remove newlines only when there's no text before or after + filtered_text = re.sub(r"^\s*\n", "", text, flags=re.MULTILINE) # Remove backticks from inline code, but not from code blocks - text = re.sub(r"(?", "", html) + filtered_text = re.sub("<[^<]+?>", "", filtered_text) # Replace HTML entities filtered_text = filtered_text.replace(" ", " ") @@ -89,17 +96,22 @@ class MarkdownTextFilter(BaseTextFilter): # Restore leading and trailing spaces filtered_text = re.sub("§", " ", filtered_text) - return filtered_text + return filtered_text or " " # Return a space to satisfy TTS services else: return text def handle_interruption(self): self._interrupted = True self._in_code_block = False + self._in_table = False def reset_interruption(self): self._interrupted = False + # + # Filter code + # + def remove_code_blocks(self, text: str) -> str: """ Main method to remove code blocks from the input text. @@ -140,7 +152,6 @@ class MarkdownTextFilter(BaseTextFilter): start_index = match.start() if start_index == 0 or text[:start_index].isspace(): return self._handle_start_of_code_block(text, start_index) - return self._handle_code_block_within_text(text, code_block_pattern) def _handle_start_of_code_block(self, text, start_index): @@ -162,3 +173,40 @@ class MarkdownTextFilter(BaseTextFilter): return (parts[0] + " " + parts[-1]).strip() self._in_code_block = True return parts[0].strip() + + # + # Filter tables + # + def remove_tables(self, text: str) -> str: + """ + Remove tables from the input text, handling cases where + both start and end tags are in the same input. + """ + if self._interrupted: + self._in_table = False + return text + + # Pattern to match entire table or parts of it + table_pattern = r".*?
" + partial_table_start = r".*" + partial_table_end = r".*
" + + # Remove complete tables + text = re.sub(table_pattern, "", text, flags=re.DOTALL | re.IGNORECASE) + + # Handle partial tables at the start + if self._in_table: + match = re.match(partial_table_end, text, re.DOTALL | re.IGNORECASE) + if match: + self._in_table = False + return text[match.end() :].strip() + else: + return "" # Still inside a table, remove all content + + # Handle partial tables at the end + match = re.search(partial_table_start, text, re.DOTALL | re.IGNORECASE) + if match: + self._in_table = True + return text[: match.start()].strip() + + return text.strip() From d9c900f872a44e9f7e4583f1131d353bfdf127a7 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 12 Oct 2024 21:27:37 -0400 Subject: [PATCH 4/6] Satisfy minimal text requirements for Cartesia and OpenAI --- src/pipecat/services/cartesia.py | 4 ++-- src/pipecat/services/openai.py | 2 +- src/pipecat/utils/text/markdown_text_filter.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 5166a3df8..c0e4a89a2 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -152,7 +152,7 @@ class CartesiaTTSService(WordTTSService): voice_config["__experimental_controls"]["emotion"] = self._settings["emotion"] msg = { - "transcript": text, + "transcript": text or " ", # Text must contain at least one character "continue": continue_transcript, "context_id": self._context_id, "model_id": self.model_name, @@ -271,7 +271,7 @@ class CartesiaTTSService(WordTTSService): yield TTSStartedFrame() self._context_id = str(uuid.uuid4()) - msg = self._build_msg(text=text) + msg = self._build_msg(text=text or " ") # Text must contain at least one character try: await self._get_websocket().send(msg) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 2f98a7e10..f530d924b 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -409,7 +409,7 @@ class OpenAITTSService(TTSService): await self.start_ttfb_metrics() async with self._client.audio.speech.with_streaming_response.create( - input=text, + input=text or " ", # Text must contain at least one character model=self.model_name, voice=VALID_VOICES[self._voice_id], response_format="pcm", diff --git a/src/pipecat/utils/text/markdown_text_filter.py b/src/pipecat/utils/text/markdown_text_filter.py index d7203e6e7..d91922e32 100644 --- a/src/pipecat/utils/text/markdown_text_filter.py +++ b/src/pipecat/utils/text/markdown_text_filter.py @@ -96,7 +96,7 @@ class MarkdownTextFilter(BaseTextFilter): # Restore leading and trailing spaces filtered_text = re.sub("§", " ", filtered_text) - return filtered_text or " " # Return a space to satisfy TTS services + return filtered_text else: return text From c26a45721f595e6919338b7e0bb8616c6f65953a Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 12 Oct 2024 21:52:56 -0400 Subject: [PATCH 5/6] Set inputs as Optional --- src/pipecat/utils/text/markdown_text_filter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pipecat/utils/text/markdown_text_filter.py b/src/pipecat/utils/text/markdown_text_filter.py index d91922e32..1a8f25553 100644 --- a/src/pipecat/utils/text/markdown_text_filter.py +++ b/src/pipecat/utils/text/markdown_text_filter.py @@ -5,7 +5,7 @@ # import re -from typing import Any, Mapping +from typing import Any, Mapping, Optional from markdown import Markdown from pydantic import BaseModel @@ -22,9 +22,9 @@ class MarkdownTextFilter(BaseTextFilter): """ class InputParams(BaseModel): - enable_text_filter: bool = True - filter_code: bool = False - filter_tables: bool = False + enable_text_filter: Optional[bool] = True + filter_code: Optional[bool] = False + filter_tables: Optional[bool] = False def __init__(self, params: InputParams = InputParams(), **kwargs): super().__init__(**kwargs) From b0890b1f753e8dfe43af24058fdca7b40a41e67d Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 14 Oct 2024 12:52:16 -0400 Subject: [PATCH 6/6] Code review fixes --- src/pipecat/utils/text/markdown_text_filter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipecat/utils/text/markdown_text_filter.py b/src/pipecat/utils/text/markdown_text_filter.py index 1a8f25553..76c739c4b 100644 --- a/src/pipecat/utils/text/markdown_text_filter.py +++ b/src/pipecat/utils/text/markdown_text_filter.py @@ -88,7 +88,7 @@ class MarkdownTextFilter(BaseTextFilter): # Remove code blocks if self._settings.filter_code: - filtered_text = self.remove_code_blocks(filtered_text) + filtered_text = self._remove_code_blocks(filtered_text) # Restore numbered list items filtered_text = filtered_text.replace("§NUM§", "") @@ -112,7 +112,7 @@ class MarkdownTextFilter(BaseTextFilter): # Filter code # - def remove_code_blocks(self, text: str) -> str: + def _remove_code_blocks(self, text: str) -> str: """ Main method to remove code blocks from the input text. Handles interruptions and delegates to specific methods based on the current state. @@ -139,7 +139,7 @@ class MarkdownTextFilter(BaseTextFilter): self._in_code_block = False end_index = match.end() return text[end_index:].strip() - return " " # Skip content inside code block + return "" # Skip content inside code block def _handle_not_in_code_block(self, match, text, code_block_pattern): """