diff --git a/CHANGELOG.md b/CHANGELOG.md index b700e6110..1f761a5f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added new input params to the `MarkdownTextFilter` utility. You can set - `filter_code` to filter code from text. + `filter_code` to filter code from text and `filter_tables` to filter tables + from text. ## [0.0.43] - 2024-10-10 diff --git a/src/pipecat/utils/text/markdown_text_filter.py b/src/pipecat/utils/text/markdown_text_filter.py index d1f5d0a5b..d7203e6e7 100644 --- a/src/pipecat/utils/text/markdown_text_filter.py +++ b/src/pipecat/utils/text/markdown_text_filter.py @@ -24,11 +24,13 @@ class MarkdownTextFilter(BaseTextFilter): class InputParams(BaseModel): enable_text_filter: bool = True filter_code: bool = False + filter_tables: bool = False def __init__(self, params: InputParams = InputParams(), **kwargs): super().__init__(**kwargs) self._settings = params self._in_code_block = False + self._in_table = False self._interrupted = False def update_settings(self, settings: Mapping[str, Any]): @@ -38,30 +40,35 @@ class MarkdownTextFilter(BaseTextFilter): def filter(self, text: str) -> str: if self._settings.enable_text_filter: - # Replace newlines with spaces only when there's no text before or after - text = re.sub(r"^\s*\n", " ", text, flags=re.MULTILINE) + # Remove newlines only when there's no text before or after + filtered_text = re.sub(r"^\s*\n", "", text, flags=re.MULTILINE) # Remove backticks from inline code, but not from code blocks - text = re.sub(r"(?", "", html) + filtered_text = re.sub("<[^<]+?>", "", filtered_text) # Replace HTML entities filtered_text = filtered_text.replace(" ", " ") @@ -89,17 +96,22 @@ class MarkdownTextFilter(BaseTextFilter): # Restore leading and trailing spaces filtered_text = re.sub("§", " ", filtered_text) - return filtered_text + return filtered_text or " " # Return a space to satisfy TTS services else: return text def handle_interruption(self): self._interrupted = True self._in_code_block = False + self._in_table = False def reset_interruption(self): self._interrupted = False + # + # Filter code + # + def remove_code_blocks(self, text: str) -> str: """ Main method to remove code blocks from the input text. @@ -140,7 +152,6 @@ class MarkdownTextFilter(BaseTextFilter): start_index = match.start() if start_index == 0 or text[:start_index].isspace(): return self._handle_start_of_code_block(text, start_index) - return self._handle_code_block_within_text(text, code_block_pattern) def _handle_start_of_code_block(self, text, start_index): @@ -162,3 +173,40 @@ class MarkdownTextFilter(BaseTextFilter): return (parts[0] + " " + parts[-1]).strip() self._in_code_block = True return parts[0].strip() + + # + # Filter tables + # + def remove_tables(self, text: str) -> str: + """ + Remove tables from the input text, handling cases where + both start and end tags are in the same input. + """ + if self._interrupted: + self._in_table = False + return text + + # Pattern to match entire table or parts of it + table_pattern = r".*?
" + partial_table_start = r".*" + partial_table_end = r".*
" + + # Remove complete tables + text = re.sub(table_pattern, "", text, flags=re.DOTALL | re.IGNORECASE) + + # Handle partial tables at the start + if self._in_table: + match = re.match(partial_table_end, text, re.DOTALL | re.IGNORECASE) + if match: + self._in_table = False + return text[match.end() :].strip() + else: + return "" # Still inside a table, remove all content + + # Handle partial tables at the end + match = re.search(partial_table_start, text, re.DOTALL | re.IGNORECASE) + if match: + self._in_table = True + return text[: match.start()].strip() + + return text.strip()