Add table filtering

CHANGELOG.md

@@ -10,7 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Added new input params to the `MarkdownTextFilter` utility. You can set
-  `filter_code` to filter code from text.
+  `filter_code` to filter code from text and `filter_tables` to filter tables
+  from text.
 
 ## [0.0.43] - 2024-10-10
 

src/pipecat/utils/text/markdown_text_filter.py

@@ -24,11 +24,13 @@ class MarkdownTextFilter(BaseTextFilter):
     class InputParams(BaseModel):
         enable_text_filter: bool = True
         filter_code: bool = False
+        filter_tables: bool = False
 
     def __init__(self, params: InputParams = InputParams(), **kwargs):
         super().__init__(**kwargs)
         self._settings = params
         self._in_code_block = False
+        self._in_table = False
         self._interrupted = False
 
     def update_settings(self, settings: Mapping[str, Any]):
@@ -38,30 +40,35 @@ class MarkdownTextFilter(BaseTextFilter):
 
     def filter(self, text: str) -> str:
         if self._settings.enable_text_filter:
-            # Replace newlines with spaces only when there's no text before or after
+            # Remove newlines only when there's no text before or after
-            text = re.sub(r"^\s*\n", " ", text, flags=re.MULTILINE)
+            filtered_text = re.sub(r"^\s*\n", "", text, flags=re.MULTILINE)
 
             # Remove backticks from inline code, but not from code blocks
-            text = re.sub(r"(?<!`)`([^`\n]+)`(?!`)", r"\1", text)
+            filtered_text = re.sub(r"(?<!`)`([^`\n]+)`(?!`)", r"\1", filtered_text)
 
             # Remove repeated sequences of 5 or more characters
-            text = re.sub(r"(\S)(\1{4,})", "", text)
+            filtered_text = re.sub(r"(\S)(\1{4,})", "", filtered_text)
 
             # Preserve numbered list items with a unique marker, §NUM§
-            text = re.sub(r"^(\d+\.)\s", r"§NUM§\1 ", text)
+            filtered_text = re.sub(r"^(\d+\.)\s", r"§NUM§\1 ", filtered_text)
 
             # Preserve leading/trailing spaces with a unique marker, §
             # Critical for word-by-word streaming in bot-tts-text
-            preserved_markdown = re.sub(
+            filtered_text = re.sub(
-                r"^( +)|\s+$", lambda m: "§" * len(m.group(0)), text, flags=re.MULTILINE
+                r"^( +)|\s+$", lambda m: "§" * len(m.group(0)), filtered_text, flags=re.MULTILINE
             )
 
             # Convert markdown to HTML
-            md = Markdown()
+            extension = ["tables"] if self._settings.filter_tables else []
-            html = md.convert(preserved_markdown)
+            md = Markdown(extensions=extension)
+            filtered_text = md.convert(filtered_text)
+
+            # Remove tables
+            if self._settings.filter_tables:
+                filtered_text = self.remove_tables(filtered_text)
 
             # Remove HTML tags
-            filtered_text = re.sub("<[^<]+?>", "", html)
+            filtered_text = re.sub("<[^<]+?>", "", filtered_text)
 
             # Replace HTML entities
             filtered_text = filtered_text.replace("&nbsp;", " ")
@@ -89,17 +96,22 @@ class MarkdownTextFilter(BaseTextFilter):
             # Restore leading and trailing spaces
             filtered_text = re.sub("§", " ", filtered_text)
 
-            return filtered_text
+            return filtered_text or " "  # Return a space to satisfy TTS services
         else:
             return text
 
     def handle_interruption(self):
         self._interrupted = True
         self._in_code_block = False
+        self._in_table = False
 
     def reset_interruption(self):
         self._interrupted = False
 
+    #
+    # Filter code
+    #
+
     def remove_code_blocks(self, text: str) -> str:
         """
         Main method to remove code blocks from the input text.
@@ -140,7 +152,6 @@ class MarkdownTextFilter(BaseTextFilter):
         start_index = match.start()
         if start_index == 0 or text[:start_index].isspace():
             return self._handle_start_of_code_block(text, start_index)
-
         return self._handle_code_block_within_text(text, code_block_pattern)
 
     def _handle_start_of_code_block(self, text, start_index):
@@ -162,3 +173,40 @@ class MarkdownTextFilter(BaseTextFilter):
             return (parts[0] + " " + parts[-1]).strip()
         self._in_code_block = True
         return parts[0].strip()
+
+    #
+    # Filter tables
+    #
+    def remove_tables(self, text: str) -> str:
+        """
+        Remove tables from the input text, handling cases where
+        both start and end tags are in the same input.
+        """
+        if self._interrupted:
+            self._in_table = False
+            return text
+
+        # Pattern to match entire table or parts of it
+        table_pattern = r"<table>.*?</table>"
+        partial_table_start = r"<table>.*"
+        partial_table_end = r".*</table>"
+
+        # Remove complete tables
+        text = re.sub(table_pattern, "", text, flags=re.DOTALL | re.IGNORECASE)
+
+        # Handle partial tables at the start
+        if self._in_table:
+            match = re.match(partial_table_end, text, re.DOTALL | re.IGNORECASE)
+            if match:
+                self._in_table = False
+                return text[match.end() :].strip()
+            else:
+                return ""  # Still inside a table, remove all content
+
+        # Handle partial tables at the end
+        match = re.search(partial_table_start, text, re.DOTALL | re.IGNORECASE)
+        if match:
+            self._in_table = True
+            return text[: match.start()].strip()
+
+        return text.strip()