From bb00dbefbcf6b88bc21f83ac1617a40148285613 Mon Sep 17 00:00:00 2001
From: Mark Backman <mark@daily.co>
Date: Tue, 15 Oct 2024 14:08:27 -0400
Subject: [PATCH] More edge case handling for text filtering

---
 src/pipecat/utils/text/markdown_text_filter.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/pipecat/utils/text/markdown_text_filter.py b/src/pipecat/utils/text/markdown_text_filter.py
index 76c739c4b..2344d3779 100644
--- a/src/pipecat/utils/text/markdown_text_filter.py
+++ b/src/pipecat/utils/text/markdown_text_filter.py
@@ -40,8 +40,8 @@ class MarkdownTextFilter(BaseTextFilter):
 
     def filter(self, text: str) -> str:
         if self._settings.enable_text_filter:
-            # Remove newlines only when there's no text before or after
-            filtered_text = re.sub(r"^\s*\n", "", text, flags=re.MULTILINE)
+            # Remove newlines and replace with a space only when there's no text before or after
+            filtered_text = re.sub(r"^\s*\n", " ", text, flags=re.MULTILINE)
 
             # Remove backticks from inline code, but not from code blocks
             filtered_text = re.sub(r"(?<!`)`([^`\n]+)`(?!`)", r"\1", filtered_text)
@@ -58,6 +58,10 @@ class MarkdownTextFilter(BaseTextFilter):
                 r"^( +)|\s+$", lambda m: "§" * len(m.group(0)), filtered_text, flags=re.MULTILINE
             )
 
+            # Remove space placeholders before tables, so that tables are converted to HTML
+            # correctly
+            filtered_text = re.sub(r"§\| ", "| ", filtered_text)
+
             # Convert markdown to HTML
             extension = ["tables"] if self._settings.filter_tables else []
             md = Markdown(extensions=extension)