wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #594 from pipecat-ai/mb/more-text-filter-massaging

Browse Source 
More edge case handling for text filtering
This commit is contained in:
Mark Backman
2024-10-15 14:51:43 -04:00
committed by GitHub
parent 0c250c0603 bb00dbefbc
commit 79b52d4301
1 changed files with 6 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/pipecat/utils/text/markdown_text_filter.py
View File

@@ -40,8 +40,8 @@ class MarkdownTextFilter(BaseTextFilter):
def filter(self, text: str) -> str:
if self._settings.enable_text_filter:
# Remove newlines only when there's no text before or after
filtered_text = re.sub(r"^\s*\n", "", text, flags=re.MULTILINE)
# Remove newlines and replace with a space only when there's no text before or after
filtered_text = re.sub(r"^\s*\n", " ", text, flags=re.MULTILINE)
# Remove backticks from inline code, but not from code blocks
filtered_text = re.sub(r"(?<!`)`([^`\n]+)`(?!`)", r"\1", filtered_text)
@@ -58,6 +58,10 @@ class MarkdownTextFilter(BaseTextFilter):
r"^( +)|\s+$", lambda m: "§" * len(m.group(0)), filtered_text, flags=re.MULTILINE
)
# Remove space placeholders before tables, so that tables are converted to HTML
# correctly
filtered_text = re.sub(r"§\| ", "| ", filtered_text)
# Convert markdown to HTML
extension = ["tables"] if self._settings.filter_tables else []
md = Markdown(extensions=extension)