wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More edge case improvements

Browse Source
This commit is contained in:
Mark Backman
2024-10-10 12:07:00 -04:00
parent e32e56d0bc
commit 3c5f9457f1
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/pipecat/processors/text/markdown_remover.py
View File

@@ -34,8 +34,11 @@ class MarkdownRemovalProcessor(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
def _remove_markdown(self, markdown_string: str) -> str: def _remove_markdown(self, markdown_string: str) -> str:
# Replace newlines with spaces to handle cases with leading newlines # Replace newlines with spaces only when there's no text before or after
markdown_string = markdown_string.replace("\n", " ") markdown_string = re.sub(r"^\s*\n", "", markdown_string, flags=re.MULTILINE)
# Remove repeated sequences of 5 or more characters
markdown_string = re.sub(r"(\S)(\1{4,})", "", markdown_string)
# Preserve numbered list items with a unique marker, §NUM§ # Preserve numbered list items with a unique marker, §NUM§
markdown_string = re.sub(r"^(\d+\.)\s", r"§NUM§\1 ", markdown_string) markdown_string = re.sub(r"^(\d+\.)\s", r"§NUM§\1 ", markdown_string)