ViperEkura/DataPipeline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(utils): 更新 comprehensive_normalization 函数

Browse Source
This commit is contained in:
ViperEkura
2025-08-25 19:36:59 +08:00
parent be79108f84
commit a976dc931b
1 changed files with 3 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/utils.py
+3 -5
View File
@@ -28,11 +28,9 @@ def fetch_folders(root_dir, filter_func=None):
def comprehensive_normalization(text): def comprehensive_normalization(text):
replacements = { replacements = {
'\u2018': "'", '\u2019': "'", '\u0060': "'", "\\[": "$$", "\\]": "$$", "\\(": "$", "\\)": "$",
'\u201C': '"', '\u201D': '"', '\u2018': "'", '\u2019': "'", '\u0060': "'", '\u201C': '"', '\u201D': '"',
'\u2013': '-', '\u2014': '--', '\u2212': '-', '\u2013': '-', '\u2014': '--', '\u2212': '-', '\u00A0': ' ', '\u2026': '...'
'\u00A0': ' ',
'\u2026': '...'
} }
pattern = re.compile('|'.join(re.escape(k) for k in replacements)) pattern = re.compile('|'.join(re.escape(k) for k in replacements))
return pattern.sub(lambda m: replacements[m.group()], text) return pattern.sub(lambda m: replacements[m.group()], text)