refactor(utils): 更新 comprehensive_normalization 函数

2025-08-25 19:36:59 +08:00
parent be79108f84
commit a976dc931b
1 changed files with 3 additions and 5 deletions
@@ -28,11 +28,9 @@ def fetch_folders(root_dir, filter_func=None):

 def comprehensive_normalization(text):
    replacements = {
-        '\u2018': "'", '\u2019': "'", '\u0060': "'",
-        '\u201C': '"', '\u201D': '"', 
-        '\u2013': '-', '\u2014': '--', '\u2212': '-',
-        '\u00A0': ' ',
-        '\u2026': '...'
+        "\\[": "$$", "\\]": "$$", "\\(": "$", "\\)": "$",
+        '\u2018': "'", '\u2019': "'", '\u0060': "'", '\u201C': '"', '\u201D': '"', 
+        '\u2013': '-', '\u2014': '--', '\u2212': '-', '\u00A0': ' ', '\u2026': '...'
    }
    pattern = re.compile('|'.join(re.escape(k) for k in replacements))
    return pattern.sub(lambda m: replacements[m.group()], text)