import re from typing import Dict, Optional class TextNormalizer: """Text normalization.""" DEFAULT_REPLACEMENTS = { "\\[": "$$", "\\]": "$$", "\\(": "$", "\\)": "$", '\u2018': "'", '\u2019': "'", '\u0060': "'", '\u201C': '"', '\u201D': '"', '\u2013': '-', '\u2014': '--', '\u2212': '-', '\u00A0': ' ', '\u2026': '...' } def __init__(self, custom_rules: Optional[Dict[str, str]] = None): self.replacements = {**self.DEFAULT_REPLACEMENTS, **(custom_rules or {})} self._pattern = re.compile('|'.join(re.escape(k) for k in self.replacements)) def normalize(self, text: str) -> str: return self._pattern.sub(lambda m: self.replacements[m.group()], text)