refactor(dump_pt_file): 重构数据处理流程并优化代码结构

2025-07-21 20:27:07 +08:00
parent f9ce67e03c
commit 7f45c701b4
2 changed files with 14 additions and 10 deletions
@@ -25,12 +25,6 @@ def comprehensive_normalization(text):
    pattern = re.compile('|'.join(re.escape(k) for k in replacements))
    return pattern.sub(lambda m: replacements[m.group()], text) 

-def pt_processor(text):
-    text = comprehensive_normalization(text)
-    text = text.lower()
-    return text    
-
-
 def pack_sequences(sequences: List[Tensor], pack_size: int, pad_value: int) -> List[Tensor]:
    packages = []
    sequences.sort(key=lambda x: x.numel(), reverse=True)