refactor(dataset): 更新数据集加载路径和处理代码

2025-07-15 14:06:18 +08:00
parent e77fb56c45
commit 6edc31ce2e
2 changed files with 1 additions and 2 deletions
@@ -66,7 +66,6 @@ def process_dataset(
    column_name: str = "text",
    normalization_func=comprehensive_normalization,
 ):
-
    train_dataset = dataset_dict[split_name]
    total_samples = len(train_dataset)
    num_chunks = (total_samples // chunk_size) + 1