fix(dataset): 修正数据集输出路径和名称

2025-07-11 10:55:28 +08:00
parent 9a417ab6a5
commit d4df706bc5
3 changed files with 4 additions and 4 deletions
@@ -33,7 +33,7 @@ if __name__ == "__main__":
        end_idx = min((i + 1) * chunk_size, total_samples)
        chunk = train_dataset.select(range(start_idx, end_idx))
        
-        output_path = f"{output_dir}/{output_dir}text_chunk_{i}.jsonl"
+        output_path = f"{output_dir}/english-fineweb_text_chunk_{i}.jsonl"
        with open(output_path, "w", encoding="utf-8") as f:
            for example in chunk:
                json_line = {"text": comprehensive_normalization(example["text"])}